Pileus Git - ~andy/linux/blob - drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include "i915_drv.h"
  30 #include "intel_drv.h"
  31 #include "../../../platform/x86/intel_ips.h"
  32 #include <linux/module.h>
  33 #include <drm/i915_powerwell.h>
  34
  35 /**
  36  * RC6 is a special power stage which allows the GPU to enter an very
  37  * low-voltage mode when idle, using down to 0V while at this stage.  This
  38  * stage is entered automatically when the GPU is idle when RC6 support is
  39  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  40  *
  41  * There are different RC6 modes available in Intel GPU, which differentiate
  42  * among each other with the latency required to enter and leave RC6 and
  43  * voltage consumed by the GPU in different states.
  44  *
  45  * The combination of the following flags define which states GPU is allowed
  46  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  47  * RC6pp is deepest RC6. Their support by hardware varies according to the
  48  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  49  * which brings the most power savings; deeper states save more power, but
  50  * require higher latency to switch to and wake up.
  51  */
  52 #define INTEL_RC6_ENABLE                        (1<<0)
  53 #define INTEL_RC6p_ENABLE                       (1<<1)
  54 #define INTEL_RC6pp_ENABLE                      (1<<2)
  55
  56 /* FBC, or Frame Buffer Compression, is a technique employed to compress the
  57  * framebuffer contents in-memory, aiming at reducing the required bandwidth
  58  * during in-memory transfers and, therefore, reduce the power packet.
  59  *
  60  * The benefits of FBC are mostly visible with solid backgrounds and
  61  * variation-less patterns.
  62  *
  63  * FBC-related functionality can be enabled by the means of the
  64  * i915.i915_enable_fbc parameter
  65  */
  66
  67 static void i8xx_disable_fbc(struct drm_device *dev)
  68 {
  69         struct drm_i915_private *dev_priv = dev->dev_private;
  70         u32 fbc_ctl;
  71
  72         /* Disable compression */
  73         fbc_ctl = I915_READ(FBC_CONTROL);
  74         if ((fbc_ctl & FBC_CTL_EN) == 0)
  75                 return;
  76
  77         fbc_ctl &= ~FBC_CTL_EN;
  78         I915_WRITE(FBC_CONTROL, fbc_ctl);
  79
  80         /* Wait for compressing bit to clear */
  81         if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
  82                 DRM_DEBUG_KMS("FBC idle timed out\n");
  83                 return;
  84         }
  85
  86         DRM_DEBUG_KMS("disabled FBC\n");
  87 }
  88
  89 static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
  90 {
  91         struct drm_device *dev = crtc->dev;
  92         struct drm_i915_private *dev_priv = dev->dev_private;
  93         struct drm_framebuffer *fb = crtc->fb;
  94         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
  95         struct drm_i915_gem_object *obj = intel_fb->obj;
  96         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
  97         int cfb_pitch;
  98         int plane, i;
  99         u32 fbc_ctl, fbc_ctl2;
 100
 101         cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
 102         if (fb->pitches[0] < cfb_pitch)
 103                 cfb_pitch = fb->pitches[0];
 104
 105         /* FBC_CTL wants 64B units */
 106         cfb_pitch = (cfb_pitch / 64) - 1;
 107         plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
 108
 109         /* Clear old tags */
 110         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
 111                 I915_WRITE(FBC_TAG + (i * 4), 0);
 112
 113         /* Set it up... */
 114         fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
 115         fbc_ctl2 |= plane;
 116         I915_WRITE(FBC_CONTROL2, fbc_ctl2);
 117         I915_WRITE(FBC_FENCE_OFF, crtc->y);
 118
 119         /* enable it... */
 120         fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
 121         if (IS_I945GM(dev))
 122                 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
 123         fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
 124         fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
 125         fbc_ctl |= obj->fence_reg;
 126         I915_WRITE(FBC_CONTROL, fbc_ctl);
 127
 128         DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
 129                       cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
 130 }
 131
 132 static bool i8xx_fbc_enabled(struct drm_device *dev)
 133 {
 134         struct drm_i915_private *dev_priv = dev->dev_private;
 135
 136         return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
 137 }
 138
 139 static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 140 {
 141         struct drm_device *dev = crtc->dev;
 142         struct drm_i915_private *dev_priv = dev->dev_private;
 143         struct drm_framebuffer *fb = crtc->fb;
 144         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 145         struct drm_i915_gem_object *obj = intel_fb->obj;
 146         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 147         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
 148         unsigned long stall_watermark = 200;
 149         u32 dpfc_ctl;
 150
 151         dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
 152         dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
 153         I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
 154
 155         I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
 156                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
 157                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
 158         I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
 159
 160         /* enable it... */
 161         I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
 162
 163         DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
 164 }
 165
 166 static void g4x_disable_fbc(struct drm_device *dev)
 167 {
 168         struct drm_i915_private *dev_priv = dev->dev_private;
 169         u32 dpfc_ctl;
 170
 171         /* Disable compression */
 172         dpfc_ctl = I915_READ(DPFC_CONTROL);
 173         if (dpfc_ctl & DPFC_CTL_EN) {
 174                 dpfc_ctl &= ~DPFC_CTL_EN;
 175                 I915_WRITE(DPFC_CONTROL, dpfc_ctl);
 176
 177                 DRM_DEBUG_KMS("disabled FBC\n");
 178         }
 179 }
 180
 181 static bool g4x_fbc_enabled(struct drm_device *dev)
 182 {
 183         struct drm_i915_private *dev_priv = dev->dev_private;
 184
 185         return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
 186 }
 187
 188 static void sandybridge_blit_fbc_update(struct drm_device *dev)
 189 {
 190         struct drm_i915_private *dev_priv = dev->dev_private;
 191         u32 blt_ecoskpd;
 192
 193         /* Make sure blitter notifies FBC of writes */
 194         gen6_gt_force_wake_get(dev_priv);
 195         blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
 196         blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
 197                 GEN6_BLITTER_LOCK_SHIFT;
 198         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
 199         blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
 200         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
 201         blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
 202                          GEN6_BLITTER_LOCK_SHIFT);
 203         I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
 204         POSTING_READ(GEN6_BLITTER_ECOSKPD);
 205         gen6_gt_force_wake_put(dev_priv);
 206 }
 207
 208 static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 209 {
 210         struct drm_device *dev = crtc->dev;
 211         struct drm_i915_private *dev_priv = dev->dev_private;
 212         struct drm_framebuffer *fb = crtc->fb;
 213         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 214         struct drm_i915_gem_object *obj = intel_fb->obj;
 215         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 216         int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
 217         unsigned long stall_watermark = 200;
 218         u32 dpfc_ctl;
 219
 220         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
 221         dpfc_ctl &= DPFC_RESERVED;
 222         dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
 223         /* Set persistent mode for front-buffer rendering, ala X. */
 224         dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
 225         dpfc_ctl |= DPFC_CTL_FENCE_EN;
 226         if (IS_GEN5(dev))
 227                 dpfc_ctl |= obj->fence_reg;
 228         I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
 229
 230         I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
 231                    (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
 232                    (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
 233         I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
 234         I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID);
 235         /* enable it... */
 236         I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 237
 238         if (IS_GEN6(dev)) {
 239                 I915_WRITE(SNB_DPFC_CTL_SA,
 240                            SNB_CPU_FENCE_ENABLE | obj->fence_reg);
 241                 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
 242                 sandybridge_blit_fbc_update(dev);
 243         }
 244
 245         DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
 246 }
 247
 248 static void ironlake_disable_fbc(struct drm_device *dev)
 249 {
 250         struct drm_i915_private *dev_priv = dev->dev_private;
 251         u32 dpfc_ctl;
 252
 253         /* Disable compression */
 254         dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
 255         if (dpfc_ctl & DPFC_CTL_EN) {
 256                 dpfc_ctl &= ~DPFC_CTL_EN;
 257                 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
 258
 259                 DRM_DEBUG_KMS("disabled FBC\n");
 260         }
 261 }
 262
 263 static bool ironlake_fbc_enabled(struct drm_device *dev)
 264 {
 265         struct drm_i915_private *dev_priv = dev->dev_private;
 266
 267         return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
 268 }
 269
 270 static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 271 {
 272         struct drm_device *dev = crtc->dev;
 273         struct drm_i915_private *dev_priv = dev->dev_private;
 274         struct drm_framebuffer *fb = crtc->fb;
 275         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 276         struct drm_i915_gem_object *obj = intel_fb->obj;
 277         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 278
 279         I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj));
 280
 281         I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X |
 282                    IVB_DPFC_CTL_FENCE_EN |
 283                    intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
 284
 285         if (IS_IVYBRIDGE(dev)) {
 286                 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
 287                 I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
 288         } else {
 289                 /* WaFbcAsynchFlipDisableFbcQueue:hsw */
 290                 I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
 291                            HSW_BYPASS_FBC_QUEUE);
 292         }
 293
 294         I915_WRITE(SNB_DPFC_CTL_SA,
 295                    SNB_CPU_FENCE_ENABLE | obj->fence_reg);
 296         I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
 297
 298         sandybridge_blit_fbc_update(dev);
 299
 300         DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
 301 }
 302
 303 bool intel_fbc_enabled(struct drm_device *dev)
 304 {
 305         struct drm_i915_private *dev_priv = dev->dev_private;
 306
 307         if (!dev_priv->display.fbc_enabled)
 308                 return false;
 309
 310         return dev_priv->display.fbc_enabled(dev);
 311 }
 312
 313 static void intel_fbc_work_fn(struct work_struct *__work)
 314 {
 315         struct intel_fbc_work *work =
 316                 container_of(to_delayed_work(__work),
 317                              struct intel_fbc_work, work);
 318         struct drm_device *dev = work->crtc->dev;
 319         struct drm_i915_private *dev_priv = dev->dev_private;
 320
 321         mutex_lock(&dev->struct_mutex);
 322         if (work == dev_priv->fbc.fbc_work) {
 323                 /* Double check that we haven't switched fb without cancelling
 324                  * the prior work.
 325                  */
 326                 if (work->crtc->fb == work->fb) {
 327                         dev_priv->display.enable_fbc(work->crtc,
 328                                                      work->interval);
 329
 330                         dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
 331                         dev_priv->fbc.fb_id = work->crtc->fb->base.id;
 332                         dev_priv->fbc.y = work->crtc->y;
 333                 }
 334
 335                 dev_priv->fbc.fbc_work = NULL;
 336         }
 337         mutex_unlock(&dev->struct_mutex);
 338
 339         kfree(work);
 340 }
 341
 342 static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
 343 {
 344         if (dev_priv->fbc.fbc_work == NULL)
 345                 return;
 346
 347         DRM_DEBUG_KMS("cancelling pending FBC enable\n");
 348
 349         /* Synchronisation is provided by struct_mutex and checking of
 350          * dev_priv->fbc.fbc_work, so we can perform the cancellation
 351          * entirely asynchronously.
 352          */
 353         if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
 354                 /* tasklet was killed before being run, clean up */
 355                 kfree(dev_priv->fbc.fbc_work);
 356
 357         /* Mark the work as no longer wanted so that if it does
 358          * wake-up (because the work was already running and waiting
 359          * for our mutex), it will discover that is no longer
 360          * necessary to run.
 361          */
 362         dev_priv->fbc.fbc_work = NULL;
 363 }
 364
 365 static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
 366 {
 367         struct intel_fbc_work *work;
 368         struct drm_device *dev = crtc->dev;
 369         struct drm_i915_private *dev_priv = dev->dev_private;
 370
 371         if (!dev_priv->display.enable_fbc)
 372                 return;
 373
 374         intel_cancel_fbc_work(dev_priv);
 375
 376         work = kzalloc(sizeof(*work), GFP_KERNEL);
 377         if (work == NULL) {
 378                 DRM_ERROR("Failed to allocate FBC work structure\n");
 379                 dev_priv->display.enable_fbc(crtc, interval);
 380                 return;
 381         }
 382
 383         work->crtc = crtc;
 384         work->fb = crtc->fb;
 385         work->interval = interval;
 386         INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
 387
 388         dev_priv->fbc.fbc_work = work;
 389
 390         /* Delay the actual enabling to let pageflipping cease and the
 391          * display to settle before starting the compression. Note that
 392          * this delay also serves a second purpose: it allows for a
 393          * vblank to pass after disabling the FBC before we attempt
 394          * to modify the control registers.
 395          *
 396          * A more complicated solution would involve tracking vblanks
 397          * following the termination of the page-flipping sequence
 398          * and indeed performing the enable as a co-routine and not
 399          * waiting synchronously upon the vblank.
 400          *
 401          * WaFbcWaitForVBlankBeforeEnable:ilk,snb
 402          */
 403         schedule_delayed_work(&work->work, msecs_to_jiffies(50));
 404 }
 405
 406 void intel_disable_fbc(struct drm_device *dev)
 407 {
 408         struct drm_i915_private *dev_priv = dev->dev_private;
 409
 410         intel_cancel_fbc_work(dev_priv);
 411
 412         if (!dev_priv->display.disable_fbc)
 413                 return;
 414
 415         dev_priv->display.disable_fbc(dev);
 416         dev_priv->fbc.plane = -1;
 417 }
 418
 419 static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
 420                               enum no_fbc_reason reason)
 421 {
 422         if (dev_priv->fbc.no_fbc_reason == reason)
 423                 return false;
 424
 425         dev_priv->fbc.no_fbc_reason = reason;
 426         return true;
 427 }
 428
 429 /**
 430  * intel_update_fbc - enable/disable FBC as needed
 431  * @dev: the drm_device
 432  *
 433  * Set up the framebuffer compression hardware at mode set time.  We
 434  * enable it if possible:
 435  *   - plane A only (on pre-965)
 436  *   - no pixel mulitply/line duplication
 437  *   - no alpha buffer discard
 438  *   - no dual wide
 439  *   - framebuffer <= max_hdisplay in width, max_vdisplay in height
 440  *
 441  * We can't assume that any compression will take place (worst case),
 442  * so the compressed buffer has to be the same size as the uncompressed
 443  * one.  It also must reside (along with the line length buffer) in
 444  * stolen memory.
 445  *
 446  * We need to enable/disable FBC on a global basis.
 447  */
 448 void intel_update_fbc(struct drm_device *dev)
 449 {
 450         struct drm_i915_private *dev_priv = dev->dev_private;
 451         struct drm_crtc *crtc = NULL, *tmp_crtc;
 452         struct intel_crtc *intel_crtc;
 453         struct drm_framebuffer *fb;
 454         struct intel_framebuffer *intel_fb;
 455         struct drm_i915_gem_object *obj;
 456         const struct drm_display_mode *adjusted_mode;
 457         unsigned int max_width, max_height;
 458
 459         if (!I915_HAS_FBC(dev)) {
 460                 set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
 461                 return;
 462         }
 463
 464         if (!i915_powersave) {
 465                 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
 466                         DRM_DEBUG_KMS("fbc disabled per module param\n");
 467                 return;
 468         }
 469
 470         /*
 471          * If FBC is already on, we just have to verify that we can
 472          * keep it that way...
 473          * Need to disable if:
 474          *   - more than one pipe is active
 475          *   - changing FBC params (stride, fence, mode)
 476          *   - new fb is too large to fit in compressed buffer
 477          *   - going to an unsupported config (interlace, pixel multiply, etc.)
 478          */
 479         list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
 480                 if (intel_crtc_active(tmp_crtc) &&
 481                     to_intel_crtc(tmp_crtc)->primary_enabled) {
 482                         if (crtc) {
 483                                 if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
 484                                         DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
 485                                 goto out_disable;
 486                         }
 487                         crtc = tmp_crtc;
 488                 }
 489         }
 490
 491         if (!crtc || crtc->fb == NULL) {
 492                 if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
 493                         DRM_DEBUG_KMS("no output, disabling\n");
 494                 goto out_disable;
 495         }
 496
 497         intel_crtc = to_intel_crtc(crtc);
 498         fb = crtc->fb;
 499         intel_fb = to_intel_framebuffer(fb);
 500         obj = intel_fb->obj;
 501         adjusted_mode = &intel_crtc->config.adjusted_mode;
 502
 503         if (i915_enable_fbc < 0 &&
 504             INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) {
 505                 if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
 506                         DRM_DEBUG_KMS("disabled per chip default\n");
 507                 goto out_disable;
 508         }
 509         if (!i915_enable_fbc) {
 510                 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
 511                         DRM_DEBUG_KMS("fbc disabled per module param\n");
 512                 goto out_disable;
 513         }
 514         if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) ||
 515             (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
 516                 if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
 517                         DRM_DEBUG_KMS("mode incompatible with compression, "
 518                                       "disabling\n");
 519                 goto out_disable;
 520         }
 521
 522         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 523                 max_width = 4096;
 524                 max_height = 2048;
 525         } else {
 526                 max_width = 2048;
 527                 max_height = 1536;
 528         }
 529         if (intel_crtc->config.pipe_src_w > max_width ||
 530             intel_crtc->config.pipe_src_h > max_height) {
 531                 if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
 532                         DRM_DEBUG_KMS("mode too large for compression, disabling\n");
 533                 goto out_disable;
 534         }
 535         if ((IS_I915GM(dev) || IS_I945GM(dev) || IS_HASWELL(dev)) &&
 536             intel_crtc->plane != 0) {
 537                 if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
 538                         DRM_DEBUG_KMS("plane not 0, disabling compression\n");
 539                 goto out_disable;
 540         }
 541
 542         /* The use of a CPU fence is mandatory in order to detect writes
 543          * by the CPU to the scanout and trigger updates to the FBC.
 544          */
 545         if (obj->tiling_mode != I915_TILING_X ||
 546             obj->fence_reg == I915_FENCE_REG_NONE) {
 547                 if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
 548                         DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
 549                 goto out_disable;
 550         }
 551
 552         /* If the kernel debugger is active, always disable compression */
 553         if (in_dbg_master())
 554                 goto out_disable;
 555
 556         if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
 557                 if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
 558                         DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
 559                 goto out_disable;
 560         }
 561
 562         /* If the scanout has not changed, don't modify the FBC settings.
 563          * Note that we make the fundamental assumption that the fb->obj
 564          * cannot be unpinned (and have its GTT offset and fence revoked)
 565          * without first being decoupled from the scanout and FBC disabled.
 566          */
 567         if (dev_priv->fbc.plane == intel_crtc->plane &&
 568             dev_priv->fbc.fb_id == fb->base.id &&
 569             dev_priv->fbc.y == crtc->y)
 570                 return;
 571
 572         if (intel_fbc_enabled(dev)) {
 573                 /* We update FBC along two paths, after changing fb/crtc
 574                  * configuration (modeswitching) and after page-flipping
 575                  * finishes. For the latter, we know that not only did
 576                  * we disable the FBC at the start of the page-flip
 577                  * sequence, but also more than one vblank has passed.
 578                  *
 579                  * For the former case of modeswitching, it is possible
 580                  * to switch between two FBC valid configurations
 581                  * instantaneously so we do need to disable the FBC
 582                  * before we can modify its control registers. We also
 583                  * have to wait for the next vblank for that to take
 584                  * effect. However, since we delay enabling FBC we can
 585                  * assume that a vblank has passed since disabling and
 586                  * that we can safely alter the registers in the deferred
 587                  * callback.
 588                  *
 589                  * In the scenario that we go from a valid to invalid
 590                  * and then back to valid FBC configuration we have
 591                  * no strict enforcement that a vblank occurred since
 592                  * disabling the FBC. However, along all current pipe
 593                  * disabling paths we do need to wait for a vblank at
 594                  * some point. And we wait before enabling FBC anyway.
 595                  */
 596                 DRM_DEBUG_KMS("disabling active FBC for update\n");
 597                 intel_disable_fbc(dev);
 598         }
 599
 600         intel_enable_fbc(crtc, 500);
 601         dev_priv->fbc.no_fbc_reason = FBC_OK;
 602         return;
 603
 604 out_disable:
 605         /* Multiple disables should be harmless */
 606         if (intel_fbc_enabled(dev)) {
 607                 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
 608                 intel_disable_fbc(dev);
 609         }
 610         i915_gem_stolen_cleanup_compression(dev);
 611 }
 612
 613 static void i915_pineview_get_mem_freq(struct drm_device *dev)
 614 {
 615         drm_i915_private_t *dev_priv = dev->dev_private;
 616         u32 tmp;
 617
 618         tmp = I915_READ(CLKCFG);
 619
 620         switch (tmp & CLKCFG_FSB_MASK) {
 621         case CLKCFG_FSB_533:
 622                 dev_priv->fsb_freq = 533; /* 133*4 */
 623                 break;
 624         case CLKCFG_FSB_800:
 625                 dev_priv->fsb_freq = 800; /* 200*4 */
 626                 break;
 627         case CLKCFG_FSB_667:
 628                 dev_priv->fsb_freq =  667; /* 167*4 */
 629                 break;
 630         case CLKCFG_FSB_400:
 631                 dev_priv->fsb_freq = 400; /* 100*4 */
 632                 break;
 633         }
 634
 635         switch (tmp & CLKCFG_MEM_MASK) {
 636         case CLKCFG_MEM_533:
 637                 dev_priv->mem_freq = 533;
 638                 break;
 639         case CLKCFG_MEM_667:
 640                 dev_priv->mem_freq = 667;
 641                 break;
 642         case CLKCFG_MEM_800:
 643                 dev_priv->mem_freq = 800;
 644                 break;
 645         }
 646
 647         /* detect pineview DDR3 setting */
 648         tmp = I915_READ(CSHRDDR3CTL);
 649         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 650 }
 651
 652 static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 653 {
 654         drm_i915_private_t *dev_priv = dev->dev_private;
 655         u16 ddrpll, csipll;
 656
 657         ddrpll = I915_READ16(DDRMPLL1);
 658         csipll = I915_READ16(CSIPLL0);
 659
 660         switch (ddrpll & 0xff) {
 661         case 0xc:
 662                 dev_priv->mem_freq = 800;
 663                 break;
 664         case 0x10:
 665                 dev_priv->mem_freq = 1066;
 666                 break;
 667         case 0x14:
 668                 dev_priv->mem_freq = 1333;
 669                 break;
 670         case 0x18:
 671                 dev_priv->mem_freq = 1600;
 672                 break;
 673         default:
 674                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 675                                  ddrpll & 0xff);
 676                 dev_priv->mem_freq = 0;
 677                 break;
 678         }
 679
 680         dev_priv->ips.r_t = dev_priv->mem_freq;
 681
 682         switch (csipll & 0x3ff) {
 683         case 0x00c:
 684                 dev_priv->fsb_freq = 3200;
 685                 break;
 686         case 0x00e:
 687                 dev_priv->fsb_freq = 3733;
 688                 break;
 689         case 0x010:
 690                 dev_priv->fsb_freq = 4266;
 691                 break;
 692         case 0x012:
 693                 dev_priv->fsb_freq = 4800;
 694                 break;
 695         case 0x014:
 696                 dev_priv->fsb_freq = 5333;
 697                 break;
 698         case 0x016:
 699                 dev_priv->fsb_freq = 5866;
 700                 break;
 701         case 0x018:
 702                 dev_priv->fsb_freq = 6400;
 703                 break;
 704         default:
 705                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 706                                  csipll & 0x3ff);
 707                 dev_priv->fsb_freq = 0;
 708                 break;
 709         }
 710
 711         if (dev_priv->fsb_freq == 3200) {
 712                 dev_priv->ips.c_m = 0;
 713         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 714                 dev_priv->ips.c_m = 1;
 715         } else {
 716                 dev_priv->ips.c_m = 2;
 717         }
 718 }
 719
 720 static const struct cxsr_latency cxsr_latency_table[] = {
 721         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 722         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 723         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 724         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 725         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 726
 727         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 728         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 729         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 730         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 731         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 732
 733         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 734         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 735         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 736         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 737         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 738
 739         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 740         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 741         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 742         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 743         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 744
 745         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 746         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 747         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 748         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 749         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 750
 751         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 752         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 753         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 754         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 755         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 756 };
 757
 758 static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
 759                                                          int is_ddr3,
 760                                                          int fsb,
 761                                                          int mem)
 762 {
 763         const struct cxsr_latency *latency;
 764         int i;
 765
 766         if (fsb == 0 || mem == 0)
 767                 return NULL;
 768
 769         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 770                 latency = &cxsr_latency_table[i];
 771                 if (is_desktop == latency->is_desktop &&
 772                     is_ddr3 == latency->is_ddr3 &&
 773                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 774                         return latency;
 775         }
 776
 777         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 778
 779         return NULL;
 780 }
 781
 782 static void pineview_disable_cxsr(struct drm_device *dev)
 783 {
 784         struct drm_i915_private *dev_priv = dev->dev_private;
 785
 786         /* deactivate cxsr */
 787         I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
 788 }
 789
 790 /*
 791  * Latency for FIFO fetches is dependent on several factors:
 792  *   - memory configuration (speed, channels)
 793  *   - chipset
 794  *   - current MCH state
 795  * It can be fairly high in some situations, so here we assume a fairly
 796  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 797  * set this value too high, the FIFO will fetch frequently to stay full)
 798  * and power consumption (set it too low to save power and we might see
 799  * FIFO underruns and display "flicker").
 800  *
 801  * A value of 5us seems to be a good balance; safe for very low end
 802  * platforms but not overly aggressive on lower latency configs.
 803  */
 804 static const int latency_ns = 5000;
 805
 806 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 807 {
 808         struct drm_i915_private *dev_priv = dev->dev_private;
 809         uint32_t dsparb = I915_READ(DSPARB);
 810         int size;
 811
 812         size = dsparb & 0x7f;
 813         if (plane)
 814                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 815
 816         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 817                       plane ? "B" : "A", size);
 818
 819         return size;
 820 }
 821
 822 static int i85x_get_fifo_size(struct drm_device *dev, int plane)
 823 {
 824         struct drm_i915_private *dev_priv = dev->dev_private;
 825         uint32_t dsparb = I915_READ(DSPARB);
 826         int size;
 827
 828         size = dsparb & 0x1ff;
 829         if (plane)
 830                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 831         size >>= 1; /* Convert to cachelines */
 832
 833         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 834                       plane ? "B" : "A", size);
 835
 836         return size;
 837 }
 838
 839 static int i845_get_fifo_size(struct drm_device *dev, int plane)
 840 {
 841         struct drm_i915_private *dev_priv = dev->dev_private;
 842         uint32_t dsparb = I915_READ(DSPARB);
 843         int size;
 844
 845         size = dsparb & 0x7f;
 846         size >>= 2; /* Convert to cachelines */
 847
 848         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 849                       plane ? "B" : "A",
 850                       size);
 851
 852         return size;
 853 }
 854
 855 static int i830_get_fifo_size(struct drm_device *dev, int plane)
 856 {
 857         struct drm_i915_private *dev_priv = dev->dev_private;
 858         uint32_t dsparb = I915_READ(DSPARB);
 859         int size;
 860
 861         size = dsparb & 0x7f;
 862         size >>= 1; /* Convert to cachelines */
 863
 864         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 865                       plane ? "B" : "A", size);
 866
 867         return size;
 868 }
 869
 870 /* Pineview has different values for various configs */
 871 static const struct intel_watermark_params pineview_display_wm = {
 872         PINEVIEW_DISPLAY_FIFO,
 873         PINEVIEW_MAX_WM,
 874         PINEVIEW_DFT_WM,
 875         PINEVIEW_GUARD_WM,
 876         PINEVIEW_FIFO_LINE_SIZE
 877 };
 878 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 879         PINEVIEW_DISPLAY_FIFO,
 880         PINEVIEW_MAX_WM,
 881         PINEVIEW_DFT_HPLLOFF_WM,
 882         PINEVIEW_GUARD_WM,
 883         PINEVIEW_FIFO_LINE_SIZE
 884 };
 885 static const struct intel_watermark_params pineview_cursor_wm = {
 886         PINEVIEW_CURSOR_FIFO,
 887         PINEVIEW_CURSOR_MAX_WM,
 888         PINEVIEW_CURSOR_DFT_WM,
 889         PINEVIEW_CURSOR_GUARD_WM,
 890         PINEVIEW_FIFO_LINE_SIZE,
 891 };
 892 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 893         PINEVIEW_CURSOR_FIFO,
 894         PINEVIEW_CURSOR_MAX_WM,
 895         PINEVIEW_CURSOR_DFT_WM,
 896         PINEVIEW_CURSOR_GUARD_WM,
 897         PINEVIEW_FIFO_LINE_SIZE
 898 };
 899 static const struct intel_watermark_params g4x_wm_info = {
 900         G4X_FIFO_SIZE,
 901         G4X_MAX_WM,
 902         G4X_MAX_WM,
 903         2,
 904         G4X_FIFO_LINE_SIZE,
 905 };
 906 static const struct intel_watermark_params g4x_cursor_wm_info = {
 907         I965_CURSOR_FIFO,
 908         I965_CURSOR_MAX_WM,
 909         I965_CURSOR_DFT_WM,
 910         2,
 911         G4X_FIFO_LINE_SIZE,
 912 };
 913 static const struct intel_watermark_params valleyview_wm_info = {
 914         VALLEYVIEW_FIFO_SIZE,
 915         VALLEYVIEW_MAX_WM,
 916         VALLEYVIEW_MAX_WM,
 917         2,
 918         G4X_FIFO_LINE_SIZE,
 919 };
 920 static const struct intel_watermark_params valleyview_cursor_wm_info = {
 921         I965_CURSOR_FIFO,
 922         VALLEYVIEW_CURSOR_MAX_WM,
 923         I965_CURSOR_DFT_WM,
 924         2,
 925         G4X_FIFO_LINE_SIZE,
 926 };
 927 static const struct intel_watermark_params i965_cursor_wm_info = {
 928         I965_CURSOR_FIFO,
 929         I965_CURSOR_MAX_WM,
 930         I965_CURSOR_DFT_WM,
 931         2,
 932         I915_FIFO_LINE_SIZE,
 933 };
 934 static const struct intel_watermark_params i945_wm_info = {
 935         I945_FIFO_SIZE,
 936         I915_MAX_WM,
 937         1,
 938         2,
 939         I915_FIFO_LINE_SIZE
 940 };
 941 static const struct intel_watermark_params i915_wm_info = {
 942         I915_FIFO_SIZE,
 943         I915_MAX_WM,
 944         1,
 945         2,
 946         I915_FIFO_LINE_SIZE
 947 };
 948 static const struct intel_watermark_params i855_wm_info = {
 949         I855GM_FIFO_SIZE,
 950         I915_MAX_WM,
 951         1,
 952         2,
 953         I830_FIFO_LINE_SIZE
 954 };
 955 static const struct intel_watermark_params i830_wm_info = {
 956         I830_FIFO_SIZE,
 957         I915_MAX_WM,
 958         1,
 959         2,
 960         I830_FIFO_LINE_SIZE
 961 };
 962
 963 static const struct intel_watermark_params ironlake_display_wm_info = {
 964         ILK_DISPLAY_FIFO,
 965         ILK_DISPLAY_MAXWM,
 966         ILK_DISPLAY_DFTWM,
 967         2,
 968         ILK_FIFO_LINE_SIZE
 969 };
 970 static const struct intel_watermark_params ironlake_cursor_wm_info = {
 971         ILK_CURSOR_FIFO,
 972         ILK_CURSOR_MAXWM,
 973         ILK_CURSOR_DFTWM,
 974         2,
 975         ILK_FIFO_LINE_SIZE
 976 };
 977 static const struct intel_watermark_params ironlake_display_srwm_info = {
 978         ILK_DISPLAY_SR_FIFO,
 979         ILK_DISPLAY_MAX_SRWM,
 980         ILK_DISPLAY_DFT_SRWM,
 981         2,
 982         ILK_FIFO_LINE_SIZE
 983 };
 984 static const struct intel_watermark_params ironlake_cursor_srwm_info = {
 985         ILK_CURSOR_SR_FIFO,
 986         ILK_CURSOR_MAX_SRWM,
 987         ILK_CURSOR_DFT_SRWM,
 988         2,
 989         ILK_FIFO_LINE_SIZE
 990 };
 991
 992 static const struct intel_watermark_params sandybridge_display_wm_info = {
 993         SNB_DISPLAY_FIFO,
 994         SNB_DISPLAY_MAXWM,
 995         SNB_DISPLAY_DFTWM,
 996         2,
 997         SNB_FIFO_LINE_SIZE
 998 };
 999 static const struct intel_watermark_params sandybridge_cursor_wm_info = {
1000         SNB_CURSOR_FIFO,
1001         SNB_CURSOR_MAXWM,
1002         SNB_CURSOR_DFTWM,
1003         2,
1004         SNB_FIFO_LINE_SIZE
1005 };
1006 static const struct intel_watermark_params sandybridge_display_srwm_info = {
1007         SNB_DISPLAY_SR_FIFO,
1008         SNB_DISPLAY_MAX_SRWM,
1009         SNB_DISPLAY_DFT_SRWM,
1010         2,
1011         SNB_FIFO_LINE_SIZE
1012 };
1013 static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
1014         SNB_CURSOR_SR_FIFO,
1015         SNB_CURSOR_MAX_SRWM,
1016         SNB_CURSOR_DFT_SRWM,
1017         2,
1018         SNB_FIFO_LINE_SIZE
1019 };
1020
1021
1022 /**
1023  * intel_calculate_wm - calculate watermark level
1024  * @clock_in_khz: pixel clock
1025  * @wm: chip FIFO params
1026  * @pixel_size: display pixel size
1027  * @latency_ns: memory latency for the platform
1028  *
1029  * Calculate the watermark level (the level at which the display plane will
1030  * start fetching from memory again).  Each chip has a different display
1031  * FIFO size and allocation, so the caller needs to figure that out and pass
1032  * in the correct intel_watermark_params structure.
1033  *
1034  * As the pixel clock runs, the FIFO will be drained at a rate that depends
1035  * on the pixel size.  When it reaches the watermark level, it'll start
1036  * fetching FIFO line sized based chunks from memory until the FIFO fills
1037  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
1038  * will occur, and a display engine hang could result.
1039  */
1040 static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
1041                                         const struct intel_watermark_params *wm,
1042                                         int fifo_size,
1043                                         int pixel_size,
1044                                         unsigned long latency_ns)
1045 {
1046         long entries_required, wm_size;
1047
1048         /*
1049          * Note: we need to make sure we don't overflow for various clock &
1050          * latency values.
1051          * clocks go from a few thousand to several hundred thousand.
1052          * latency is usually a few thousand
1053          */
1054         entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
1055                 1000;
1056         entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
1057
1058         DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
1059
1060         wm_size = fifo_size - (entries_required + wm->guard_size);
1061
1062         DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
1063
1064         /* Don't promote wm_size to unsigned... */
1065         if (wm_size > (long)wm->max_wm)
1066                 wm_size = wm->max_wm;
1067         if (wm_size <= 0)
1068                 wm_size = wm->default_wm;
1069         return wm_size;
1070 }
1071
1072 static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
1073 {
1074         struct drm_crtc *crtc, *enabled = NULL;
1075
1076         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1077                 if (intel_crtc_active(crtc)) {
1078                         if (enabled)
1079                                 return NULL;
1080                         enabled = crtc;
1081                 }
1082         }
1083
1084         return enabled;
1085 }
1086
1087 static void pineview_update_wm(struct drm_crtc *unused_crtc)
1088 {
1089         struct drm_device *dev = unused_crtc->dev;
1090         struct drm_i915_private *dev_priv = dev->dev_private;
1091         struct drm_crtc *crtc;
1092         const struct cxsr_latency *latency;
1093         u32 reg;
1094         unsigned long wm;
1095
1096         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
1097                                          dev_priv->fsb_freq, dev_priv->mem_freq);
1098         if (!latency) {
1099                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
1100                 pineview_disable_cxsr(dev);
1101                 return;
1102         }
1103
1104         crtc = single_enabled_crtc(dev);
1105         if (crtc) {
1106                 const struct drm_display_mode *adjusted_mode;
1107                 int pixel_size = crtc->fb->bits_per_pixel / 8;
1108                 int clock;
1109
1110                 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1111                 clock = adjusted_mode->crtc_clock;
1112
1113                 /* Display SR */
1114                 wm = intel_calculate_wm(clock, &pineview_display_wm,
1115                                         pineview_display_wm.fifo_size,
1116                                         pixel_size, latency->display_sr);
1117                 reg = I915_READ(DSPFW1);
1118                 reg &= ~DSPFW_SR_MASK;
1119                 reg |= wm << DSPFW_SR_SHIFT;
1120                 I915_WRITE(DSPFW1, reg);
1121                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
1122
1123                 /* cursor SR */
1124                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
1125                                         pineview_display_wm.fifo_size,
1126                                         pixel_size, latency->cursor_sr);
1127                 reg = I915_READ(DSPFW3);
1128                 reg &= ~DSPFW_CURSOR_SR_MASK;
1129                 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
1130                 I915_WRITE(DSPFW3, reg);
1131
1132                 /* Display HPLL off SR */
1133                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
1134                                         pineview_display_hplloff_wm.fifo_size,
1135                                         pixel_size, latency->display_hpll_disable);
1136                 reg = I915_READ(DSPFW3);
1137                 reg &= ~DSPFW_HPLL_SR_MASK;
1138                 reg |= wm & DSPFW_HPLL_SR_MASK;
1139                 I915_WRITE(DSPFW3, reg);
1140
1141                 /* cursor HPLL off SR */
1142                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
1143                                         pineview_display_hplloff_wm.fifo_size,
1144                                         pixel_size, latency->cursor_hpll_disable);
1145                 reg = I915_READ(DSPFW3);
1146                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
1147                 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
1148                 I915_WRITE(DSPFW3, reg);
1149                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
1150
1151                 /* activate cxsr */
1152                 I915_WRITE(DSPFW3,
1153                            I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
1154                 DRM_DEBUG_KMS("Self-refresh is enabled\n");
1155         } else {
1156                 pineview_disable_cxsr(dev);
1157                 DRM_DEBUG_KMS("Self-refresh is disabled\n");
1158         }
1159 }
1160
1161 static bool g4x_compute_wm0(struct drm_device *dev,
1162                             int plane,
1163                             const struct intel_watermark_params *display,
1164                             int display_latency_ns,
1165                             const struct intel_watermark_params *cursor,
1166                             int cursor_latency_ns,
1167                             int *plane_wm,
1168                             int *cursor_wm)
1169 {
1170         struct drm_crtc *crtc;
1171         const struct drm_display_mode *adjusted_mode;
1172         int htotal, hdisplay, clock, pixel_size;
1173         int line_time_us, line_count;
1174         int entries, tlb_miss;
1175
1176         crtc = intel_get_crtc_for_plane(dev, plane);
1177         if (!intel_crtc_active(crtc)) {
1178                 *cursor_wm = cursor->guard_size;
1179                 *plane_wm = display->guard_size;
1180                 return false;
1181         }
1182
1183         adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1184         clock = adjusted_mode->crtc_clock;
1185         htotal = adjusted_mode->htotal;
1186         hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1187         pixel_size = crtc->fb->bits_per_pixel / 8;
1188
1189         /* Use the small buffer method to calculate plane watermark */
1190         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
1191         tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
1192         if (tlb_miss > 0)
1193                 entries += tlb_miss;
1194         entries = DIV_ROUND_UP(entries, display->cacheline_size);
1195         *plane_wm = entries + display->guard_size;
1196         if (*plane_wm > (int)display->max_wm)
1197                 *plane_wm = display->max_wm;
1198
1199         /* Use the large buffer method to calculate cursor watermark */
1200         line_time_us = ((htotal * 1000) / clock);
1201         line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
1202         entries = line_count * 64 * pixel_size;
1203         tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
1204         if (tlb_miss > 0)
1205                 entries += tlb_miss;
1206         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1207         *cursor_wm = entries + cursor->guard_size;
1208         if (*cursor_wm > (int)cursor->max_wm)
1209                 *cursor_wm = (int)cursor->max_wm;
1210
1211         return true;
1212 }
1213
1214 /*
1215  * Check the wm result.
1216  *
1217  * If any calculated watermark values is larger than the maximum value that
1218  * can be programmed into the associated watermark register, that watermark
1219  * must be disabled.
1220  */
1221 static bool g4x_check_srwm(struct drm_device *dev,
1222                            int display_wm, int cursor_wm,
1223                            const struct intel_watermark_params *display,
1224                            const struct intel_watermark_params *cursor)
1225 {
1226         DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
1227                       display_wm, cursor_wm);
1228
1229         if (display_wm > display->max_wm) {
1230                 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
1231                               display_wm, display->max_wm);
1232                 return false;
1233         }
1234
1235         if (cursor_wm > cursor->max_wm) {
1236                 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
1237                               cursor_wm, cursor->max_wm);
1238                 return false;
1239         }
1240
1241         if (!(display_wm || cursor_wm)) {
1242                 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
1243                 return false;
1244         }
1245
1246         return true;
1247 }
1248
1249 static bool g4x_compute_srwm(struct drm_device *dev,
1250                              int plane,
1251                              int latency_ns,
1252                              const struct intel_watermark_params *display,
1253                              const struct intel_watermark_params *cursor,
1254                              int *display_wm, int *cursor_wm)
1255 {
1256         struct drm_crtc *crtc;
1257         const struct drm_display_mode *adjusted_mode;
1258         int hdisplay, htotal, pixel_size, clock;
1259         unsigned long line_time_us;
1260         int line_count, line_size;
1261         int small, large;
1262         int entries;
1263
1264         if (!latency_ns) {
1265                 *display_wm = *cursor_wm = 0;
1266                 return false;
1267         }
1268
1269         crtc = intel_get_crtc_for_plane(dev, plane);
1270         adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1271         clock = adjusted_mode->crtc_clock;
1272         htotal = adjusted_mode->htotal;
1273         hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1274         pixel_size = crtc->fb->bits_per_pixel / 8;
1275
1276         line_time_us = (htotal * 1000) / clock;
1277         line_count = (latency_ns / line_time_us + 1000) / 1000;
1278         line_size = hdisplay * pixel_size;
1279
1280         /* Use the minimum of the small and large buffer method for primary */
1281         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
1282         large = line_count * line_size;
1283
1284         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
1285         *display_wm = entries + display->guard_size;
1286
1287         /* calculate the self-refresh watermark for display cursor */
1288         entries = line_count * pixel_size * 64;
1289         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1290         *cursor_wm = entries + cursor->guard_size;
1291
1292         return g4x_check_srwm(dev,
1293                               *display_wm, *cursor_wm,
1294                               display, cursor);
1295 }
1296
1297 static bool vlv_compute_drain_latency(struct drm_device *dev,
1298                                      int plane,
1299                                      int *plane_prec_mult,
1300                                      int *plane_dl,
1301                                      int *cursor_prec_mult,
1302                                      int *cursor_dl)
1303 {
1304         struct drm_crtc *crtc;
1305         int clock, pixel_size;
1306         int entries;
1307
1308         crtc = intel_get_crtc_for_plane(dev, plane);
1309         if (!intel_crtc_active(crtc))
1310                 return false;
1311
1312         clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
1313         pixel_size = crtc->fb->bits_per_pixel / 8;      /* BPP */
1314
1315         entries = (clock / 1000) * pixel_size;
1316         *plane_prec_mult = (entries > 256) ?
1317                 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1318         *plane_dl = (64 * (*plane_prec_mult) * 4) / ((clock / 1000) *
1319                                                      pixel_size);
1320
1321         entries = (clock / 1000) * 4;   /* BPP is always 4 for cursor */
1322         *cursor_prec_mult = (entries > 256) ?
1323                 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1324         *cursor_dl = (64 * (*cursor_prec_mult) * 4) / ((clock / 1000) * 4);
1325
1326         return true;
1327 }
1328
1329 /*
1330  * Update drain latency registers of memory arbiter
1331  *
1332  * Valleyview SoC has a new memory arbiter and needs drain latency registers
1333  * to be programmed. Each plane has a drain latency multiplier and a drain
1334  * latency value.
1335  */
1336
1337 static void vlv_update_drain_latency(struct drm_device *dev)
1338 {
1339         struct drm_i915_private *dev_priv = dev->dev_private;
1340         int planea_prec, planea_dl, planeb_prec, planeb_dl;
1341         int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
1342         int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
1343                                                         either 16 or 32 */
1344
1345         /* For plane A, Cursor A */
1346         if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
1347                                       &cursor_prec_mult, &cursora_dl)) {
1348                 cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1349                         DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
1350                 planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1351                         DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
1352
1353                 I915_WRITE(VLV_DDL1, cursora_prec |
1354                                 (cursora_dl << DDL_CURSORA_SHIFT) |
1355                                 planea_prec | planea_dl);
1356         }
1357
1358         /* For plane B, Cursor B */
1359         if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
1360                                       &cursor_prec_mult, &cursorb_dl)) {
1361                 cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1362                         DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
1363                 planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1364                         DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
1365
1366                 I915_WRITE(VLV_DDL2, cursorb_prec |
1367                                 (cursorb_dl << DDL_CURSORB_SHIFT) |
1368                                 planeb_prec | planeb_dl);
1369         }
1370 }
1371
1372 #define single_plane_enabled(mask) is_power_of_2(mask)
1373
1374 static void valleyview_update_wm(struct drm_crtc *crtc)
1375 {
1376         struct drm_device *dev = crtc->dev;
1377         static const int sr_latency_ns = 12000;
1378         struct drm_i915_private *dev_priv = dev->dev_private;
1379         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1380         int plane_sr, cursor_sr;
1381         int ignore_plane_sr, ignore_cursor_sr;
1382         unsigned int enabled = 0;
1383
1384         vlv_update_drain_latency(dev);
1385
1386         if (g4x_compute_wm0(dev, PIPE_A,
1387                             &valleyview_wm_info, latency_ns,
1388                             &valleyview_cursor_wm_info, latency_ns,
1389                             &planea_wm, &cursora_wm))
1390                 enabled |= 1 << PIPE_A;
1391
1392         if (g4x_compute_wm0(dev, PIPE_B,
1393                             &valleyview_wm_info, latency_ns,
1394                             &valleyview_cursor_wm_info, latency_ns,
1395                             &planeb_wm, &cursorb_wm))
1396                 enabled |= 1 << PIPE_B;
1397
1398         if (single_plane_enabled(enabled) &&
1399             g4x_compute_srwm(dev, ffs(enabled) - 1,
1400                              sr_latency_ns,
1401                              &valleyview_wm_info,
1402                              &valleyview_cursor_wm_info,
1403                              &plane_sr, &ignore_cursor_sr) &&
1404             g4x_compute_srwm(dev, ffs(enabled) - 1,
1405                              2*sr_latency_ns,
1406                              &valleyview_wm_info,
1407                              &valleyview_cursor_wm_info,
1408                              &ignore_plane_sr, &cursor_sr)) {
1409                 I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
1410         } else {
1411                 I915_WRITE(FW_BLC_SELF_VLV,
1412                            I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
1413                 plane_sr = cursor_sr = 0;
1414         }
1415
1416         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1417                       planea_wm, cursora_wm,
1418                       planeb_wm, cursorb_wm,
1419                       plane_sr, cursor_sr);
1420
1421         I915_WRITE(DSPFW1,
1422                    (plane_sr << DSPFW_SR_SHIFT) |
1423                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1424                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
1425                    planea_wm);
1426         I915_WRITE(DSPFW2,
1427                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1428                    (cursora_wm << DSPFW_CURSORA_SHIFT));
1429         I915_WRITE(DSPFW3,
1430                    (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
1431                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1432 }
1433
1434 static void g4x_update_wm(struct drm_crtc *crtc)
1435 {
1436         struct drm_device *dev = crtc->dev;
1437         static const int sr_latency_ns = 12000;
1438         struct drm_i915_private *dev_priv = dev->dev_private;
1439         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1440         int plane_sr, cursor_sr;
1441         unsigned int enabled = 0;
1442
1443         if (g4x_compute_wm0(dev, PIPE_A,
1444                             &g4x_wm_info, latency_ns,
1445                             &g4x_cursor_wm_info, latency_ns,
1446                             &planea_wm, &cursora_wm))
1447                 enabled |= 1 << PIPE_A;
1448
1449         if (g4x_compute_wm0(dev, PIPE_B,
1450                             &g4x_wm_info, latency_ns,
1451                             &g4x_cursor_wm_info, latency_ns,
1452                             &planeb_wm, &cursorb_wm))
1453                 enabled |= 1 << PIPE_B;
1454
1455         if (single_plane_enabled(enabled) &&
1456             g4x_compute_srwm(dev, ffs(enabled) - 1,
1457                              sr_latency_ns,
1458                              &g4x_wm_info,
1459                              &g4x_cursor_wm_info,
1460                              &plane_sr, &cursor_sr)) {
1461                 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
1462         } else {
1463                 I915_WRITE(FW_BLC_SELF,
1464                            I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
1465                 plane_sr = cursor_sr = 0;
1466         }
1467
1468         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1469                       planea_wm, cursora_wm,
1470                       planeb_wm, cursorb_wm,
1471                       plane_sr, cursor_sr);
1472
1473         I915_WRITE(DSPFW1,
1474                    (plane_sr << DSPFW_SR_SHIFT) |
1475                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1476                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
1477                    planea_wm);
1478         I915_WRITE(DSPFW2,
1479                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1480                    (cursora_wm << DSPFW_CURSORA_SHIFT));
1481         /* HPLL off in SR has some issues on G4x... disable it */
1482         I915_WRITE(DSPFW3,
1483                    (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
1484                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1485 }
1486
1487 static void i965_update_wm(struct drm_crtc *unused_crtc)
1488 {
1489         struct drm_device *dev = unused_crtc->dev;
1490         struct drm_i915_private *dev_priv = dev->dev_private;
1491         struct drm_crtc *crtc;
1492         int srwm = 1;
1493         int cursor_sr = 16;
1494
1495         /* Calc sr entries for one plane configs */
1496         crtc = single_enabled_crtc(dev);
1497         if (crtc) {
1498                 /* self-refresh has much higher latency */
1499                 static const int sr_latency_ns = 12000;
1500                 const struct drm_display_mode *adjusted_mode =
1501                         &to_intel_crtc(crtc)->config.adjusted_mode;
1502                 int clock = adjusted_mode->crtc_clock;
1503                 int htotal = adjusted_mode->htotal;
1504                 int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1505                 int pixel_size = crtc->fb->bits_per_pixel / 8;
1506                 unsigned long line_time_us;
1507                 int entries;
1508
1509                 line_time_us = ((htotal * 1000) / clock);
1510
1511                 /* Use ns/us then divide to preserve precision */
1512                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1513                         pixel_size * hdisplay;
1514                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1515                 srwm = I965_FIFO_SIZE - entries;
1516                 if (srwm < 0)
1517                         srwm = 1;
1518                 srwm &= 0x1ff;
1519                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1520                               entries, srwm);
1521
1522                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1523                         pixel_size * 64;
1524                 entries = DIV_ROUND_UP(entries,
1525                                           i965_cursor_wm_info.cacheline_size);
1526                 cursor_sr = i965_cursor_wm_info.fifo_size -
1527                         (entries + i965_cursor_wm_info.guard_size);
1528
1529                 if (cursor_sr > i965_cursor_wm_info.max_wm)
1530                         cursor_sr = i965_cursor_wm_info.max_wm;
1531
1532                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1533                               "cursor %d\n", srwm, cursor_sr);
1534
1535                 if (IS_CRESTLINE(dev))
1536                         I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
1537         } else {
1538                 /* Turn off self refresh if both pipes are enabled */
1539                 if (IS_CRESTLINE(dev))
1540                         I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
1541                                    & ~FW_BLC_SELF_EN);
1542         }
1543
1544         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1545                       srwm);
1546
1547         /* 965 has limitations... */
1548         I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
1549                    (8 << 16) | (8 << 8) | (8 << 0));
1550         I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
1551         /* update cursor SR watermark */
1552         I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1553 }
1554
1555 static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1556 {
1557         struct drm_device *dev = unused_crtc->dev;
1558         struct drm_i915_private *dev_priv = dev->dev_private;
1559         const struct intel_watermark_params *wm_info;
1560         uint32_t fwater_lo;
1561         uint32_t fwater_hi;
1562         int cwm, srwm = 1;
1563         int fifo_size;
1564         int planea_wm, planeb_wm;
1565         struct drm_crtc *crtc, *enabled = NULL;
1566
1567         if (IS_I945GM(dev))
1568                 wm_info = &i945_wm_info;
1569         else if (!IS_GEN2(dev))
1570                 wm_info = &i915_wm_info;
1571         else
1572                 wm_info = &i855_wm_info;
1573
1574         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1575         crtc = intel_get_crtc_for_plane(dev, 0);
1576         if (intel_crtc_active(crtc)) {
1577                 const struct drm_display_mode *adjusted_mode;
1578                 int cpp = crtc->fb->bits_per_pixel / 8;
1579                 if (IS_GEN2(dev))
1580                         cpp = 4;
1581
1582                 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1583                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1584                                                wm_info, fifo_size, cpp,
1585                                                latency_ns);
1586                 enabled = crtc;
1587         } else
1588                 planea_wm = fifo_size - wm_info->guard_size;
1589
1590         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1591         crtc = intel_get_crtc_for_plane(dev, 1);
1592         if (intel_crtc_active(crtc)) {
1593                 const struct drm_display_mode *adjusted_mode;
1594                 int cpp = crtc->fb->bits_per_pixel / 8;
1595                 if (IS_GEN2(dev))
1596                         cpp = 4;
1597
1598                 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1599                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1600                                                wm_info, fifo_size, cpp,
1601                                                latency_ns);
1602                 if (enabled == NULL)
1603                         enabled = crtc;
1604                 else
1605                         enabled = NULL;
1606         } else
1607                 planeb_wm = fifo_size - wm_info->guard_size;
1608
1609         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1610
1611         /*
1612          * Overlay gets an aggressive default since video jitter is bad.
1613          */
1614         cwm = 2;
1615
1616         /* Play safe and disable self-refresh before adjusting watermarks. */
1617         if (IS_I945G(dev) || IS_I945GM(dev))
1618                 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
1619         else if (IS_I915GM(dev))
1620                 I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
1621
1622         /* Calc sr entries for one plane configs */
1623         if (HAS_FW_BLC(dev) && enabled) {
1624                 /* self-refresh has much higher latency */
1625                 static const int sr_latency_ns = 6000;
1626                 const struct drm_display_mode *adjusted_mode =
1627                         &to_intel_crtc(enabled)->config.adjusted_mode;
1628                 int clock = adjusted_mode->crtc_clock;
1629                 int htotal = adjusted_mode->htotal;
1630                 int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1631                 int pixel_size = enabled->fb->bits_per_pixel / 8;
1632                 unsigned long line_time_us;
1633                 int entries;
1634
1635                 line_time_us = (htotal * 1000) / clock;
1636
1637                 /* Use ns/us then divide to preserve precision */
1638                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1639                         pixel_size * hdisplay;
1640                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1641                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1642                 srwm = wm_info->fifo_size - entries;
1643                 if (srwm < 0)
1644                         srwm = 1;
1645
1646                 if (IS_I945G(dev) || IS_I945GM(dev))
1647                         I915_WRITE(FW_BLC_SELF,
1648                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1649                 else if (IS_I915GM(dev))
1650                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1651         }
1652
1653         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1654                       planea_wm, planeb_wm, cwm, srwm);
1655
1656         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1657         fwater_hi = (cwm & 0x1f);
1658
1659         /* Set request length to 8 cachelines per fetch */
1660         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1661         fwater_hi = fwater_hi | (1 << 8);
1662
1663         I915_WRITE(FW_BLC, fwater_lo);
1664         I915_WRITE(FW_BLC2, fwater_hi);
1665
1666         if (HAS_FW_BLC(dev)) {
1667                 if (enabled) {
1668                         if (IS_I945G(dev) || IS_I945GM(dev))
1669                                 I915_WRITE(FW_BLC_SELF,
1670                                            FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
1671                         else if (IS_I915GM(dev))
1672                                 I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
1673                         DRM_DEBUG_KMS("memory self refresh enabled\n");
1674                 } else
1675                         DRM_DEBUG_KMS("memory self refresh disabled\n");
1676         }
1677 }
1678
1679 static void i830_update_wm(struct drm_crtc *unused_crtc)
1680 {
1681         struct drm_device *dev = unused_crtc->dev;
1682         struct drm_i915_private *dev_priv = dev->dev_private;
1683         struct drm_crtc *crtc;
1684         const struct drm_display_mode *adjusted_mode;
1685         uint32_t fwater_lo;
1686         int planea_wm;
1687
1688         crtc = single_enabled_crtc(dev);
1689         if (crtc == NULL)
1690                 return;
1691
1692         adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1693         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1694                                        &i830_wm_info,
1695                                        dev_priv->display.get_fifo_size(dev, 0),
1696                                        4, latency_ns);
1697         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1698         fwater_lo |= (3<<8) | planea_wm;
1699
1700         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1701
1702         I915_WRITE(FW_BLC, fwater_lo);
1703 }
1704
1705 /*
1706  * Check the wm result.
1707  *
1708  * If any calculated watermark values is larger than the maximum value that
1709  * can be programmed into the associated watermark register, that watermark
1710  * must be disabled.
1711  */
1712 static bool ironlake_check_srwm(struct drm_device *dev, int level,
1713                                 int fbc_wm, int display_wm, int cursor_wm,
1714                                 const struct intel_watermark_params *display,
1715                                 const struct intel_watermark_params *cursor)
1716 {
1717         struct drm_i915_private *dev_priv = dev->dev_private;
1718
1719         DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
1720                       " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
1721
1722         if (fbc_wm > SNB_FBC_MAX_SRWM) {
1723                 DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
1724                               fbc_wm, SNB_FBC_MAX_SRWM, level);
1725
1726                 /* fbc has it's own way to disable FBC WM */
1727                 I915_WRITE(DISP_ARB_CTL,
1728                            I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
1729                 return false;
1730         } else if (INTEL_INFO(dev)->gen >= 6) {
1731                 /* enable FBC WM (except on ILK, where it must remain off) */
1732                 I915_WRITE(DISP_ARB_CTL,
1733                            I915_READ(DISP_ARB_CTL) & ~DISP_FBC_WM_DIS);
1734         }
1735
1736         if (display_wm > display->max_wm) {
1737                 DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
1738                               display_wm, SNB_DISPLAY_MAX_SRWM, level);
1739                 return false;
1740         }
1741
1742         if (cursor_wm > cursor->max_wm) {
1743                 DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
1744                               cursor_wm, SNB_CURSOR_MAX_SRWM, level);
1745                 return false;
1746         }
1747
1748         if (!(fbc_wm || display_wm || cursor_wm)) {
1749                 DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
1750                 return false;
1751         }
1752
1753         return true;
1754 }
1755
1756 /*
1757  * Compute watermark values of WM[1-3],
1758  */
1759 static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
1760                                   int latency_ns,
1761                                   const struct intel_watermark_params *display,
1762                                   const struct intel_watermark_params *cursor,
1763                                   int *fbc_wm, int *display_wm, int *cursor_wm)
1764 {
1765         struct drm_crtc *crtc;
1766         const struct drm_display_mode *adjusted_mode;
1767         unsigned long line_time_us;
1768         int hdisplay, htotal, pixel_size, clock;
1769         int line_count, line_size;
1770         int small, large;
1771         int entries;
1772
1773         if (!latency_ns) {
1774                 *fbc_wm = *display_wm = *cursor_wm = 0;
1775                 return false;
1776         }
1777
1778         crtc = intel_get_crtc_for_plane(dev, plane);
1779         adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1780         clock = adjusted_mode->crtc_clock;
1781         htotal = adjusted_mode->htotal;
1782         hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
1783         pixel_size = crtc->fb->bits_per_pixel / 8;
1784
1785         line_time_us = (htotal * 1000) / clock;
1786         line_count = (latency_ns / line_time_us + 1000) / 1000;
1787         line_size = hdisplay * pixel_size;
1788
1789         /* Use the minimum of the small and large buffer method for primary */
1790         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
1791         large = line_count * line_size;
1792
1793         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
1794         *display_wm = entries + display->guard_size;
1795
1796         /*
1797          * Spec says:
1798          * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
1799          */
1800         *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
1801
1802         /* calculate the self-refresh watermark for display cursor */
1803         entries = line_count * pixel_size * 64;
1804         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
1805         *cursor_wm = entries + cursor->guard_size;
1806
1807         return ironlake_check_srwm(dev, level,
1808                                    *fbc_wm, *display_wm, *cursor_wm,
1809                                    display, cursor);
1810 }
1811
1812 static void ironlake_update_wm(struct drm_crtc *crtc)
1813 {
1814         struct drm_device *dev = crtc->dev;
1815         struct drm_i915_private *dev_priv = dev->dev_private;
1816         int fbc_wm, plane_wm, cursor_wm;
1817         unsigned int enabled;
1818
1819         enabled = 0;
1820         if (g4x_compute_wm0(dev, PIPE_A,
1821                             &ironlake_display_wm_info,
1822                             dev_priv->wm.pri_latency[0] * 100,
1823                             &ironlake_cursor_wm_info,
1824                             dev_priv->wm.cur_latency[0] * 100,
1825                             &plane_wm, &cursor_wm)) {
1826                 I915_WRITE(WM0_PIPEA_ILK,
1827                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
1828                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1829                               " plane %d, " "cursor: %d\n",
1830                               plane_wm, cursor_wm);
1831                 enabled |= 1 << PIPE_A;
1832         }
1833
1834         if (g4x_compute_wm0(dev, PIPE_B,
1835                             &ironlake_display_wm_info,
1836                             dev_priv->wm.pri_latency[0] * 100,
1837                             &ironlake_cursor_wm_info,
1838                             dev_priv->wm.cur_latency[0] * 100,
1839                             &plane_wm, &cursor_wm)) {
1840                 I915_WRITE(WM0_PIPEB_ILK,
1841                            (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
1842                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1843                               " plane %d, cursor: %d\n",
1844                               plane_wm, cursor_wm);
1845                 enabled |= 1 << PIPE_B;
1846         }
1847
1848         /*
1849          * Calculate and update the self-refresh watermark only when one
1850          * display plane is used.
1851          */
1852         I915_WRITE(WM3_LP_ILK, 0);
1853         I915_WRITE(WM2_LP_ILK, 0);
1854         I915_WRITE(WM1_LP_ILK, 0);
1855
1856         if (!single_plane_enabled(enabled))
1857                 return;
1858         enabled = ffs(enabled) - 1;
1859
1860         /* WM1 */
1861         if (!ironlake_compute_srwm(dev, 1, enabled,
1862                                    dev_priv->wm.pri_latency[1] * 500,
1863                                    &ironlake_display_srwm_info,
1864                                    &ironlake_cursor_srwm_info,
1865                                    &fbc_wm, &plane_wm, &cursor_wm))
1866                 return;
1867
1868         I915_WRITE(WM1_LP_ILK,
1869                    WM1_LP_SR_EN |
1870                    (dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) |
1871                    (fbc_wm << WM1_LP_FBC_SHIFT) |
1872                    (plane_wm << WM1_LP_SR_SHIFT) |
1873                    cursor_wm);
1874
1875         /* WM2 */
1876         if (!ironlake_compute_srwm(dev, 2, enabled,
1877                                    dev_priv->wm.pri_latency[2] * 500,
1878                                    &ironlake_display_srwm_info,
1879                                    &ironlake_cursor_srwm_info,
1880                                    &fbc_wm, &plane_wm, &cursor_wm))
1881                 return;
1882
1883         I915_WRITE(WM2_LP_ILK,
1884                    WM2_LP_EN |
1885                    (dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) |
1886                    (fbc_wm << WM1_LP_FBC_SHIFT) |
1887                    (plane_wm << WM1_LP_SR_SHIFT) |
1888                    cursor_wm);
1889
1890         /*
1891          * WM3 is unsupported on ILK, probably because we don't have latency
1892          * data for that power state
1893          */
1894 }
1895
1896 static void sandybridge_update_wm(struct drm_crtc *crtc)
1897 {
1898         struct drm_device *dev = crtc->dev;
1899         struct drm_i915_private *dev_priv = dev->dev_private;
1900         int latency = dev_priv->wm.pri_latency[0] * 100;        /* In unit 0.1us */
1901         u32 val;
1902         int fbc_wm, plane_wm, cursor_wm;
1903         unsigned int enabled;
1904
1905         enabled = 0;
1906         if (g4x_compute_wm0(dev, PIPE_A,
1907                             &sandybridge_display_wm_info, latency,
1908                             &sandybridge_cursor_wm_info, latency,
1909                             &plane_wm, &cursor_wm)) {
1910                 val = I915_READ(WM0_PIPEA_ILK);
1911                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
1912                 I915_WRITE(WM0_PIPEA_ILK, val |
1913                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
1914                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
1915                               " plane %d, " "cursor: %d\n",
1916                               plane_wm, cursor_wm);
1917                 enabled |= 1 << PIPE_A;
1918         }
1919
1920         if (g4x_compute_wm0(dev, PIPE_B,
1921                             &sandybridge_display_wm_info, latency,
1922                             &sandybridge_cursor_wm_info, latency,
1923                             &plane_wm, &cursor_wm)) {
1924                 val = I915_READ(WM0_PIPEB_ILK);
1925                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
1926                 I915_WRITE(WM0_PIPEB_ILK, val |
1927                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
1928                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
1929                               " plane %d, cursor: %d\n",
1930                               plane_wm, cursor_wm);
1931                 enabled |= 1 << PIPE_B;
1932         }
1933
1934         /*
1935          * Calculate and update the self-refresh watermark only when one
1936          * display plane is used.
1937          *
1938          * SNB support 3 levels of watermark.
1939          *
1940          * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
1941          * and disabled in the descending order
1942          *
1943          */
1944         I915_WRITE(WM3_LP_ILK, 0);
1945         I915_WRITE(WM2_LP_ILK, 0);
1946         I915_WRITE(WM1_LP_ILK, 0);
1947
1948         if (!single_plane_enabled(enabled) ||
1949             dev_priv->sprite_scaling_enabled)
1950                 return;
1951         enabled = ffs(enabled) - 1;
1952
1953         /* WM1 */
1954         if (!ironlake_compute_srwm(dev, 1, enabled,
1955                                    dev_priv->wm.pri_latency[1] * 500,
1956                                    &sandybridge_display_srwm_info,
1957                                    &sandybridge_cursor_srwm_info,
1958                                    &fbc_wm, &plane_wm, &cursor_wm))
1959                 return;
1960
1961         I915_WRITE(WM1_LP_ILK,
1962                    WM1_LP_SR_EN |
1963                    (dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) |
1964                    (fbc_wm << WM1_LP_FBC_SHIFT) |
1965                    (plane_wm << WM1_LP_SR_SHIFT) |
1966                    cursor_wm);
1967
1968         /* WM2 */
1969         if (!ironlake_compute_srwm(dev, 2, enabled,
1970                                    dev_priv->wm.pri_latency[2] * 500,
1971                                    &sandybridge_display_srwm_info,
1972                                    &sandybridge_cursor_srwm_info,
1973                                    &fbc_wm, &plane_wm, &cursor_wm))
1974                 return;
1975
1976         I915_WRITE(WM2_LP_ILK,
1977                    WM2_LP_EN |
1978                    (dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) |
1979                    (fbc_wm << WM1_LP_FBC_SHIFT) |
1980                    (plane_wm << WM1_LP_SR_SHIFT) |
1981                    cursor_wm);
1982
1983         /* WM3 */
1984         if (!ironlake_compute_srwm(dev, 3, enabled,
1985                                    dev_priv->wm.pri_latency[3] * 500,
1986                                    &sandybridge_display_srwm_info,
1987                                    &sandybridge_cursor_srwm_info,
1988                                    &fbc_wm, &plane_wm, &cursor_wm))
1989                 return;
1990
1991         I915_WRITE(WM3_LP_ILK,
1992                    WM3_LP_EN |
1993                    (dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) |
1994                    (fbc_wm << WM1_LP_FBC_SHIFT) |
1995                    (plane_wm << WM1_LP_SR_SHIFT) |
1996                    cursor_wm);
1997 }
1998
1999 static void ivybridge_update_wm(struct drm_crtc *crtc)
2000 {
2001         struct drm_device *dev = crtc->dev;
2002         struct drm_i915_private *dev_priv = dev->dev_private;
2003         int latency = dev_priv->wm.pri_latency[0] * 100;        /* In unit 0.1us */
2004         u32 val;
2005         int fbc_wm, plane_wm, cursor_wm;
2006         int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
2007         unsigned int enabled;
2008
2009         enabled = 0;
2010         if (g4x_compute_wm0(dev, PIPE_A,
2011                             &sandybridge_display_wm_info, latency,
2012                             &sandybridge_cursor_wm_info, latency,
2013                             &plane_wm, &cursor_wm)) {
2014                 val = I915_READ(WM0_PIPEA_ILK);
2015                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
2016                 I915_WRITE(WM0_PIPEA_ILK, val |
2017                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
2018                 DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
2019                               " plane %d, " "cursor: %d\n",
2020                               plane_wm, cursor_wm);
2021                 enabled |= 1 << PIPE_A;
2022         }
2023
2024         if (g4x_compute_wm0(dev, PIPE_B,
2025                             &sandybridge_display_wm_info, latency,
2026                             &sandybridge_cursor_wm_info, latency,
2027                             &plane_wm, &cursor_wm)) {
2028                 val = I915_READ(WM0_PIPEB_ILK);
2029                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
2030                 I915_WRITE(WM0_PIPEB_ILK, val |
2031                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
2032                 DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
2033                               " plane %d, cursor: %d\n",
2034                               plane_wm, cursor_wm);
2035                 enabled |= 1 << PIPE_B;
2036         }
2037
2038         if (g4x_compute_wm0(dev, PIPE_C,
2039                             &sandybridge_display_wm_info, latency,
2040                             &sandybridge_cursor_wm_info, latency,
2041                             &plane_wm, &cursor_wm)) {
2042                 val = I915_READ(WM0_PIPEC_IVB);
2043                 val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
2044                 I915_WRITE(WM0_PIPEC_IVB, val |
2045                            ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
2046                 DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
2047                               " plane %d, cursor: %d\n",
2048                               plane_wm, cursor_wm);
2049                 enabled |= 1 << PIPE_C;
2050         }
2051
2052         /*
2053          * Calculate and update the self-refresh watermark only when one
2054          * display plane is used.
2055          *
2056          * SNB support 3 levels of watermark.
2057          *
2058          * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
2059          * and disabled in the descending order
2060          *
2061          */
2062         I915_WRITE(WM3_LP_ILK, 0);
2063         I915_WRITE(WM2_LP_ILK, 0);
2064         I915_WRITE(WM1_LP_ILK, 0);
2065
2066         if (!single_plane_enabled(enabled) ||
2067             dev_priv->sprite_scaling_enabled)
2068                 return;
2069         enabled = ffs(enabled) - 1;
2070
2071         /* WM1 */
2072         if (!ironlake_compute_srwm(dev, 1, enabled,
2073                                    dev_priv->wm.pri_latency[1] * 500,
2074                                    &sandybridge_display_srwm_info,
2075                                    &sandybridge_cursor_srwm_info,
2076                                    &fbc_wm, &plane_wm, &cursor_wm))
2077                 return;
2078
2079         I915_WRITE(WM1_LP_ILK,
2080                    WM1_LP_SR_EN |
2081                    (dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) |
2082                    (fbc_wm << WM1_LP_FBC_SHIFT) |
2083                    (plane_wm << WM1_LP_SR_SHIFT) |
2084                    cursor_wm);
2085
2086         /* WM2 */
2087         if (!ironlake_compute_srwm(dev, 2, enabled,
2088                                    dev_priv->wm.pri_latency[2] * 500,
2089                                    &sandybridge_display_srwm_info,
2090                                    &sandybridge_cursor_srwm_info,
2091                                    &fbc_wm, &plane_wm, &cursor_wm))
2092                 return;
2093
2094         I915_WRITE(WM2_LP_ILK,
2095                    WM2_LP_EN |
2096                    (dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) |
2097                    (fbc_wm << WM1_LP_FBC_SHIFT) |
2098                    (plane_wm << WM1_LP_SR_SHIFT) |
2099                    cursor_wm);
2100
2101         /* WM3, note we have to correct the cursor latency */
2102         if (!ironlake_compute_srwm(dev, 3, enabled,
2103                                    dev_priv->wm.pri_latency[3] * 500,
2104                                    &sandybridge_display_srwm_info,
2105                                    &sandybridge_cursor_srwm_info,
2106                                    &fbc_wm, &plane_wm, &ignore_cursor_wm) ||
2107             !ironlake_compute_srwm(dev, 3, enabled,
2108                                    dev_priv->wm.cur_latency[3] * 500,
2109                                    &sandybridge_display_srwm_info,
2110                                    &sandybridge_cursor_srwm_info,
2111                                    &ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
2112                 return;
2113
2114         I915_WRITE(WM3_LP_ILK,
2115                    WM3_LP_EN |
2116                    (dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) |
2117                    (fbc_wm << WM1_LP_FBC_SHIFT) |
2118                    (plane_wm << WM1_LP_SR_SHIFT) |
2119                    cursor_wm);
2120 }
2121
2122 static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
2123                                     struct drm_crtc *crtc)
2124 {
2125         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2126         uint32_t pixel_rate;
2127
2128         pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
2129
2130         /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
2131          * adjust the pixel_rate here. */
2132
2133         if (intel_crtc->config.pch_pfit.enabled) {
2134                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
2135                 uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
2136
2137                 pipe_w = intel_crtc->config.pipe_src_w;
2138                 pipe_h = intel_crtc->config.pipe_src_h;
2139                 pfit_w = (pfit_size >> 16) & 0xFFFF;
2140                 pfit_h = pfit_size & 0xFFFF;
2141                 if (pipe_w < pfit_w)
2142                         pipe_w = pfit_w;
2143                 if (pipe_h < pfit_h)
2144                         pipe_h = pfit_h;
2145
2146                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
2147                                      pfit_w * pfit_h);
2148         }
2149
2150         return pixel_rate;
2151 }
2152
2153 /* latency must be in 0.1us units. */
2154 static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
2155                                uint32_t latency)
2156 {
2157         uint64_t ret;
2158
2159         if (WARN(latency == 0, "Latency value missing\n"))
2160                 return UINT_MAX;
2161
2162         ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
2163         ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
2164
2165         return ret;
2166 }
2167
2168 /* latency must be in 0.1us units. */
2169 static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
2170                                uint32_t horiz_pixels, uint8_t bytes_per_pixel,
2171                                uint32_t latency)
2172 {
2173         uint32_t ret;
2174
2175         if (WARN(latency == 0, "Latency value missing\n"))
2176                 return UINT_MAX;
2177
2178         ret = (latency * pixel_rate) / (pipe_htotal * 10000);
2179         ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
2180         ret = DIV_ROUND_UP(ret, 64) + 2;
2181         return ret;
2182 }
2183
2184 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
2185                            uint8_t bytes_per_pixel)
2186 {
2187         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
2188 }
2189
2190 struct hsw_pipe_wm_parameters {
2191         bool active;
2192         uint32_t pipe_htotal;
2193         uint32_t pixel_rate;
2194         struct intel_plane_wm_parameters pri;
2195         struct intel_plane_wm_parameters spr;
2196         struct intel_plane_wm_parameters cur;
2197 };
2198
2199 struct hsw_wm_maximums {
2200         uint16_t pri;
2201         uint16_t spr;
2202         uint16_t cur;
2203         uint16_t fbc;
2204 };
2205
2206 /* used in computing the new watermarks state */
2207 struct intel_wm_config {
2208         unsigned int num_pipes_active;
2209         bool sprites_enabled;
2210         bool sprites_scaled;
2211 };
2212
2213 /*
2214  * For both WM_PIPE and WM_LP.
2215  * mem_value must be in 0.1us units.
2216  */
2217 static uint32_t ilk_compute_pri_wm(const struct hsw_pipe_wm_parameters *params,
2218                                    uint32_t mem_value,
2219                                    bool is_lp)
2220 {
2221         uint32_t method1, method2;
2222
2223         if (!params->active || !params->pri.enabled)
2224                 return 0;
2225
2226         method1 = ilk_wm_method1(params->pixel_rate,
2227                                  params->pri.bytes_per_pixel,
2228                                  mem_value);
2229
2230         if (!is_lp)
2231                 return method1;
2232
2233         method2 = ilk_wm_method2(params->pixel_rate,
2234                                  params->pipe_htotal,
2235                                  params->pri.horiz_pixels,
2236                                  params->pri.bytes_per_pixel,
2237                                  mem_value);
2238
2239         return min(method1, method2);
2240 }
2241
2242 /*
2243  * For both WM_PIPE and WM_LP.
2244  * mem_value must be in 0.1us units.
2245  */
2246 static uint32_t ilk_compute_spr_wm(const struct hsw_pipe_wm_parameters *params,
2247                                    uint32_t mem_value)
2248 {
2249         uint32_t method1, method2;
2250
2251         if (!params->active || !params->spr.enabled)
2252                 return 0;
2253
2254         method1 = ilk_wm_method1(params->pixel_rate,
2255                                  params->spr.bytes_per_pixel,
2256                                  mem_value);
2257         method2 = ilk_wm_method2(params->pixel_rate,
2258                                  params->pipe_htotal,
2259                                  params->spr.horiz_pixels,
2260                                  params->spr.bytes_per_pixel,
2261                                  mem_value);
2262         return min(method1, method2);
2263 }
2264
2265 /*
2266  * For both WM_PIPE and WM_LP.
2267  * mem_value must be in 0.1us units.
2268  */
2269 static uint32_t ilk_compute_cur_wm(const struct hsw_pipe_wm_parameters *params,
2270                                    uint32_t mem_value)
2271 {
2272         if (!params->active || !params->cur.enabled)
2273                 return 0;
2274
2275         return ilk_wm_method2(params->pixel_rate,
2276                               params->pipe_htotal,
2277                               params->cur.horiz_pixels,
2278                               params->cur.bytes_per_pixel,
2279                               mem_value);
2280 }
2281
2282 /* Only for WM_LP. */
2283 static uint32_t ilk_compute_fbc_wm(const struct hsw_pipe_wm_parameters *params,
2284                                    uint32_t pri_val)
2285 {
2286         if (!params->active || !params->pri.enabled)
2287                 return 0;
2288
2289         return ilk_wm_fbc(pri_val,
2290                           params->pri.horiz_pixels,
2291                           params->pri.bytes_per_pixel);
2292 }
2293
2294 static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
2295 {
2296         if (INTEL_INFO(dev)->gen >= 8)
2297                 return 3072;
2298         else if (INTEL_INFO(dev)->gen >= 7)
2299                 return 768;
2300         else
2301                 return 512;
2302 }
2303
2304 /* Calculate the maximum primary/sprite plane watermark */
2305 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
2306                                      int level,
2307                                      const struct intel_wm_config *config,
2308                                      enum intel_ddb_partitioning ddb_partitioning,
2309                                      bool is_sprite)
2310 {
2311         unsigned int fifo_size = ilk_display_fifo_size(dev);
2312         unsigned int max;
2313
2314         /* if sprites aren't enabled, sprites get nothing */
2315         if (is_sprite && !config->sprites_enabled)
2316                 return 0;
2317
2318         /* HSW allows LP1+ watermarks even with multiple pipes */
2319         if (level == 0 || config->num_pipes_active > 1) {
2320                 fifo_size /= INTEL_INFO(dev)->num_pipes;
2321
2322                 /*
2323                  * For some reason the non self refresh
2324                  * FIFO size is only half of the self
2325                  * refresh FIFO size on ILK/SNB.
2326                  */
2327                 if (INTEL_INFO(dev)->gen <= 6)
2328                         fifo_size /= 2;
2329         }
2330
2331         if (config->sprites_enabled) {
2332                 /* level 0 is always calculated with 1:1 split */
2333                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2334                         if (is_sprite)
2335                                 fifo_size *= 5;
2336                         fifo_size /= 6;
2337                 } else {
2338                         fifo_size /= 2;
2339                 }
2340         }
2341
2342         /* clamp to max that the registers can hold */
2343         if (INTEL_INFO(dev)->gen >= 8)
2344                 max = level == 0 ? 255 : 2047;
2345         else if (INTEL_INFO(dev)->gen >= 7)
2346                 /* IVB/HSW primary/sprite plane watermarks */
2347                 max = level == 0 ? 127 : 1023;
2348         else if (!is_sprite)
2349                 /* ILK/SNB primary plane watermarks */
2350                 max = level == 0 ? 127 : 511;
2351         else
2352                 /* ILK/SNB sprite plane watermarks */
2353                 max = level == 0 ? 63 : 255;
2354
2355         return min(fifo_size, max);
2356 }
2357
2358 /* Calculate the maximum cursor plane watermark */
2359 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
2360                                       int level,
2361                                       const struct intel_wm_config *config)
2362 {
2363         /* HSW LP1+ watermarks w/ multiple pipes */
2364         if (level > 0 && config->num_pipes_active > 1)
2365                 return 64;
2366
2367         /* otherwise just report max that registers can hold */
2368         if (INTEL_INFO(dev)->gen >= 7)
2369                 return level == 0 ? 63 : 255;
2370         else
2371                 return level == 0 ? 31 : 63;
2372 }
2373
2374 /* Calculate the maximum FBC watermark */
2375 static unsigned int ilk_fbc_wm_max(struct drm_device *dev)
2376 {
2377         /* max that registers can hold */
2378         if (INTEL_INFO(dev)->gen >= 8)
2379                 return 31;
2380         else
2381                 return 15;
2382 }
2383
2384 static void ilk_compute_wm_maximums(struct drm_device *dev,
2385                                     int level,
2386                                     const struct intel_wm_config *config,
2387                                     enum intel_ddb_partitioning ddb_partitioning,
2388                                     struct hsw_wm_maximums *max)
2389 {
2390         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
2391         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
2392         max->cur = ilk_cursor_wm_max(dev, level, config);
2393         max->fbc = ilk_fbc_wm_max(dev);
2394 }
2395
2396 static bool ilk_validate_wm_level(int level,
2397                                   const struct hsw_wm_maximums *max,
2398                                   struct intel_wm_level *result)
2399 {
2400         bool ret;
2401
2402         /* already determined to be invalid? */
2403         if (!result->enable)
2404                 return false;
2405
2406         result->enable = result->pri_val <= max->pri &&
2407                          result->spr_val <= max->spr &&
2408                          result->cur_val <= max->cur;
2409
2410         ret = result->enable;
2411
2412         /*
2413          * HACK until we can pre-compute everything,
2414          * and thus fail gracefully if LP0 watermarks
2415          * are exceeded...
2416          */
2417         if (level == 0 && !result->enable) {
2418                 if (result->pri_val > max->pri)
2419                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2420                                       level, result->pri_val, max->pri);
2421                 if (result->spr_val > max->spr)
2422                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2423                                       level, result->spr_val, max->spr);
2424                 if (result->cur_val > max->cur)
2425                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2426                                       level, result->cur_val, max->cur);
2427
2428                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
2429                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
2430                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
2431                 result->enable = true;
2432         }
2433
2434         return ret;
2435 }
2436
2437 static void ilk_compute_wm_level(struct drm_i915_private *dev_priv,
2438                                  int level,
2439                                  const struct hsw_pipe_wm_parameters *p,
2440                                  struct intel_wm_level *result)
2441 {
2442         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
2443         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
2444         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
2445
2446         /* WM1+ latency values stored in 0.5us units */
2447         if (level > 0) {
2448                 pri_latency *= 5;
2449                 spr_latency *= 5;
2450                 cur_latency *= 5;
2451         }
2452
2453         result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
2454         result->spr_val = ilk_compute_spr_wm(p, spr_latency);
2455         result->cur_val = ilk_compute_cur_wm(p, cur_latency);
2456         result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
2457         result->enable = true;
2458 }
2459
2460 static uint32_t
2461 hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
2462 {
2463         struct drm_i915_private *dev_priv = dev->dev_private;
2464         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2465         struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
2466         u32 linetime, ips_linetime;
2467
2468         if (!intel_crtc_active(crtc))
2469                 return 0;
2470
2471         /* The WM are computed with base on how long it takes to fill a single
2472          * row at the given clock rate, multiplied by 8.
2473          * */
2474         linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8, mode->clock);
2475         ips_linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8,
2476                                          intel_ddi_get_cdclk_freq(dev_priv));
2477
2478         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
2479                PIPE_WM_LINETIME_TIME(linetime);
2480 }
2481
2482 static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
2483 {
2484         struct drm_i915_private *dev_priv = dev->dev_private;
2485
2486         if (IS_HASWELL(dev)) {
2487                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2488
2489                 wm[0] = (sskpd >> 56) & 0xFF;
2490                 if (wm[0] == 0)
2491                         wm[0] = sskpd & 0xF;
2492                 wm[1] = (sskpd >> 4) & 0xFF;
2493                 wm[2] = (sskpd >> 12) & 0xFF;
2494                 wm[3] = (sskpd >> 20) & 0x1FF;
2495                 wm[4] = (sskpd >> 32) & 0x1FF;
2496         } else if (INTEL_INFO(dev)->gen >= 6) {
2497                 uint32_t sskpd = I915_READ(MCH_SSKPD);
2498
2499                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2500                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2501                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2502                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2503         } else if (INTEL_INFO(dev)->gen >= 5) {
2504                 uint32_t mltr = I915_READ(MLTR_ILK);
2505
2506                 /* ILK primary LP0 latency is 700 ns */
2507                 wm[0] = 7;
2508                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2509                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2510         }
2511 }
2512
2513 static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
2514 {
2515         /* ILK sprite LP0 latency is 1300 ns */
2516         if (INTEL_INFO(dev)->gen == 5)
2517                 wm[0] = 13;
2518 }
2519
2520 static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
2521 {
2522         /* ILK cursor LP0 latency is 1300 ns */
2523         if (INTEL_INFO(dev)->gen == 5)
2524                 wm[0] = 13;
2525
2526         /* WaDoubleCursorLP3Latency:ivb */
2527         if (IS_IVYBRIDGE(dev))
2528                 wm[3] *= 2;
2529 }
2530
2531 static int ilk_wm_max_level(const struct drm_device *dev)
2532 {
2533         /* how many WM levels are we expecting */
2534         if (IS_HASWELL(dev))
2535                 return 4;
2536         else if (INTEL_INFO(dev)->gen >= 6)
2537                 return 3;
2538         else
2539                 return 2;
2540 }
2541
2542 static void intel_print_wm_latency(struct drm_device *dev,
2543                                    const char *name,
2544                                    const uint16_t wm[5])
2545 {
2546         int level, max_level = ilk_wm_max_level(dev);
2547
2548         for (level = 0; level <= max_level; level++) {
2549                 unsigned int latency = wm[level];
2550
2551                 if (latency == 0) {
2552                         DRM_ERROR("%s WM%d latency not provided\n",
2553                                   name, level);
2554                         continue;
2555                 }
2556
2557                 /* WM1+ latency values in 0.5us units */
2558                 if (level > 0)
2559                         latency *= 5;
2560
2561                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2562                               name, level, wm[level],
2563                               latency / 10, latency % 10);
2564         }
2565 }
2566
2567 static void intel_setup_wm_latency(struct drm_device *dev)
2568 {
2569         struct drm_i915_private *dev_priv = dev->dev_private;
2570
2571         intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
2572
2573         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2574                sizeof(dev_priv->wm.pri_latency));
2575         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2576                sizeof(dev_priv->wm.pri_latency));
2577
2578         intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
2579         intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
2580
2581         intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2582         intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2583         intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2584 }
2585
2586 static void hsw_compute_wm_parameters(struct drm_crtc *crtc,
2587                                       struct hsw_pipe_wm_parameters *p,
2588                                       struct intel_wm_config *config)
2589 {
2590         struct drm_device *dev = crtc->dev;
2591         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2592         enum pipe pipe = intel_crtc->pipe;
2593         struct drm_plane *plane;
2594
2595         p->active = intel_crtc_active(crtc);
2596         if (p->active) {
2597                 p->pipe_htotal = intel_crtc->config.adjusted_mode.htotal;
2598                 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
2599                 p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8;
2600                 p->cur.bytes_per_pixel = 4;
2601                 p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
2602                 p->cur.horiz_pixels = 64;
2603                 /* TODO: for now, assume primary and cursor planes are always enabled. */
2604                 p->pri.enabled = true;
2605                 p->cur.enabled = true;
2606         }
2607
2608         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
2609                 config->num_pipes_active += intel_crtc_active(crtc);
2610
2611         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2612                 struct intel_plane *intel_plane = to_intel_plane(plane);
2613
2614                 if (intel_plane->pipe == pipe)
2615                         p->spr = intel_plane->wm;
2616
2617                 config->sprites_enabled |= intel_plane->wm.enabled;
2618                 config->sprites_scaled |= intel_plane->wm.scaled;
2619         }
2620 }
2621
2622 /* Compute new watermarks for the pipe */
2623 static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
2624                                   const struct hsw_pipe_wm_parameters *params,
2625                                   struct intel_pipe_wm *pipe_wm)
2626 {
2627         struct drm_device *dev = crtc->dev;
2628         struct drm_i915_private *dev_priv = dev->dev_private;
2629         int level, max_level = ilk_wm_max_level(dev);
2630         /* LP0 watermark maximums depend on this pipe alone */
2631         struct intel_wm_config config = {
2632                 .num_pipes_active = 1,
2633                 .sprites_enabled = params->spr.enabled,
2634                 .sprites_scaled = params->spr.scaled,
2635         };
2636         struct hsw_wm_maximums max;
2637
2638         /* LP0 watermarks always use 1/2 DDB partitioning */
2639         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2640
2641         for (level = 0; level <= max_level; level++)
2642                 ilk_compute_wm_level(dev_priv, level, params,
2643                                      &pipe_wm->wm[level]);
2644
2645         pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
2646
2647         /* At least LP0 must be valid */
2648         return ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]);
2649 }
2650
2651 /*
2652  * Merge the watermarks from all active pipes for a specific level.
2653  */
2654 static void ilk_merge_wm_level(struct drm_device *dev,
2655                                int level,
2656                                struct intel_wm_level *ret_wm)
2657 {
2658         const struct intel_crtc *intel_crtc;
2659
2660         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
2661                 const struct intel_wm_level *wm =
2662                         &intel_crtc->wm.active.wm[level];
2663
2664                 if (!wm->enable)
2665                         return;
2666
2667                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2668                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2669                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2670                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2671         }
2672
2673         ret_wm->enable = true;
2674 }
2675
2676 /*
2677  * Merge all low power watermarks for all active pipes.
2678  */
2679 static void ilk_wm_merge(struct drm_device *dev,
2680                          const struct hsw_wm_maximums *max,
2681                          struct intel_pipe_wm *merged)
2682 {
2683         int level, max_level = ilk_wm_max_level(dev);
2684
2685         merged->fbc_wm_enabled = true;
2686
2687         /* merge each WM1+ level */
2688         for (level = 1; level <= max_level; level++) {
2689                 struct intel_wm_level *wm = &merged->wm[level];
2690
2691                 ilk_merge_wm_level(dev, level, wm);
2692
2693                 if (!ilk_validate_wm_level(level, max, wm))
2694                         break;
2695
2696                 /*
2697                  * The spec says it is preferred to disable
2698                  * FBC WMs instead of disabling a WM level.
2699                  */
2700                 if (wm->fbc_val > max->fbc) {
2701                         merged->fbc_wm_enabled = false;
2702                         wm->fbc_val = 0;
2703                 }
2704         }
2705 }
2706
2707 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2708 {
2709         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2710         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2711 }
2712
2713 static void hsw_compute_wm_results(struct drm_device *dev,
2714                                    const struct intel_pipe_wm *merged,
2715                                    enum intel_ddb_partitioning partitioning,
2716                                    struct hsw_wm_values *results)
2717 {
2718         struct intel_crtc *intel_crtc;
2719         int level, wm_lp;
2720
2721         results->enable_fbc_wm = merged->fbc_wm_enabled;
2722         results->partitioning = partitioning;
2723
2724         /* LP1+ register values */
2725         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2726                 const struct intel_wm_level *r;
2727
2728                 level = ilk_wm_lp_to_level(wm_lp, merged);
2729
2730                 r = &merged->wm[level];
2731                 if (!r->enable)
2732                         break;
2733
2734                 results->wm_lp[wm_lp - 1] = WM3_LP_EN |
2735                         ((level * 2) << WM1_LP_LATENCY_SHIFT) |
2736                         (r->pri_val << WM1_LP_SR_SHIFT) |
2737                         r->cur_val;
2738
2739                 if (INTEL_INFO(dev)->gen >= 8)
2740                         results->wm_lp[wm_lp - 1] |=
2741                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2742                 else
2743                         results->wm_lp[wm_lp - 1] |=
2744                                 r->fbc_val << WM1_LP_FBC_SHIFT;
2745
2746                 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
2747         }
2748
2749         /* LP0 register values */
2750         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
2751                 enum pipe pipe = intel_crtc->pipe;
2752                 const struct intel_wm_level *r =
2753                         &intel_crtc->wm.active.wm[0];
2754
2755                 if (WARN_ON(!r->enable))
2756                         continue;
2757
2758                 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
2759
2760                 results->wm_pipe[pipe] =
2761                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2762                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2763                         r->cur_val;
2764         }
2765 }
2766
2767 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
2768  * case both are at the same level. Prefer r1 in case they're the same. */
2769 static struct intel_pipe_wm *hsw_find_best_result(struct drm_device *dev,
2770                                                   struct intel_pipe_wm *r1,
2771                                                   struct intel_pipe_wm *r2)
2772 {
2773         int level, max_level = ilk_wm_max_level(dev);
2774         int level1 = 0, level2 = 0;
2775
2776         for (level = 1; level <= max_level; level++) {
2777                 if (r1->wm[level].enable)
2778                         level1 = level;
2779                 if (r2->wm[level].enable)
2780                         level2 = level;
2781         }
2782
2783         if (level1 == level2) {
2784                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
2785                         return r2;
2786                 else
2787                         return r1;
2788         } else if (level1 > level2) {
2789                 return r1;
2790         } else {
2791                 return r2;
2792         }
2793 }
2794
2795 /* dirty bits used to track which watermarks need changes */
2796 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2797 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2798 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2799 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2800 #define WM_DIRTY_FBC (1 << 24)
2801 #define WM_DIRTY_DDB (1 << 25)
2802
2803 static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
2804                                          const struct hsw_wm_values *old,
2805                                          const struct hsw_wm_values *new)
2806 {
2807         unsigned int dirty = 0;
2808         enum pipe pipe;
2809         int wm_lp;
2810
2811         for_each_pipe(pipe) {
2812                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2813                         dirty |= WM_DIRTY_LINETIME(pipe);
2814                         /* Must disable LP1+ watermarks too */
2815                         dirty |= WM_DIRTY_LP_ALL;
2816                 }
2817
2818                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2819                         dirty |= WM_DIRTY_PIPE(pipe);
2820                         /* Must disable LP1+ watermarks too */
2821                         dirty |= WM_DIRTY_LP_ALL;
2822                 }
2823         }
2824
2825         if (old->enable_fbc_wm != new->enable_fbc_wm) {
2826                 dirty |= WM_DIRTY_FBC;
2827                 /* Must disable LP1+ watermarks too */
2828                 dirty |= WM_DIRTY_LP_ALL;
2829         }
2830
2831         if (old->partitioning != new->partitioning) {
2832                 dirty |= WM_DIRTY_DDB;
2833                 /* Must disable LP1+ watermarks too */
2834                 dirty |= WM_DIRTY_LP_ALL;
2835         }
2836
2837         /* LP1+ watermarks already deemed dirty, no need to continue */
2838         if (dirty & WM_DIRTY_LP_ALL)
2839                 return dirty;
2840
2841         /* Find the lowest numbered LP1+ watermark in need of an update... */
2842         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2843                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2844                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2845                         break;
2846         }
2847
2848         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2849         for (; wm_lp <= 3; wm_lp++)
2850                 dirty |= WM_DIRTY_LP(wm_lp);
2851
2852         return dirty;
2853 }
2854
2855 /*
2856  * The spec says we shouldn't write when we don't need, because every write
2857  * causes WMs to be re-evaluated, expending some power.
2858  */
2859 static void hsw_write_wm_values(struct drm_i915_private *dev_priv,
2860                                 struct hsw_wm_values *results)
2861 {
2862         struct hsw_wm_values *previous = &dev_priv->wm.hw;
2863         unsigned int dirty;
2864         uint32_t val;
2865
2866         dirty = ilk_compute_wm_dirty(dev_priv->dev, previous, results);
2867         if (!dirty)
2868                 return;
2869
2870         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != 0)
2871                 I915_WRITE(WM3_LP_ILK, 0);
2872         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != 0)
2873                 I915_WRITE(WM2_LP_ILK, 0);
2874         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != 0)
2875                 I915_WRITE(WM1_LP_ILK, 0);
2876
2877         if (dirty & WM_DIRTY_PIPE(PIPE_A))
2878                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
2879         if (dirty & WM_DIRTY_PIPE(PIPE_B))
2880                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
2881         if (dirty & WM_DIRTY_PIPE(PIPE_C))
2882                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2883
2884         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
2885                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
2886         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
2887                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
2888         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
2889                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2890
2891         if (dirty & WM_DIRTY_DDB) {
2892                 val = I915_READ(WM_MISC);
2893                 if (results->partitioning == INTEL_DDB_PART_1_2)
2894                         val &= ~WM_MISC_DATA_PARTITION_5_6;
2895                 else
2896                         val |= WM_MISC_DATA_PARTITION_5_6;
2897                 I915_WRITE(WM_MISC, val);
2898         }
2899
2900         if (dirty & WM_DIRTY_FBC) {
2901                 val = I915_READ(DISP_ARB_CTL);
2902                 if (results->enable_fbc_wm)
2903                         val &= ~DISP_FBC_WM_DIS;
2904                 else
2905                         val |= DISP_FBC_WM_DIS;
2906                 I915_WRITE(DISP_ARB_CTL, val);
2907         }
2908
2909         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2910                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2911         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2912                 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2913         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2914                 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2915
2916         if (dirty & WM_DIRTY_LP(1) && results->wm_lp[0] != 0)
2917                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
2918         if (dirty & WM_DIRTY_LP(2) && results->wm_lp[1] != 0)
2919                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
2920         if (dirty & WM_DIRTY_LP(3) && results->wm_lp[2] != 0)
2921                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
2922
2923         dev_priv->wm.hw = *results;
2924 }
2925
2926 static void haswell_update_wm(struct drm_crtc *crtc)
2927 {
2928         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2929         struct drm_device *dev = crtc->dev;
2930         struct drm_i915_private *dev_priv = dev->dev_private;
2931         struct hsw_wm_maximums max;
2932         struct hsw_pipe_wm_parameters params = {};
2933         struct hsw_wm_values results = {};
2934         enum intel_ddb_partitioning partitioning;
2935         struct intel_pipe_wm pipe_wm = {};
2936         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
2937         struct intel_wm_config config = {};
2938
2939         hsw_compute_wm_parameters(crtc, &params, &config);
2940
2941         intel_compute_pipe_wm(crtc, &params, &pipe_wm);
2942
2943         if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
2944                 return;
2945
2946         intel_crtc->wm.active = pipe_wm;
2947
2948         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
2949         ilk_wm_merge(dev, &max, &lp_wm_1_2);
2950
2951         /* 5/6 split only in single pipe config on IVB+ */
2952         if (INTEL_INFO(dev)->gen >= 7 &&
2953             config.num_pipes_active == 1 && config.sprites_enabled) {
2954                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
2955                 ilk_wm_merge(dev, &max, &lp_wm_5_6);
2956
2957                 best_lp_wm = hsw_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
2958         } else {
2959                 best_lp_wm = &lp_wm_1_2;
2960         }
2961
2962         partitioning = (best_lp_wm == &lp_wm_1_2) ?
2963                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
2964
2965         hsw_compute_wm_results(dev, best_lp_wm, partitioning, &results);
2966
2967         hsw_write_wm_values(dev_priv, &results);
2968 }
2969
2970 static void haswell_update_sprite_wm(struct drm_plane *plane,
2971                                      struct drm_crtc *crtc,
2972                                      uint32_t sprite_width, int pixel_size,
2973                                      bool enabled, bool scaled)
2974 {
2975         struct intel_plane *intel_plane = to_intel_plane(plane);
2976
2977         intel_plane->wm.enabled = enabled;
2978         intel_plane->wm.scaled = scaled;
2979         intel_plane->wm.horiz_pixels = sprite_width;
2980         intel_plane->wm.bytes_per_pixel = pixel_size;
2981
2982         haswell_update_wm(crtc);
2983 }
2984
2985 static bool
2986 sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
2987                               uint32_t sprite_width, int pixel_size,
2988                               const struct intel_watermark_params *display,
2989                               int display_latency_ns, int *sprite_wm)
2990 {
2991         struct drm_crtc *crtc;
2992         int clock;
2993         int entries, tlb_miss;
2994
2995         crtc = intel_get_crtc_for_plane(dev, plane);
2996         if (!intel_crtc_active(crtc)) {
2997                 *sprite_wm = display->guard_size;
2998                 return false;
2999         }
3000
3001         clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
3002
3003         /* Use the small buffer method to calculate the sprite watermark */
3004         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
3005         tlb_miss = display->fifo_size*display->cacheline_size -
3006                 sprite_width * 8;
3007         if (tlb_miss > 0)
3008                 entries += tlb_miss;
3009         entries = DIV_ROUND_UP(entries, display->cacheline_size);
3010         *sprite_wm = entries + display->guard_size;
3011         if (*sprite_wm > (int)display->max_wm)
3012                 *sprite_wm = display->max_wm;
3013
3014         return true;
3015 }
3016
3017 static bool
3018 sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
3019                                 uint32_t sprite_width, int pixel_size,
3020                                 const struct intel_watermark_params *display,
3021                                 int latency_ns, int *sprite_wm)
3022 {
3023         struct drm_crtc *crtc;
3024         unsigned long line_time_us;
3025         int clock;
3026         int line_count, line_size;
3027         int small, large;
3028         int entries;
3029
3030         if (!latency_ns) {
3031                 *sprite_wm = 0;
3032                 return false;
3033         }
3034
3035         crtc = intel_get_crtc_for_plane(dev, plane);
3036         clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
3037         if (!clock) {
3038                 *sprite_wm = 0;
3039                 return false;
3040         }
3041
3042         line_time_us = (sprite_width * 1000) / clock;
3043         if (!line_time_us) {
3044                 *sprite_wm = 0;
3045                 return false;
3046         }
3047
3048         line_count = (latency_ns / line_time_us + 1000) / 1000;
3049         line_size = sprite_width * pixel_size;
3050
3051         /* Use the minimum of the small and large buffer method for primary */
3052         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
3053         large = line_count * line_size;
3054
3055         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
3056         *sprite_wm = entries + display->guard_size;
3057
3058         return *sprite_wm > 0x3ff ? false : true;
3059 }
3060
3061 static void sandybridge_update_sprite_wm(struct drm_plane *plane,
3062                                          struct drm_crtc *crtc,
3063                                          uint32_t sprite_width, int pixel_size,
3064                                          bool enabled, bool scaled)
3065 {
3066         struct drm_device *dev = plane->dev;
3067         struct drm_i915_private *dev_priv = dev->dev_private;
3068         int pipe = to_intel_plane(plane)->pipe;
3069         int latency = dev_priv->wm.spr_latency[0] * 100;        /* In unit 0.1us */
3070         u32 val;
3071         int sprite_wm, reg;
3072         int ret;
3073
3074         if (!enabled)
3075                 return;
3076
3077         switch (pipe) {
3078         case 0:
3079                 reg = WM0_PIPEA_ILK;
3080                 break;
3081         case 1:
3082                 reg = WM0_PIPEB_ILK;
3083                 break;
3084         case 2:
3085                 reg = WM0_PIPEC_IVB;
3086                 break;
3087         default:
3088                 return; /* bad pipe */
3089         }
3090
3091         ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
3092                                             &sandybridge_display_wm_info,
3093                                             latency, &sprite_wm);
3094         if (!ret) {
3095                 DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n",
3096                               pipe_name(pipe));
3097                 return;
3098         }
3099
3100         val = I915_READ(reg);
3101         val &= ~WM0_PIPE_SPRITE_MASK;
3102         I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
3103         DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe), sprite_wm);
3104
3105
3106         ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
3107                                               pixel_size,
3108                                               &sandybridge_display_srwm_info,
3109                                               dev_priv->wm.spr_latency[1] * 500,
3110                                               &sprite_wm);
3111         if (!ret) {
3112                 DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n",
3113                               pipe_name(pipe));
3114                 return;
3115         }
3116         I915_WRITE(WM1S_LP_ILK, sprite_wm);
3117
3118         /* Only IVB has two more LP watermarks for sprite */
3119         if (!IS_IVYBRIDGE(dev))
3120                 return;
3121
3122         ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
3123                                               pixel_size,
3124                                               &sandybridge_display_srwm_info,
3125                                               dev_priv->wm.spr_latency[2] * 500,
3126                                               &sprite_wm);
3127         if (!ret) {
3128                 DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n",
3129                               pipe_name(pipe));
3130                 return;
3131         }
3132         I915_WRITE(WM2S_LP_IVB, sprite_wm);
3133
3134         ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
3135                                               pixel_size,
3136                                               &sandybridge_display_srwm_info,
3137                                               dev_priv->wm.spr_latency[3] * 500,
3138                                               &sprite_wm);
3139         if (!ret) {
3140                 DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n",
3141                               pipe_name(pipe));
3142                 return;
3143         }
3144         I915_WRITE(WM3S_LP_IVB, sprite_wm);
3145 }
3146
3147 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3148 {
3149         struct drm_device *dev = crtc->dev;
3150         struct drm_i915_private *dev_priv = dev->dev_private;
3151         struct hsw_wm_values *hw = &dev_priv->wm.hw;
3152         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3153         struct intel_pipe_wm *active = &intel_crtc->wm.active;
3154         enum pipe pipe = intel_crtc->pipe;
3155         static const unsigned int wm0_pipe_reg[] = {
3156                 [PIPE_A] = WM0_PIPEA_ILK,
3157                 [PIPE_B] = WM0_PIPEB_ILK,
3158                 [PIPE_C] = WM0_PIPEC_IVB,
3159         };
3160
3161         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
3162         hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3163
3164         if (intel_crtc_active(crtc)) {
3165                 u32 tmp = hw->wm_pipe[pipe];
3166
3167                 /*
3168                  * For active pipes LP0 watermark is marked as
3169                  * enabled, and LP1+ watermaks as disabled since
3170                  * we can't really reverse compute them in case
3171                  * multiple pipes are active.
3172                  */
3173                 active->wm[0].enable = true;
3174                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
3175                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
3176                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
3177                 active->linetime = hw->wm_linetime[pipe];
3178         } else {
3179                 int level, max_level = ilk_wm_max_level(dev);
3180
3181                 /*
3182                  * For inactive pipes, all watermark levels
3183                  * should be marked as enabled but zeroed,
3184                  * which is what we'd compute them to.
3185                  */
3186                 for (level = 0; level <= max_level; level++)
3187                         active->wm[level].enable = true;
3188         }
3189 }
3190
3191 void ilk_wm_get_hw_state(struct drm_device *dev)
3192 {
3193         struct drm_i915_private *dev_priv = dev->dev_private;
3194         struct hsw_wm_values *hw = &dev_priv->wm.hw;
3195         struct drm_crtc *crtc;
3196
3197         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3198                 ilk_pipe_wm_get_hw_state(crtc);
3199
3200         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
3201         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
3202         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3203
3204         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
3205         hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
3206         hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
3207
3208         hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
3209                 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3210
3211         hw->enable_fbc_wm =
3212                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3213 }
3214
3215 /**
3216  * intel_update_watermarks - update FIFO watermark values based on current modes
3217  *
3218  * Calculate watermark values for the various WM regs based on current mode
3219  * and plane configuration.
3220  *
3221  * There are several cases to deal with here:
3222  *   - normal (i.e. non-self-refresh)
3223  *   - self-refresh (SR) mode
3224  *   - lines are large relative to FIFO size (buffer can hold up to 2)
3225  *   - lines are small relative to FIFO size (buffer can hold more than 2
3226  *     lines), so need to account for TLB latency
3227  *
3228  *   The normal calculation is:
3229  *     watermark = dotclock * bytes per pixel * latency
3230  *   where latency is platform & configuration dependent (we assume pessimal
3231  *   values here).
3232  *
3233  *   The SR calculation is:
3234  *     watermark = (trunc(latency/line time)+1) * surface width *
3235  *       bytes per pixel
3236  *   where
3237  *     line time = htotal / dotclock
3238  *     surface width = hdisplay for normal plane and 64 for cursor
3239  *   and latency is assumed to be high, as above.
3240  *
3241  * The final value programmed to the register should always be rounded up,
3242  * and include an extra 2 entries to account for clock crossings.
3243  *
3244  * We don't use the sprite, so we can ignore that.  And on Crestline we have
3245  * to set the non-SR watermarks to 8.
3246  */
3247 void intel_update_watermarks(struct drm_crtc *crtc)
3248 {
3249         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3250
3251         if (dev_priv->display.update_wm)
3252                 dev_priv->display.update_wm(crtc);
3253 }
3254
3255 void intel_update_sprite_watermarks(struct drm_plane *plane,
3256                                     struct drm_crtc *crtc,
3257                                     uint32_t sprite_width, int pixel_size,
3258                                     bool enabled, bool scaled)
3259 {
3260         struct drm_i915_private *dev_priv = plane->dev->dev_private;
3261
3262         if (dev_priv->display.update_sprite_wm)
3263                 dev_priv->display.update_sprite_wm(plane, crtc, sprite_width,
3264                                                    pixel_size, enabled, scaled);
3265 }
3266
3267 static struct drm_i915_gem_object *
3268 intel_alloc_context_page(struct drm_device *dev)
3269 {
3270         struct drm_i915_gem_object *ctx;
3271         int ret;
3272
3273         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
3274
3275         ctx = i915_gem_alloc_object(dev, 4096);
3276         if (!ctx) {
3277                 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
3278                 return NULL;
3279         }
3280
3281         ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false);
3282         if (ret) {
3283                 DRM_ERROR("failed to pin power context: %d\n", ret);
3284                 goto err_unref;
3285         }
3286
3287         ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
3288         if (ret) {
3289                 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
3290                 goto err_unpin;
3291         }
3292
3293         return ctx;
3294
3295 err_unpin:
3296         i915_gem_object_unpin(ctx);
3297 err_unref:
3298         drm_gem_object_unreference(&ctx->base);
3299         return NULL;
3300 }
3301
3302 /**
3303  * Lock protecting IPS related data structures
3304  */
3305 DEFINE_SPINLOCK(mchdev_lock);
3306
3307 /* Global for IPS driver to get at the current i915 device. Protected by
3308  * mchdev_lock. */
3309 static struct drm_i915_private *i915_mch_dev;
3310
3311 bool ironlake_set_drps(struct drm_device *dev, u8 val)
3312 {
3313         struct drm_i915_private *dev_priv = dev->dev_private;
3314         u16 rgvswctl;
3315
3316         assert_spin_locked(&mchdev_lock);
3317
3318         rgvswctl = I915_READ16(MEMSWCTL);
3319         if (rgvswctl & MEMCTL_CMD_STS) {
3320                 DRM_DEBUG("gpu busy, RCS change rejected\n");
3321                 return false; /* still busy with another command */
3322         }
3323
3324         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
3325                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
3326         I915_WRITE16(MEMSWCTL, rgvswctl);
3327         POSTING_READ16(MEMSWCTL);
3328
3329         rgvswctl |= MEMCTL_CMD_STS;
3330         I915_WRITE16(MEMSWCTL, rgvswctl);
3331
3332         return true;
3333 }
3334
3335 static void ironlake_enable_drps(struct drm_device *dev)
3336 {
3337         struct drm_i915_private *dev_priv = dev->dev_private;
3338         u32 rgvmodectl = I915_READ(MEMMODECTL);
3339         u8 fmax, fmin, fstart, vstart;
3340
3341         spin_lock_irq(&mchdev_lock);
3342
3343         /* Enable temp reporting */
3344         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
3345         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
3346
3347         /* 100ms RC evaluation intervals */
3348         I915_WRITE(RCUPEI, 100000);
3349         I915_WRITE(RCDNEI, 100000);
3350
3351         /* Set max/min thresholds to 90ms and 80ms respectively */
3352         I915_WRITE(RCBMAXAVG, 90000);
3353         I915_WRITE(RCBMINAVG, 80000);
3354
3355         I915_WRITE(MEMIHYST, 1);
3356
3357         /* Set up min, max, and cur for interrupt handling */
3358         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
3359         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
3360         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
3361                 MEMMODE_FSTART_SHIFT;
3362
3363         vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
3364                 PXVFREQ_PX_SHIFT;
3365
3366         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
3367         dev_priv->ips.fstart = fstart;
3368
3369         dev_priv->ips.max_delay = fstart;
3370         dev_priv->ips.min_delay = fmin;
3371         dev_priv->ips.cur_delay = fstart;
3372
3373         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
3374                          fmax, fmin, fstart);
3375
3376         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
3377
3378         /*
3379          * Interrupts will be enabled in ironlake_irq_postinstall
3380          */
3381
3382         I915_WRITE(VIDSTART, vstart);
3383         POSTING_READ(VIDSTART);
3384
3385         rgvmodectl |= MEMMODE_SWMODE_EN;
3386         I915_WRITE(MEMMODECTL, rgvmodectl);
3387
3388         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
3389                 DRM_ERROR("stuck trying to change perf mode\n");
3390         mdelay(1);
3391
3392         ironlake_set_drps(dev, fstart);
3393
3394         dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
3395                 I915_READ(0x112e0);
3396         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
3397         dev_priv->ips.last_count2 = I915_READ(0x112f4);
3398         getrawmonotonic(&dev_priv->ips.last_time2);
3399
3400         spin_unlock_irq(&mchdev_lock);
3401 }
3402
3403 static void ironlake_disable_drps(struct drm_device *dev)
3404 {
3405         struct drm_i915_private *dev_priv = dev->dev_private;
3406         u16 rgvswctl;
3407
3408         spin_lock_irq(&mchdev_lock);
3409
3410         rgvswctl = I915_READ16(MEMSWCTL);
3411
3412         /* Ack interrupts, disable EFC interrupt */
3413         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
3414         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
3415         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
3416         I915_WRITE(DEIIR, DE_PCU_EVENT);
3417         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
3418
3419         /* Go back to the starting frequency */
3420         ironlake_set_drps(dev, dev_priv->ips.fstart);
3421         mdelay(1);
3422         rgvswctl |= MEMCTL_CMD_STS;
3423         I915_WRITE(MEMSWCTL, rgvswctl);
3424         mdelay(1);
3425
3426         spin_unlock_irq(&mchdev_lock);
3427 }
3428
3429 /* There's a funny hw issue where the hw returns all 0 when reading from
3430  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
3431  * ourselves, instead of doing a rmw cycle (which might result in us clearing
3432  * all limits and the gpu stuck at whatever frequency it is at atm).
3433  */
3434 static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3435 {
3436         u32 limits;
3437
3438         /* Only set the down limit when we've reached the lowest level to avoid
3439          * getting more interrupts, otherwise leave this clear. This prevents a
3440          * race in the hw when coming out of rc6: There's a tiny window where
3441          * the hw runs at the minimal clock before selecting the desired
3442          * frequency, if the down threshold expires in that window we will not
3443          * receive a down interrupt. */
3444         limits = dev_priv->rps.max_delay << 24;
3445         if (val <= dev_priv->rps.min_delay)
3446                 limits |= dev_priv->rps.min_delay << 16;
3447
3448         return limits;
3449 }
3450
3451 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
3452 {
3453         int new_power;
3454
3455         new_power = dev_priv->rps.power;
3456         switch (dev_priv->rps.power) {
3457         case LOW_POWER:
3458                 if (val > dev_priv->rps.rpe_delay + 1 && val > dev_priv->rps.cur_delay)
3459                         new_power = BETWEEN;
3460                 break;
3461
3462         case BETWEEN:
3463                 if (val <= dev_priv->rps.rpe_delay && val < dev_priv->rps.cur_delay)
3464                         new_power = LOW_POWER;
3465                 else if (val >= dev_priv->rps.rp0_delay && val > dev_priv->rps.cur_delay)
3466                         new_power = HIGH_POWER;
3467                 break;
3468
3469         case HIGH_POWER:
3470                 if (val < (dev_priv->rps.rp1_delay + dev_priv->rps.rp0_delay) >> 1 && val < dev_priv->rps.cur_delay)
3471                         new_power = BETWEEN;
3472                 break;
3473         }
3474         /* Max/min bins are special */
3475         if (val == dev_priv->rps.min_delay)
3476                 new_power = LOW_POWER;
3477         if (val == dev_priv->rps.max_delay)
3478                 new_power = HIGH_POWER;
3479         if (new_power == dev_priv->rps.power)
3480                 return;
3481
3482         /* Note the units here are not exactly 1us, but 1280ns. */
3483         switch (new_power) {
3484         case LOW_POWER:
3485                 /* Upclock if more than 95% busy over 16ms */
3486                 I915_WRITE(GEN6_RP_UP_EI, 12500);
3487                 I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
3488
3489                 /* Downclock if less than 85% busy over 32ms */
3490                 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3491                 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
3492
3493                 I915_WRITE(GEN6_RP_CONTROL,
3494                            GEN6_RP_MEDIA_TURBO |
3495                            GEN6_RP_MEDIA_HW_NORMAL_MODE |
3496                            GEN6_RP_MEDIA_IS_GFX |
3497                            GEN6_RP_ENABLE |
3498                            GEN6_RP_UP_BUSY_AVG |
3499                            GEN6_RP_DOWN_IDLE_AVG);
3500                 break;
3501
3502         case BETWEEN:
3503                 /* Upclock if more than 90% busy over 13ms */
3504                 I915_WRITE(GEN6_RP_UP_EI, 10250);
3505                 I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
3506
3507                 /* Downclock if less than 75% busy over 32ms */
3508                 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3509                 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
3510
3511                 I915_WRITE(GEN6_RP_CONTROL,
3512                            GEN6_RP_MEDIA_TURBO |
3513                            GEN6_RP_MEDIA_HW_NORMAL_MODE |
3514                            GEN6_RP_MEDIA_IS_GFX |
3515                            GEN6_RP_ENABLE |
3516                            GEN6_RP_UP_BUSY_AVG |
3517                            GEN6_RP_DOWN_IDLE_AVG);
3518                 break;
3519
3520         case HIGH_POWER:
3521                 /* Upclock if more than 85% busy over 10ms */
3522                 I915_WRITE(GEN6_RP_UP_EI, 8000);
3523                 I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
3524
3525                 /* Downclock if less than 60% busy over 32ms */
3526                 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
3527                 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
3528
3529                 I915_WRITE(GEN6_RP_CONTROL,
3530                            GEN6_RP_MEDIA_TURBO |
3531                            GEN6_RP_MEDIA_HW_NORMAL_MODE |
3532                            GEN6_RP_MEDIA_IS_GFX |
3533                            GEN6_RP_ENABLE |
3534                            GEN6_RP_UP_BUSY_AVG |
3535                            GEN6_RP_DOWN_IDLE_AVG);
3536                 break;
3537         }
3538
3539         dev_priv->rps.power = new_power;
3540         dev_priv->rps.last_adj = 0;
3541 }
3542
3543 void gen6_set_rps(struct drm_device *dev, u8 val)
3544 {
3545         struct drm_i915_private *dev_priv = dev->dev_private;
3546
3547         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3548         WARN_ON(val > dev_priv->rps.max_delay);
3549         WARN_ON(val < dev_priv->rps.min_delay);
3550
3551         if (val == dev_priv->rps.cur_delay)
3552                 return;
3553
3554         gen6_set_rps_thresholds(dev_priv, val);
3555
3556         if (IS_HASWELL(dev))
3557                 I915_WRITE(GEN6_RPNSWREQ,
3558                            HSW_FREQUENCY(val));
3559         else
3560                 I915_WRITE(GEN6_RPNSWREQ,
3561                            GEN6_FREQUENCY(val) |
3562                            GEN6_OFFSET(0) |
3563                            GEN6_AGGRESSIVE_TURBO);
3564
3565         /* Make sure we continue to get interrupts
3566          * until we hit the minimum or maximum frequencies.
3567          */
3568         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
3569                    gen6_rps_limits(dev_priv, val));
3570
3571         POSTING_READ(GEN6_RPNSWREQ);
3572
3573         dev_priv->rps.cur_delay = val;
3574
3575         trace_intel_gpu_freq_change(val * 50);
3576 }
3577
3578 void gen6_rps_idle(struct drm_i915_private *dev_priv)
3579 {
3580         mutex_lock(&dev_priv->rps.hw_lock);
3581         if (dev_priv->rps.enabled) {
3582                 if (dev_priv->info->is_valleyview)
3583                         valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
3584                 else
3585                         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
3586                 dev_priv->rps.last_adj = 0;
3587         }
3588         mutex_unlock(&dev_priv->rps.hw_lock);
3589 }
3590
3591 void gen6_rps_boost(struct drm_i915_private *dev_priv)
3592 {
3593         mutex_lock(&dev_priv->rps.hw_lock);
3594         if (dev_priv->rps.enabled) {
3595                 if (dev_priv->info->is_valleyview)
3596                         valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
3597                 else
3598                         gen6_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
3599                 dev_priv->rps.last_adj = 0;
3600         }
3601         mutex_unlock(&dev_priv->rps.hw_lock);
3602 }
3603
3604 void valleyview_set_rps(struct drm_device *dev, u8 val)
3605 {
3606         struct drm_i915_private *dev_priv = dev->dev_private;
3607
3608         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3609         WARN_ON(val > dev_priv->rps.max_delay);
3610         WARN_ON(val < dev_priv->rps.min_delay);
3611
3612         DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
3613                          vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay),
3614                          dev_priv->rps.cur_delay,
3615                          vlv_gpu_freq(dev_priv, val), val);
3616
3617         if (val == dev_priv->rps.cur_delay)
3618                 return;
3619
3620         vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
3621
3622         dev_priv->rps.cur_delay = val;
3623
3624         trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
3625 }
3626
3627 static void gen6_disable_rps_interrupts(struct drm_device *dev)
3628 {
3629         struct drm_i915_private *dev_priv = dev->dev_private;
3630
3631         I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
3632         I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS);
3633         /* Complete PM interrupt masking here doesn't race with the rps work
3634          * item again unmasking PM interrupts because that is using a different
3635          * register (PMIMR) to mask PM interrupts. The only risk is in leaving
3636          * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
3637
3638         spin_lock_irq(&dev_priv->irq_lock);
3639         dev_priv->rps.pm_iir = 0;
3640         spin_unlock_irq(&dev_priv->irq_lock);
3641
3642         I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
3643 }
3644
3645 static void gen6_disable_rps(struct drm_device *dev)
3646 {
3647         struct drm_i915_private *dev_priv = dev->dev_private;
3648
3649         I915_WRITE(GEN6_RC_CONTROL, 0);
3650         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
3651
3652         gen6_disable_rps_interrupts(dev);
3653 }
3654
3655 static void valleyview_disable_rps(struct drm_device *dev)
3656 {
3657         struct drm_i915_private *dev_priv = dev->dev_private;
3658
3659         I915_WRITE(GEN6_RC_CONTROL, 0);
3660
3661         gen6_disable_rps_interrupts(dev);
3662
3663         if (dev_priv->vlv_pctx) {
3664                 drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
3665                 dev_priv->vlv_pctx = NULL;
3666         }
3667 }
3668
3669 static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
3670 {
3671         if (IS_GEN6(dev))
3672                 DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
3673
3674         if (IS_HASWELL(dev))
3675                 DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
3676
3677         DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
3678                         (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
3679                         (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
3680                         (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
3681 }
3682
3683 int intel_enable_rc6(const struct drm_device *dev)
3684 {
3685         /* No RC6 before Ironlake */
3686         if (INTEL_INFO(dev)->gen < 5)
3687                 return 0;
3688
3689         /* Respect the kernel parameter if it is set */
3690         if (i915_enable_rc6 >= 0)
3691                 return i915_enable_rc6;
3692
3693         /* Disable RC6 on Ironlake */
3694         if (INTEL_INFO(dev)->gen == 5)
3695                 return 0;
3696
3697         if (IS_HASWELL(dev))
3698                 return INTEL_RC6_ENABLE;
3699
3700         /* snb/ivb have more than one rc6 state. */
3701         if (INTEL_INFO(dev)->gen == 6)
3702                 return INTEL_RC6_ENABLE;
3703
3704         return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
3705 }
3706
3707 static void gen6_enable_rps_interrupts(struct drm_device *dev)
3708 {
3709         struct drm_i915_private *dev_priv = dev->dev_private;
3710         u32 enabled_intrs;
3711
3712         spin_lock_irq(&dev_priv->irq_lock);
3713         WARN_ON(dev_priv->rps.pm_iir);
3714         snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
3715         I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
3716         spin_unlock_irq(&dev_priv->irq_lock);
3717
3718         /* only unmask PM interrupts we need. Mask all others. */
3719         enabled_intrs = GEN6_PM_RPS_EVENTS;
3720
3721         /* IVB and SNB hard hangs on looping batchbuffer
3722          * if GEN6_PM_UP_EI_EXPIRED is masked.
3723          */
3724         if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
3725                 enabled_intrs |= GEN6_PM_RP_UP_EI_EXPIRED;
3726
3727         I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs);
3728 }
3729
3730 static void gen8_enable_rps(struct drm_device *dev)
3731 {
3732         struct drm_i915_private *dev_priv = dev->dev_private;
3733         struct intel_ring_buffer *ring;
3734         uint32_t rc6_mask = 0, rp_state_cap;
3735         int unused;
3736
3737         /* 1a: Software RC state - RC0 */
3738         I915_WRITE(GEN6_RC_STATE, 0);
3739
3740         /* 1c & 1d: Get forcewake during program sequence. Although the driver
3741          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
3742         gen6_gt_force_wake_get(dev_priv);
3743
3744         /* 2a: Disable RC states. */
3745         I915_WRITE(GEN6_RC_CONTROL, 0);
3746
3747         rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3748
3749         /* 2b: Program RC6 thresholds.*/
3750         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
3751         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
3752         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
3753         for_each_ring(ring, dev_priv, unused)
3754                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3755         I915_WRITE(GEN6_RC_SLEEP, 0);
3756         I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
3757
3758         /* 3: Enable RC6 */
3759         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
3760                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
3761         DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
3762         I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
3763                         GEN6_RC_CTL_EI_MODE(1) |
3764                         rc6_mask);
3765
3766         /* 4 Program defaults and thresholds for RPS*/
3767         I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */
3768         I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(12)); /* Request 600 MHz */
3769         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
3770         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
3771
3772         /* Docs recommend 900MHz, and 300 MHz respectively */
3773         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
3774                    dev_priv->rps.max_delay << 24 |
3775                    dev_priv->rps.min_delay << 16);
3776
3777         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
3778         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
3779         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
3780         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
3781
3782         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
3783
3784         /* 5: Enable RPS */
3785         I915_WRITE(GEN6_RP_CONTROL,
3786                    GEN6_RP_MEDIA_TURBO |
3787                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
3788                    GEN6_RP_MEDIA_IS_GFX |
3789                    GEN6_RP_ENABLE |
3790                    GEN6_RP_UP_BUSY_AVG |
3791                    GEN6_RP_DOWN_IDLE_AVG);
3792
3793         /* 6: Ring frequency + overclocking (our driver does this later */
3794
3795         gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
3796
3797         gen6_enable_rps_interrupts(dev);
3798
3799         gen6_gt_force_wake_put(dev_priv);
3800 }
3801
3802 static void gen6_enable_rps(struct drm_device *dev)
3803 {
3804         struct drm_i915_private *dev_priv = dev->dev_private;
3805         struct intel_ring_buffer *ring;
3806         u32 rp_state_cap;
3807         u32 gt_perf_status;
3808         u32 rc6vids, pcu_mbox, rc6_mask = 0;
3809         u32 gtfifodbg;
3810         int rc6_mode;
3811         int i, ret;
3812
3813         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3814
3815         /* Here begins a magic sequence of register writes to enable
3816          * auto-downclocking.
3817          *
3818          * Perhaps there might be some value in exposing these to
3819          * userspace...
3820          */
3821         I915_WRITE(GEN6_RC_STATE, 0);
3822
3823         /* Clear the DBG now so we don't confuse earlier errors */
3824         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
3825                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
3826                 I915_WRITE(GTFIFODBG, gtfifodbg);
3827         }
3828
3829         gen6_gt_force_wake_get(dev_priv);
3830
3831         rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3832         gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
3833
3834         /* In units of 50MHz */
3835         dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
3836         dev_priv->rps.min_delay = (rp_state_cap >> 16) & 0xff;
3837         dev_priv->rps.rp1_delay = (rp_state_cap >>  8) & 0xff;
3838         dev_priv->rps.rp0_delay = (rp_state_cap >>  0) & 0xff;
3839         dev_priv->rps.rpe_delay = dev_priv->rps.rp1_delay;
3840         dev_priv->rps.cur_delay = 0;
3841
3842         /* disable the counters and set deterministic thresholds */
3843         I915_WRITE(GEN6_RC_CONTROL, 0);
3844
3845         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
3846         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
3847         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
3848         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
3849         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
3850
3851         for_each_ring(ring, dev_priv, i)
3852                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3853
3854         I915_WRITE(GEN6_RC_SLEEP, 0);
3855         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
3856         if (INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev))
3857                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
3858         else
3859                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3860         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3861         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
3862
3863         /* Check if we are enabling RC6 */
3864         rc6_mode = intel_enable_rc6(dev_priv->dev);
3865         if (rc6_mode & INTEL_RC6_ENABLE)
3866                 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
3867
3868         /* We don't use those on Haswell */
3869         if (!IS_HASWELL(dev)) {
3870                 if (rc6_mode & INTEL_RC6p_ENABLE)
3871                         rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
3872
3873                 if (rc6_mode & INTEL_RC6pp_ENABLE)
3874                         rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
3875         }
3876
3877         intel_print_rc6_info(dev, rc6_mask);
3878
3879         I915_WRITE(GEN6_RC_CONTROL,
3880                    rc6_mask |
3881                    GEN6_RC_CTL_EI_MODE(1) |
3882                    GEN6_RC_CTL_HW_ENABLE);
3883
3884         /* Power down if completely idle for over 50ms */
3885         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3886         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
3887
3888         ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
3889         if (!ret) {
3890                 pcu_mbox = 0;
3891                 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3892                 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
3893                         DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
3894                                          (dev_priv->rps.max_delay & 0xff) * 50,
3895                                          (pcu_mbox & 0xff) * 50);
3896                         dev_priv->rps.hw_max = pcu_mbox & 0xff;
3897                 }
3898         } else {
3899                 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
3900         }
3901
3902         dev_priv->rps.power = HIGH_POWER; /* force a reset */
3903         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
3904
3905         gen6_enable_rps_interrupts(dev);
3906
3907         rc6vids = 0;
3908         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
3909         if (IS_GEN6(dev) && ret) {
3910                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
3911         } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
3912                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
3913                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
3914                 rc6vids &= 0xffff00;
3915                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
3916                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
3917                 if (ret)
3918                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
3919         }
3920
3921         gen6_gt_force_wake_put(dev_priv);
3922 }
3923
3924 void gen6_update_ring_freq(struct drm_device *dev)
3925 {
3926         struct drm_i915_private *dev_priv = dev->dev_private;
3927         int min_freq = 15;
3928         unsigned int gpu_freq;
3929         unsigned int max_ia_freq, min_ring_freq;
3930         int scaling_factor = 180;
3931         struct cpufreq_policy *policy;
3932
3933         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3934
3935         policy = cpufreq_cpu_get(0);
3936         if (policy) {
3937                 max_ia_freq = policy->cpuinfo.max_freq;
3938                 cpufreq_cpu_put(policy);
3939         } else {
3940                 /*
3941                  * Default to measured freq if none found, PCU will ensure we
3942                  * don't go over
3943                  */
3944                 max_ia_freq = tsc_khz;
3945         }
3946
3947         /* Convert from kHz to MHz */
3948         max_ia_freq /= 1000;
3949
3950         min_ring_freq = I915_READ(DCLK) & 0xf;
3951         /* convert DDR frequency from units of 266.6MHz to bandwidth */
3952         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3953
3954         /*
3955          * For each potential GPU frequency, load a ring frequency we'd like
3956          * to use for memory access.  We do this by specifying the IA frequency
3957          * the PCU should use as a reference to determine the ring frequency.
3958          */
3959         for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
3960              gpu_freq--) {
3961                 int diff = dev_priv->rps.max_delay - gpu_freq;
3962                 unsigned int ia_freq = 0, ring_freq = 0;
3963
3964                 if (INTEL_INFO(dev)->gen >= 8) {
3965                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
3966                         ring_freq = max(min_ring_freq, gpu_freq);
3967                 } else if (IS_HASWELL(dev)) {
3968                         ring_freq = mult_frac(gpu_freq, 5, 4);
3969                         ring_freq = max(min_ring_freq, ring_freq);
3970                         /* leave ia_freq as the default, chosen by cpufreq */
3971                 } else {
3972                         /* On older processors, there is no separate ring
3973                          * clock domain, so in order to boost the bandwidth
3974                          * of the ring, we need to upclock the CPU (ia_freq).
3975                          *
3976                          * For GPU frequencies less than 750MHz,
3977                          * just use the lowest ring freq.
3978                          */
3979                         if (gpu_freq < min_freq)
3980                                 ia_freq = 800;
3981                         else
3982                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
3983                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3984                 }
3985
3986                 sandybridge_pcode_write(dev_priv,
3987                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3988                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
3989                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
3990                                         gpu_freq);
3991         }
3992 }
3993
3994 int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
3995 {
3996         u32 val, rp0;
3997
3998         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
3999
4000         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
4001         /* Clamp to max */
4002         rp0 = min_t(u32, rp0, 0xea);
4003
4004         return rp0;
4005 }
4006
4007 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
4008 {
4009         u32 val, rpe;
4010
4011         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
4012         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
4013         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
4014         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
4015
4016         return rpe;
4017 }
4018
4019 int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4020 {
4021         return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
4022 }
4023
4024 static void valleyview_setup_pctx(struct drm_device *dev)
4025 {
4026         struct drm_i915_private *dev_priv = dev->dev_private;
4027         struct drm_i915_gem_object *pctx;
4028         unsigned long pctx_paddr;
4029         u32 pcbr;
4030         int pctx_size = 24*1024;
4031
4032         pcbr = I915_READ(VLV_PCBR);
4033         if (pcbr) {
4034                 /* BIOS set it up already, grab the pre-alloc'd space */
4035                 int pcbr_offset;
4036
4037                 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
4038                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
4039                                                                       pcbr_offset,
4040                                                                       I915_GTT_OFFSET_NONE,
4041                                                                       pctx_size);
4042                 goto out;
4043         }
4044
4045         /*
4046          * From the Gunit register HAS:
4047          * The Gfx driver is expected to program this register and ensure
4048          * proper allocation within Gfx stolen memory.  For example, this
4049          * register should be programmed such than the PCBR range does not
4050          * overlap with other ranges, such as the frame buffer, protected
4051          * memory, or any other relevant ranges.
4052          */
4053         pctx = i915_gem_object_create_stolen(dev, pctx_size);
4054         if (!pctx) {
4055                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
4056                 return;
4057         }
4058
4059         pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
4060         I915_WRITE(VLV_PCBR, pctx_paddr);
4061
4062 out:
4063         dev_priv->vlv_pctx = pctx;
4064 }
4065
4066 static void valleyview_enable_rps(struct drm_device *dev)
4067 {
4068         struct drm_i915_private *dev_priv = dev->dev_private;
4069         struct intel_ring_buffer *ring;
4070         u32 gtfifodbg, val, rc6_mode = 0;
4071         int i;
4072
4073         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4074
4075         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4076                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4077                                  gtfifodbg);
4078                 I915_WRITE(GTFIFODBG, gtfifodbg);
4079         }
4080
4081         valleyview_setup_pctx(dev);
4082
4083         gen6_gt_force_wake_get(dev_priv);
4084
4085         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4086         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4087         I915_WRITE(GEN6_RP_UP_EI, 66000);
4088         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4089
4090         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4091
4092         I915_WRITE(GEN6_RP_CONTROL,
4093                    GEN6_RP_MEDIA_TURBO |
4094                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
4095                    GEN6_RP_MEDIA_IS_GFX |
4096                    GEN6_RP_ENABLE |
4097                    GEN6_RP_UP_BUSY_AVG |
4098                    GEN6_RP_DOWN_IDLE_CONT);
4099
4100         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
4101         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4102         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4103
4104         for_each_ring(ring, dev_priv, i)
4105                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4106
4107         I915_WRITE(GEN6_RC6_THRESHOLD, 0xc350);
4108
4109         /* allows RC6 residency counter to work */
4110         I915_WRITE(VLV_COUNTER_CONTROL,
4111                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
4112                                       VLV_MEDIA_RC6_COUNT_EN |
4113                                       VLV_RENDER_RC6_COUNT_EN));
4114         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4115                 rc6_mode = GEN7_RC_CTL_TO_MODE;
4116
4117         intel_print_rc6_info(dev, rc6_mode);
4118
4119         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
4120
4121         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4122
4123         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
4124         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
4125
4126         dev_priv->rps.cur_delay = (val >> 8) & 0xff;
4127         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
4128                          vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay),
4129                          dev_priv->rps.cur_delay);
4130
4131         dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv);
4132         dev_priv->rps.hw_max = dev_priv->rps.max_delay;
4133         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
4134                          vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay),
4135                          dev_priv->rps.max_delay);
4136
4137         dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv);
4138         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
4139                          vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay),
4140                          dev_priv->rps.rpe_delay);
4141
4142         dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv);
4143         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
4144                          vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay),
4145                          dev_priv->rps.min_delay);
4146
4147         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
4148                          vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay),
4149                          dev_priv->rps.rpe_delay);
4150
4151         valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
4152
4153         gen6_enable_rps_interrupts(dev);
4154
4155         gen6_gt_force_wake_put(dev_priv);
4156 }
4157
4158 void ironlake_teardown_rc6(struct drm_device *dev)
4159 {
4160         struct drm_i915_private *dev_priv = dev->dev_private;
4161
4162         if (dev_priv->ips.renderctx) {
4163                 i915_gem_object_unpin(dev_priv->ips.renderctx);
4164                 drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
4165                 dev_priv->ips.renderctx = NULL;
4166         }
4167
4168         if (dev_priv->ips.pwrctx) {
4169                 i915_gem_object_unpin(dev_priv->ips.pwrctx);
4170                 drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
4171                 dev_priv->ips.pwrctx = NULL;
4172         }
4173 }
4174
4175 static void ironlake_disable_rc6(struct drm_device *dev)
4176 {
4177         struct drm_i915_private *dev_priv = dev->dev_private;
4178
4179         if (I915_READ(PWRCTXA)) {
4180                 /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
4181                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
4182                 wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
4183                          50);
4184
4185                 I915_WRITE(PWRCTXA, 0);
4186                 POSTING_READ(PWRCTXA);
4187
4188                 I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
4189                 POSTING_READ(RSTDBYCTL);
4190         }
4191 }
4192
4193 static int ironlake_setup_rc6(struct drm_device *dev)
4194 {
4195         struct drm_i915_private *dev_priv = dev->dev_private;
4196
4197         if (dev_priv->ips.renderctx == NULL)
4198                 dev_priv->ips.renderctx = intel_alloc_context_page(dev);
4199         if (!dev_priv->ips.renderctx)
4200                 return -ENOMEM;
4201
4202         if (dev_priv->ips.pwrctx == NULL)
4203                 dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
4204         if (!dev_priv->ips.pwrctx) {
4205                 ironlake_teardown_rc6(dev);
4206                 return -ENOMEM;
4207         }
4208
4209         return 0;
4210 }
4211
4212 static void ironlake_enable_rc6(struct drm_device *dev)
4213 {
4214         struct drm_i915_private *dev_priv = dev->dev_private;
4215         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
4216         bool was_interruptible;
4217         int ret;
4218
4219         /* rc6 disabled by default due to repeated reports of hanging during
4220          * boot and resume.
4221          */
4222         if (!intel_enable_rc6(dev))
4223                 return;
4224
4225         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4226
4227         ret = ironlake_setup_rc6(dev);
4228         if (ret)
4229                 return;
4230
4231         was_interruptible = dev_priv->mm.interruptible;
4232         dev_priv->mm.interruptible = false;
4233
4234         /*
4235          * GPU can automatically power down the render unit if given a page
4236          * to save state.
4237          */
4238         ret = intel_ring_begin(ring, 6);
4239         if (ret) {
4240                 ironlake_teardown_rc6(dev);
4241                 dev_priv->mm.interruptible = was_interruptible;
4242                 return;
4243         }
4244
4245         intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
4246         intel_ring_emit(ring, MI_SET_CONTEXT);
4247         intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) |
4248                         MI_MM_SPACE_GTT |
4249                         MI_SAVE_EXT_STATE_EN |
4250                         MI_RESTORE_EXT_STATE_EN |
4251                         MI_RESTORE_INHIBIT);
4252         intel_ring_emit(ring, MI_SUSPEND_FLUSH);
4253         intel_ring_emit(ring, MI_NOOP);
4254         intel_ring_emit(ring, MI_FLUSH);
4255         intel_ring_advance(ring);
4256
4257         /*
4258          * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
4259          * does an implicit flush, combined with MI_FLUSH above, it should be
4260          * safe to assume that renderctx is valid
4261          */
4262         ret = intel_ring_idle(ring);
4263         dev_priv->mm.interruptible = was_interruptible;
4264         if (ret) {
4265                 DRM_ERROR("failed to enable ironlake power savings\n");
4266                 ironlake_teardown_rc6(dev);
4267                 return;
4268         }
4269
4270         I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) | PWRCTX_EN);
4271         I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
4272
4273         intel_print_rc6_info(dev, INTEL_RC6_ENABLE);
4274 }
4275
4276 static unsigned long intel_pxfreq(u32 vidfreq)
4277 {
4278         unsigned long freq;
4279         int div = (vidfreq & 0x3f0000) >> 16;
4280         int post = (vidfreq & 0x3000) >> 12;
4281         int pre = (vidfreq & 0x7);
4282
4283         if (!pre)
4284                 return 0;
4285
4286         freq = ((div * 133333) / ((1<<post) * pre));
4287
4288         return freq;
4289 }
4290
4291 static const struct cparams {
4292         u16 i;
4293         u16 t;
4294         u16 m;
4295         u16 c;
4296 } cparams[] = {
4297         { 1, 1333, 301, 28664 },
4298         { 1, 1066, 294, 24460 },
4299         { 1, 800, 294, 25192 },
4300         { 0, 1333, 276, 27605 },
4301         { 0, 1066, 276, 27605 },
4302         { 0, 800, 231, 23784 },
4303 };
4304
4305 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
4306 {
4307         u64 total_count, diff, ret;
4308         u32 count1, count2, count3, m = 0, c = 0;
4309         unsigned long now = jiffies_to_msecs(jiffies), diff1;
4310         int i;
4311
4312         assert_spin_locked(&mchdev_lock);
4313
4314         diff1 = now - dev_priv->ips.last_time1;
4315
4316         /* Prevent division-by-zero if we are asking too fast.
4317          * Also, we don't get interesting results if we are polling
4318          * faster than once in 10ms, so just return the saved value
4319          * in such cases.
4320          */
4321         if (diff1 <= 10)
4322                 return dev_priv->ips.chipset_power;
4323
4324         count1 = I915_READ(DMIEC);
4325         count2 = I915_READ(DDREC);
4326         count3 = I915_READ(CSIEC);
4327
4328         total_count = count1 + count2 + count3;
4329
4330         /* FIXME: handle per-counter overflow */
4331         if (total_count < dev_priv->ips.last_count1) {
4332                 diff = ~0UL - dev_priv->ips.last_count1;
4333                 diff += total_count;
4334         } else {
4335                 diff = total_count - dev_priv->ips.last_count1;
4336         }
4337
4338         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
4339                 if (cparams[i].i == dev_priv->ips.c_m &&
4340                     cparams[i].t == dev_priv->ips.r_t) {
4341                         m = cparams[i].m;
4342                         c = cparams[i].c;
4343                         break;
4344                 }
4345         }
4346
4347         diff = div_u64(diff, diff1);
4348         ret = ((m * diff) + c);
4349         ret = div_u64(ret, 10);
4350
4351         dev_priv->ips.last_count1 = total_count;
4352         dev_priv->ips.last_time1 = now;
4353
4354         dev_priv->ips.chipset_power = ret;
4355
4356         return ret;
4357 }
4358
4359 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
4360 {
4361         unsigned long val;
4362
4363         if (dev_priv->info->gen != 5)
4364                 return 0;
4365
4366         spin_lock_irq(&mchdev_lock);
4367
4368         val = __i915_chipset_val(dev_priv);
4369
4370         spin_unlock_irq(&mchdev_lock);
4371
4372         return val;
4373 }
4374
4375 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
4376 {
4377         unsigned long m, x, b;
4378         u32 tsfs;
4379
4380         tsfs = I915_READ(TSFS);
4381
4382         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
4383         x = I915_READ8(TR1);
4384
4385         b = tsfs & TSFS_INTR_MASK;
4386
4387         return ((m * x) / 127) - b;
4388 }
4389
4390 static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
4391 {
4392         static const struct v_table {
4393                 u16 vd; /* in .1 mil */
4394                 u16 vm; /* in .1 mil */
4395         } v_table[] = {
4396                 { 0, 0, },
4397                 { 375, 0, },
4398                 { 500, 0, },
4399                 { 625, 0, },
4400                 { 750, 0, },
4401                 { 875, 0, },
4402                 { 1000, 0, },
4403                 { 1125, 0, },
4404                 { 4125, 3000, },
4405                 { 4125, 3000, },
4406                 { 4125, 3000, },
4407                 { 4125, 3000, },
4408                 { 4125, 3000, },
4409                 { 4125, 3000, },
4410                 { 4125, 3000, },
4411                 { 4125, 3000, },
4412                 { 4125, 3000, },
4413                 { 4125, 3000, },
4414                 { 4125, 3000, },
4415                 { 4125, 3000, },
4416                 { 4125, 3000, },
4417                 { 4125, 3000, },
4418                 { 4125, 3000, },
4419                 { 4125, 3000, },
4420                 { 4125, 3000, },
4421                 { 4125, 3000, },
4422                 { 4125, 3000, },
4423                 { 4125, 3000, },
4424                 { 4125, 3000, },
4425                 { 4125, 3000, },
4426                 { 4125, 3000, },
4427                 { 4125, 3000, },
4428                 { 4250, 3125, },
4429                 { 4375, 3250, },
4430                 { 4500, 3375, },
4431                 { 4625, 3500, },
4432                 { 4750, 3625, },
4433                 { 4875, 3750, },
4434                 { 5000, 3875, },
4435                 { 5125, 4000, },
4436                 { 5250, 4125, },
4437                 { 5375, 4250, },
4438                 { 5500, 4375, },
4439                 { 5625, 4500, },
4440                 { 5750, 4625, },
4441                 { 5875, 4750, },
4442                 { 6000, 4875, },
4443                 { 6125, 5000, },
4444                 { 6250, 5125, },
4445                 { 6375, 5250, },
4446                 { 6500, 5375, },
4447                 { 6625, 5500, },
4448                 { 6750, 5625, },
4449                 { 6875, 5750, },
4450                 { 7000, 5875, },
4451                 { 7125, 6000, },
4452                 { 7250, 6125, },
4453                 { 7375, 6250, },
4454                 { 7500, 6375, },
4455                 { 7625, 6500, },
4456                 { 7750, 6625, },
4457                 { 7875, 6750, },
4458                 { 8000, 6875, },
4459                 { 8125, 7000, },
4460                 { 8250, 7125, },
4461                 { 8375, 7250, },
4462                 { 8500, 7375, },
4463                 { 8625, 7500, },
4464                 { 8750, 7625, },
4465                 { 8875, 7750, },
4466                 { 9000, 7875, },
4467                 { 9125, 8000, },
4468                 { 9250, 8125, },
4469                 { 9375, 8250, },
4470                 { 9500, 8375, },
4471                 { 9625, 8500, },
4472                 { 9750, 8625, },
4473                 { 9875, 8750, },
4474                 { 10000, 8875, },
4475                 { 10125, 9000, },
4476                 { 10250, 9125, },
4477                 { 10375, 9250, },
4478                 { 10500, 9375, },
4479                 { 10625, 9500, },
4480                 { 10750, 9625, },
4481                 { 10875, 9750, },
4482                 { 11000, 9875, },
4483                 { 11125, 10000, },
4484                 { 11250, 10125, },
4485                 { 11375, 10250, },
4486                 { 11500, 10375, },
4487                 { 11625, 10500, },
4488                 { 11750, 10625, },
4489                 { 11875, 10750, },
4490                 { 12000, 10875, },
4491                 { 12125, 11000, },
4492                 { 12250, 11125, },
4493                 { 12375, 11250, },
4494                 { 12500, 11375, },
4495                 { 12625, 11500, },
4496                 { 12750, 11625, },
4497                 { 12875, 11750, },
4498                 { 13000, 11875, },
4499                 { 13125, 12000, },
4500                 { 13250, 12125, },
4501                 { 13375, 12250, },
4502                 { 13500, 12375, },
4503                 { 13625, 12500, },
4504                 { 13750, 12625, },
4505                 { 13875, 12750, },
4506                 { 14000, 12875, },
4507                 { 14125, 13000, },
4508                 { 14250, 13125, },
4509                 { 14375, 13250, },
4510                 { 14500, 13375, },
4511                 { 14625, 13500, },
4512                 { 14750, 13625, },
4513                 { 14875, 13750, },
4514                 { 15000, 13875, },
4515                 { 15125, 14000, },
4516                 { 15250, 14125, },
4517                 { 15375, 14250, },
4518                 { 15500, 14375, },
4519                 { 15625, 14500, },
4520                 { 15750, 14625, },
4521                 { 15875, 14750, },
4522                 { 16000, 14875, },
4523                 { 16125, 15000, },
4524         };
4525         if (dev_priv->info->is_mobile)
4526                 return v_table[pxvid].vm;
4527         else
4528                 return v_table[pxvid].vd;
4529 }
4530
4531 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
4532 {
4533         struct timespec now, diff1;
4534         u64 diff;
4535         unsigned long diffms;
4536         u32 count;
4537
4538         assert_spin_locked(&mchdev_lock);
4539
4540         getrawmonotonic(&now);
4541         diff1 = timespec_sub(now, dev_priv->ips.last_time2);
4542
4543         /* Don't divide by 0 */
4544         diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
4545         if (!diffms)
4546                 return;
4547
4548         count = I915_READ(GFXEC);
4549
4550         if (count < dev_priv->ips.last_count2) {
4551                 diff = ~0UL - dev_priv->ips.last_count2;
4552                 diff += count;
4553         } else {
4554                 diff = count - dev_priv->ips.last_count2;
4555         }
4556
4557         dev_priv->ips.last_count2 = count;
4558         dev_priv->ips.last_time2 = now;
4559
4560         /* More magic constants... */
4561         diff = diff * 1181;
4562         diff = div_u64(diff, diffms * 10);
4563         dev_priv->ips.gfx_power = diff;
4564 }
4565
4566 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
4567 {
4568         if (dev_priv->info->gen != 5)
4569                 return;
4570
4571         spin_lock_irq(&mchdev_lock);
4572
4573         __i915_update_gfx_val(dev_priv);
4574
4575         spin_unlock_irq(&mchdev_lock);
4576 }
4577
4578 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
4579 {
4580         unsigned long t, corr, state1, corr2, state2;
4581         u32 pxvid, ext_v;
4582
4583         assert_spin_locked(&mchdev_lock);
4584
4585         pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
4586         pxvid = (pxvid >> 24) & 0x7f;
4587         ext_v = pvid_to_extvid(dev_priv, pxvid);
4588
4589         state1 = ext_v;
4590
4591         t = i915_mch_val(dev_priv);
4592
4593         /* Revel in the empirically derived constants */
4594
4595         /* Correction factor in 1/100000 units */
4596         if (t > 80)
4597                 corr = ((t * 2349) + 135940);
4598         else if (t >= 50)
4599                 corr = ((t * 964) + 29317);
4600         else /* < 50 */
4601                 corr = ((t * 301) + 1004);
4602
4603         corr = corr * ((150142 * state1) / 10000 - 78642);
4604         corr /= 100000;
4605         corr2 = (corr * dev_priv->ips.corr);
4606
4607         state2 = (corr2 * state1) / 10000;
4608         state2 /= 100; /* convert to mW */
4609
4610         __i915_update_gfx_val(dev_priv);
4611
4612         return dev_priv->ips.gfx_power + state2;
4613 }
4614
4615 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
4616 {
4617         unsigned long val;
4618
4619         if (dev_priv->info->gen != 5)
4620                 return 0;
4621
4622         spin_lock_irq(&mchdev_lock);
4623
4624         val = __i915_gfx_val(dev_priv);
4625
4626         spin_unlock_irq(&mchdev_lock);
4627
4628         return val;
4629 }
4630
4631 /**
4632  * i915_read_mch_val - return value for IPS use
4633  *
4634  * Calculate and return a value for the IPS driver to use when deciding whether
4635  * we have thermal and power headroom to increase CPU or GPU power budget.
4636  */
4637 unsigned long i915_read_mch_val(void)
4638 {
4639         struct drm_i915_private *dev_priv;
4640         unsigned long chipset_val, graphics_val, ret = 0;
4641
4642         spin_lock_irq(&mchdev_lock);
4643         if (!i915_mch_dev)
4644                 goto out_unlock;
4645         dev_priv = i915_mch_dev;
4646
4647         chipset_val = __i915_chipset_val(dev_priv);
4648         graphics_val = __i915_gfx_val(dev_priv);
4649
4650         ret = chipset_val + graphics_val;
4651
4652 out_unlock:
4653         spin_unlock_irq(&mchdev_lock);
4654
4655         return ret;
4656 }
4657 EXPORT_SYMBOL_GPL(i915_read_mch_val);
4658
4659 /**
4660  * i915_gpu_raise - raise GPU frequency limit
4661  *
4662  * Raise the limit; IPS indicates we have thermal headroom.
4663  */
4664 bool i915_gpu_raise(void)
4665 {
4666         struct drm_i915_private *dev_priv;
4667         bool ret = true;
4668
4669         spin_lock_irq(&mchdev_lock);
4670         if (!i915_mch_dev) {
4671                 ret = false;
4672                 goto out_unlock;
4673         }
4674         dev_priv = i915_mch_dev;
4675
4676         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
4677                 dev_priv->ips.max_delay--;
4678
4679 out_unlock:
4680         spin_unlock_irq(&mchdev_lock);
4681
4682         return ret;
4683 }
4684 EXPORT_SYMBOL_GPL(i915_gpu_raise);
4685
4686 /**
4687  * i915_gpu_lower - lower GPU frequency limit
4688  *
4689  * IPS indicates we're close to a thermal limit, so throttle back the GPU
4690  * frequency maximum.
4691  */
4692 bool i915_gpu_lower(void)
4693 {
4694         struct drm_i915_private *dev_priv;
4695         bool ret = true;
4696
4697         spin_lock_irq(&mchdev_lock);
4698         if (!i915_mch_dev) {
4699                 ret = false;
4700                 goto out_unlock;
4701         }
4702         dev_priv = i915_mch_dev;
4703
4704         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
4705                 dev_priv->ips.max_delay++;
4706
4707 out_unlock:
4708         spin_unlock_irq(&mchdev_lock);
4709
4710         return ret;
4711 }
4712 EXPORT_SYMBOL_GPL(i915_gpu_lower);
4713
4714 /**
4715  * i915_gpu_busy - indicate GPU business to IPS
4716  *
4717  * Tell the IPS driver whether or not the GPU is busy.
4718  */
4719 bool i915_gpu_busy(void)
4720 {
4721         struct drm_i915_private *dev_priv;
4722         struct intel_ring_buffer *ring;
4723         bool ret = false;
4724         int i;
4725
4726         spin_lock_irq(&mchdev_lock);
4727         if (!i915_mch_dev)
4728                 goto out_unlock;
4729         dev_priv = i915_mch_dev;
4730
4731         for_each_ring(ring, dev_priv, i)
4732                 ret |= !list_empty(&ring->request_list);
4733
4734 out_unlock:
4735         spin_unlock_irq(&mchdev_lock);
4736
4737         return ret;
4738 }
4739 EXPORT_SYMBOL_GPL(i915_gpu_busy);
4740
4741 /**
4742  * i915_gpu_turbo_disable - disable graphics turbo
4743  *
4744  * Disable graphics turbo by resetting the max frequency and setting the
4745  * current frequency to the default.
4746  */
4747 bool i915_gpu_turbo_disable(void)
4748 {
4749         struct drm_i915_private *dev_priv;
4750         bool ret = true;
4751
4752         spin_lock_irq(&mchdev_lock);
4753         if (!i915_mch_dev) {
4754                 ret = false;
4755                 goto out_unlock;
4756         }
4757         dev_priv = i915_mch_dev;
4758
4759         dev_priv->ips.max_delay = dev_priv->ips.fstart;
4760
4761         if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
4762                 ret = false;
4763
4764 out_unlock:
4765         spin_unlock_irq(&mchdev_lock);
4766
4767         return ret;
4768 }
4769 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
4770
4771 /**
4772  * Tells the intel_ips driver that the i915 driver is now loaded, if
4773  * IPS got loaded first.
4774  *
4775  * This awkward dance is so that neither module has to depend on the
4776  * other in order for IPS to do the appropriate communication of
4777  * GPU turbo limits to i915.
4778  */
4779 static void
4780 ips_ping_for_i915_load(void)
4781 {
4782         void (*link)(void);
4783
4784         link = symbol_get(ips_link_to_i915_driver);
4785         if (link) {
4786                 link();
4787                 symbol_put(ips_link_to_i915_driver);
4788         }
4789 }
4790
4791 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
4792 {
4793         /* We only register the i915 ips part with intel-ips once everything is
4794          * set up, to avoid intel-ips sneaking in and reading bogus values. */
4795         spin_lock_irq(&mchdev_lock);
4796         i915_mch_dev = dev_priv;
4797         spin_unlock_irq(&mchdev_lock);
4798
4799         ips_ping_for_i915_load();
4800 }
4801
4802 void intel_gpu_ips_teardown(void)
4803 {
4804         spin_lock_irq(&mchdev_lock);
4805         i915_mch_dev = NULL;
4806         spin_unlock_irq(&mchdev_lock);
4807 }
4808 static void intel_init_emon(struct drm_device *dev)
4809 {
4810         struct drm_i915_private *dev_priv = dev->dev_private;
4811         u32 lcfuse;
4812         u8 pxw[16];
4813         int i;
4814
4815         /* Disable to program */
4816         I915_WRITE(ECR, 0);
4817         POSTING_READ(ECR);
4818
4819         /* Program energy weights for various events */
4820         I915_WRITE(SDEW, 0x15040d00);
4821         I915_WRITE(CSIEW0, 0x007f0000);
4822         I915_WRITE(CSIEW1, 0x1e220004);
4823         I915_WRITE(CSIEW2, 0x04000004);
4824
4825         for (i = 0; i < 5; i++)
4826                 I915_WRITE(PEW + (i * 4), 0);
4827         for (i = 0; i < 3; i++)
4828                 I915_WRITE(DEW + (i * 4), 0);
4829
4830         /* Program P-state weights to account for frequency power adjustment */
4831         for (i = 0; i < 16; i++) {
4832                 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
4833                 unsigned long freq = intel_pxfreq(pxvidfreq);
4834                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
4835                         PXVFREQ_PX_SHIFT;
4836                 unsigned long val;
4837
4838                 val = vid * vid;
4839                 val *= (freq / 1000);
4840                 val *= 255;
4841                 val /= (127*127*900);
4842                 if (val > 0xff)
4843                         DRM_ERROR("bad pxval: %ld\n", val);
4844                 pxw[i] = val;
4845         }
4846         /* Render standby states get 0 weight */
4847         pxw[14] = 0;
4848         pxw[15] = 0;
4849
4850         for (i = 0; i < 4; i++) {
4851                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
4852                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
4853                 I915_WRITE(PXW + (i * 4), val);
4854         }
4855
4856         /* Adjust magic regs to magic values (more experimental results) */
4857         I915_WRITE(OGW0, 0);
4858         I915_WRITE(OGW1, 0);
4859         I915_WRITE(EG0, 0x00007f00);
4860         I915_WRITE(EG1, 0x0000000e);
4861         I915_WRITE(EG2, 0x000e0000);
4862         I915_WRITE(EG3, 0x68000300);
4863         I915_WRITE(EG4, 0x42000000);
4864         I915_WRITE(EG5, 0x00140031);
4865         I915_WRITE(EG6, 0);
4866         I915_WRITE(EG7, 0);
4867
4868         for (i = 0; i < 8; i++)
4869                 I915_WRITE(PXWL + (i * 4), 0);
4870
4871         /* Enable PMON + select events */
4872         I915_WRITE(ECR, 0x80000019);
4873
4874         lcfuse = I915_READ(LCFUSE02);
4875
4876         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
4877 }
4878
4879 void intel_disable_gt_powersave(struct drm_device *dev)
4880 {
4881         struct drm_i915_private *dev_priv = dev->dev_private;
4882
4883         /* Interrupts should be disabled already to avoid re-arming. */
4884         WARN_ON(dev->irq_enabled);
4885
4886         if (IS_IRONLAKE_M(dev)) {
4887                 ironlake_disable_drps(dev);
4888                 ironlake_disable_rc6(dev);
4889         } else if (INTEL_INFO(dev)->gen >= 6) {
4890                 cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
4891                 cancel_work_sync(&dev_priv->rps.work);
4892                 mutex_lock(&dev_priv->rps.hw_lock);
4893                 if (IS_VALLEYVIEW(dev))
4894                         valleyview_disable_rps(dev);
4895                 else
4896                         gen6_disable_rps(dev);
4897                 dev_priv->rps.enabled = false;
4898                 mutex_unlock(&dev_priv->rps.hw_lock);
4899         }
4900 }
4901
4902 static void intel_gen6_powersave_work(struct work_struct *work)
4903 {
4904         struct drm_i915_private *dev_priv =
4905                 container_of(work, struct drm_i915_private,
4906                              rps.delayed_resume_work.work);
4907         struct drm_device *dev = dev_priv->dev;
4908
4909         mutex_lock(&dev_priv->rps.hw_lock);
4910
4911         if (IS_VALLEYVIEW(dev)) {
4912                 valleyview_enable_rps(dev);
4913         } else if (IS_BROADWELL(dev)) {
4914                 gen8_enable_rps(dev);
4915                 gen6_update_ring_freq(dev);
4916         } else {
4917                 gen6_enable_rps(dev);
4918                 gen6_update_ring_freq(dev);
4919         }
4920         dev_priv->rps.enabled = true;
4921         mutex_unlock(&dev_priv->rps.hw_lock);
4922 }
4923
4924 void intel_enable_gt_powersave(struct drm_device *dev)
4925 {
4926         struct drm_i915_private *dev_priv = dev->dev_private;
4927
4928         if (IS_IRONLAKE_M(dev)) {
4929                 ironlake_enable_drps(dev);
4930                 ironlake_enable_rc6(dev);
4931                 intel_init_emon(dev);
4932         } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
4933                 /*
4934                  * PCU communication is slow and this doesn't need to be
4935                  * done at any specific time, so do this out of our fast path
4936                  * to make resume and init faster.
4937                  */
4938                 schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
4939                                       round_jiffies_up_relative(HZ));
4940         }
4941 }
4942
4943 static void ibx_init_clock_gating(struct drm_device *dev)
4944 {
4945         struct drm_i915_private *dev_priv = dev->dev_private;
4946
4947         /*
4948          * On Ibex Peak and Cougar Point, we need to disable clock
4949          * gating for the panel power sequencer or it will fail to
4950          * start up when no ports are active.
4951          */
4952         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4953 }
4954
4955 static void g4x_disable_trickle_feed(struct drm_device *dev)
4956 {
4957         struct drm_i915_private *dev_priv = dev->dev_private;
4958         int pipe;
4959
4960         for_each_pipe(pipe) {
4961                 I915_WRITE(DSPCNTR(pipe),
4962                            I915_READ(DSPCNTR(pipe)) |
4963                            DISPPLANE_TRICKLE_FEED_DISABLE);
4964                 intel_flush_primary_plane(dev_priv, pipe);
4965         }
4966 }
4967
4968 static void ironlake_init_clock_gating(struct drm_device *dev)
4969 {
4970         struct drm_i915_private *dev_priv = dev->dev_private;
4971         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4972
4973         /*
4974          * Required for FBC
4975          * WaFbcDisableDpfcClockGating:ilk
4976          */
4977         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4978                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4979                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4980
4981         I915_WRITE(PCH_3DCGDIS0,
4982                    MARIUNIT_CLOCK_GATE_DISABLE |
4983                    SVSMUNIT_CLOCK_GATE_DISABLE);
4984         I915_WRITE(PCH_3DCGDIS1,
4985                    VFMUNIT_CLOCK_GATE_DISABLE);
4986
4987         /*
4988          * According to the spec the following bits should be set in
4989          * order to enable memory self-refresh
4990          * The bit 22/21 of 0x42004
4991          * The bit 5 of 0x42020
4992          * The bit 15 of 0x45000
4993          */
4994         I915_WRITE(ILK_DISPLAY_CHICKEN2,
4995                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
4996                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4997         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4998         I915_WRITE(DISP_ARB_CTL,
4999                    (I915_READ(DISP_ARB_CTL) |
5000                     DISP_FBC_WM_DIS));
5001         I915_WRITE(WM3_LP_ILK, 0);
5002         I915_WRITE(WM2_LP_ILK, 0);
5003         I915_WRITE(WM1_LP_ILK, 0);
5004
5005         /*
5006          * Based on the document from hardware guys the following bits
5007          * should be set unconditionally in order to enable FBC.
5008          * The bit 22 of 0x42000
5009          * The bit 22 of 0x42004
5010          * The bit 7,8,9 of 0x42020.
5011          */
5012         if (IS_IRONLAKE_M(dev)) {
5013                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
5014                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
5015                            I915_READ(ILK_DISPLAY_CHICKEN1) |
5016                            ILK_FBCQ_DIS);
5017                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5018                            I915_READ(ILK_DISPLAY_CHICKEN2) |
5019                            ILK_DPARB_GATE);
5020         }
5021
5022         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5023
5024         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5025                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5026                    ILK_ELPIN_409_SELECT);
5027         I915_WRITE(_3D_CHICKEN2,
5028                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
5029                    _3D_CHICKEN2_WM_READ_PIPELINED);
5030
5031         /* WaDisableRenderCachePipelinedFlush:ilk */
5032         I915_WRITE(CACHE_MODE_0,
5033                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
5034
5035         g4x_disable_trickle_feed(dev);
5036
5037         ibx_init_clock_gating(dev);
5038 }
5039
5040 static void cpt_init_clock_gating(struct drm_device *dev)
5041 {
5042         struct drm_i915_private *dev_priv = dev->dev_private;
5043         int pipe;
5044         uint32_t val;
5045
5046         /*
5047          * On Ibex Peak and Cougar Point, we need to disable clock
5048          * gating for the panel power sequencer or it will fail to
5049          * start up when no ports are active.
5050          */
5051         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
5052                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
5053                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
5054         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
5055                    DPLS_EDP_PPS_FIX_DIS);
5056         /* The below fixes the weird display corruption, a few pixels shifted
5057          * downward, on (only) LVDS of some HP laptops with IVY.
5058          */
5059         for_each_pipe(pipe) {
5060                 val = I915_READ(TRANS_CHICKEN2(pipe));
5061                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
5062                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
5063                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
5064                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
5065                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
5066                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
5067                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
5068                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
5069         }
5070         /* WADP0ClockGatingDisable */
5071         for_each_pipe(pipe) {
5072                 I915_WRITE(TRANS_CHICKEN1(pipe),
5073                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
5074         }
5075 }
5076
5077 static void gen6_check_mch_setup(struct drm_device *dev)
5078 {
5079         struct drm_i915_private *dev_priv = dev->dev_private;
5080         uint32_t tmp;
5081
5082         tmp = I915_READ(MCH_SSKPD);
5083         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
5084                 DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
5085                 DRM_INFO("This can cause pipe underruns and display issues.\n");
5086                 DRM_INFO("Please upgrade your BIOS to fix this.\n");
5087         }
5088 }
5089
5090 static void gen6_init_clock_gating(struct drm_device *dev)
5091 {
5092         struct drm_i915_private *dev_priv = dev->dev_private;
5093         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
5094
5095         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5096
5097         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5098                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5099                    ILK_ELPIN_409_SELECT);
5100
5101         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
5102         I915_WRITE(_3D_CHICKEN,
5103                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
5104
5105         /* WaSetupGtModeTdRowDispatch:snb */
5106         if (IS_SNB_GT1(dev))
5107                 I915_WRITE(GEN6_GT_MODE,
5108                            _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
5109
5110         I915_WRITE(WM3_LP_ILK, 0);
5111         I915_WRITE(WM2_LP_ILK, 0);
5112         I915_WRITE(WM1_LP_ILK, 0);
5113
5114         I915_WRITE(CACHE_MODE_0,
5115                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
5116
5117         I915_WRITE(GEN6_UCGCTL1,
5118                    I915_READ(GEN6_UCGCTL1) |
5119                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
5120                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
5121
5122         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
5123          * gating disable must be set.  Failure to set it results in
5124          * flickering pixels due to Z write ordering failures after
5125          * some amount of runtime in the Mesa "fire" demo, and Unigine
5126          * Sanctuary and Tropics, and apparently anything else with
5127          * alpha test or pixel discard.
5128          *
5129          * According to the spec, bit 11 (RCCUNIT) must also be set,
5130          * but we didn't debug actual testcases to find it out.
5131          *
5132          * Also apply WaDisableVDSUnitClockGating:snb and
5133          * WaDisableRCPBUnitClockGating:snb.
5134          */
5135         I915_WRITE(GEN6_UCGCTL2,
5136                    GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
5137                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
5138                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
5139
5140         /* Bspec says we need to always set all mask bits. */
5141         I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) |
5142                    _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
5143
5144         /*
5145          * According to the spec the following bits should be
5146          * set in order to enable memory self-refresh and fbc:
5147          * The bit21 and bit22 of 0x42000
5148          * The bit21 and bit22 of 0x42004
5149          * The bit5 and bit7 of 0x42020
5150          * The bit14 of 0x70180
5151          * The bit14 of 0x71180
5152          *
5153          * WaFbcAsynchFlipDisableFbcQueue:snb
5154          */
5155         I915_WRITE(ILK_DISPLAY_CHICKEN1,
5156                    I915_READ(ILK_DISPLAY_CHICKEN1) |
5157                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
5158         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5159                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5160                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
5161         I915_WRITE(ILK_DSPCLK_GATE_D,
5162                    I915_READ(ILK_DSPCLK_GATE_D) |
5163                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
5164                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
5165
5166         g4x_disable_trickle_feed(dev);
5167
5168         /* The default value should be 0x200 according to docs, but the two
5169          * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
5170         I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
5171         I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
5172
5173         cpt_init_clock_gating(dev);
5174
5175         gen6_check_mch_setup(dev);
5176 }
5177
5178 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
5179 {
5180         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
5181
5182         reg &= ~GEN7_FF_SCHED_MASK;
5183         reg |= GEN7_FF_TS_SCHED_HW;
5184         reg |= GEN7_FF_VS_SCHED_HW;
5185         reg |= GEN7_FF_DS_SCHED_HW;
5186
5187         if (IS_HASWELL(dev_priv->dev))
5188                 reg &= ~GEN7_FF_VS_REF_CNT_FFME;
5189
5190         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
5191 }
5192
5193 static void lpt_init_clock_gating(struct drm_device *dev)
5194 {
5195         struct drm_i915_private *dev_priv = dev->dev_private;
5196
5197         /*
5198          * TODO: this bit should only be enabled when really needed, then
5199          * disabled when not needed anymore in order to save power.
5200          */
5201         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
5202                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
5203                            I915_READ(SOUTH_DSPCLK_GATE_D) |
5204                            PCH_LP_PARTITION_LEVEL_DISABLE);
5205
5206         /* WADPOClockGatingDisable:hsw */
5207         I915_WRITE(_TRANSA_CHICKEN1,
5208                    I915_READ(_TRANSA_CHICKEN1) |
5209                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
5210 }
5211
5212 static void lpt_suspend_hw(struct drm_device *dev)
5213 {
5214         struct drm_i915_private *dev_priv = dev->dev_private;
5215
5216         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
5217                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
5218
5219                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
5220                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
5221         }
5222 }
5223
5224 static void gen8_init_clock_gating(struct drm_device *dev)
5225 {
5226         struct drm_i915_private *dev_priv = dev->dev_private;
5227         enum pipe i;
5228
5229         I915_WRITE(WM3_LP_ILK, 0);
5230         I915_WRITE(WM2_LP_ILK, 0);
5231         I915_WRITE(WM1_LP_ILK, 0);
5232
5233         /* FIXME(BDW): Check all the w/a, some might only apply to
5234          * pre-production hw. */
5235
5236         WARN(!i915_preliminary_hw_support,
5237              "GEN8_CENTROID_PIXEL_OPT_DIS not be needed for production\n");
5238         I915_WRITE(HALF_SLICE_CHICKEN3,
5239                    _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
5240         I915_WRITE(HALF_SLICE_CHICKEN3,
5241                    _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
5242         I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
5243
5244         I915_WRITE(_3D_CHICKEN3,
5245                    _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2));
5246
5247         I915_WRITE(COMMON_SLICE_CHICKEN2,
5248                    _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
5249
5250         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
5251                    _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
5252
5253         /* WaSwitchSolVfFArbitrationPriority */
5254         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
5255
5256         /* WaPsrDPAMaskVBlankInSRD */
5257         I915_WRITE(CHICKEN_PAR1_1,
5258                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
5259
5260         /* WaPsrDPRSUnmaskVBlankInSRD */
5261         for_each_pipe(i) {
5262                 I915_WRITE(CHICKEN_PIPESL_1(i),
5263                            I915_READ(CHICKEN_PIPESL_1(i) |
5264                                      DPRS_MASK_VBLANK_SRD));
5265         }
5266 }
5267
5268 static void haswell_init_clock_gating(struct drm_device *dev)
5269 {
5270         struct drm_i915_private *dev_priv = dev->dev_private;
5271
5272         I915_WRITE(WM3_LP_ILK, 0);
5273         I915_WRITE(WM2_LP_ILK, 0);
5274         I915_WRITE(WM1_LP_ILK, 0);
5275
5276         /* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
5277          * This implements the WaDisableRCZUnitClockGating:hsw workaround.
5278          */
5279         I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
5280
5281         /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
5282         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
5283                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
5284
5285         /* WaApplyL3ControlAndL3ChickenMode:hsw */
5286         I915_WRITE(GEN7_L3CNTLREG1,
5287                         GEN7_WA_FOR_GEN7_L3_CONTROL);
5288         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
5289                         GEN7_WA_L3_CHICKEN_MODE);
5290
5291         /* L3 caching of data atomics doesn't work -- disable it. */
5292         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
5293         I915_WRITE(HSW_ROW_CHICKEN3,
5294                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
5295
5296         /* This is required by WaCatErrorRejectionIssue:hsw */
5297         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5298                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5299                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5300
5301         /* WaVSRefCountFullforceMissDisable:hsw */
5302         gen7_setup_fixed_func_scheduler(dev_priv);
5303
5304         /* WaDisable4x2SubspanOptimization:hsw */
5305         I915_WRITE(CACHE_MODE_1,
5306                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
5307
5308         /* WaSwitchSolVfFArbitrationPriority:hsw */
5309         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
5310
5311         /* WaRsPkgCStateDisplayPMReq:hsw */
5312         I915_WRITE(CHICKEN_PAR1_1,
5313                    I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
5314
5315         lpt_init_clock_gating(dev);
5316 }
5317
5318 static void ivybridge_init_clock_gating(struct drm_device *dev)
5319 {
5320         struct drm_i915_private *dev_priv = dev->dev_private;
5321         uint32_t snpcr;
5322
5323         I915_WRITE(WM3_LP_ILK, 0);
5324         I915_WRITE(WM2_LP_ILK, 0);
5325         I915_WRITE(WM1_LP_ILK, 0);
5326
5327         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
5328
5329         /* WaDisableEarlyCull:ivb */
5330         I915_WRITE(_3D_CHICKEN3,
5331                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
5332
5333         /* WaDisableBackToBackFlipFix:ivb */
5334         I915_WRITE(IVB_CHICKEN3,
5335                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
5336                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
5337
5338         /* WaDisablePSDDualDispatchEnable:ivb */
5339         if (IS_IVB_GT1(dev))
5340                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
5341                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
5342         else
5343                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
5344                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
5345
5346         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
5347         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
5348                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
5349
5350         /* WaApplyL3ControlAndL3ChickenMode:ivb */
5351         I915_WRITE(GEN7_L3CNTLREG1,
5352                         GEN7_WA_FOR_GEN7_L3_CONTROL);
5353         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
5354                    GEN7_WA_L3_CHICKEN_MODE);
5355         if (IS_IVB_GT1(dev))
5356                 I915_WRITE(GEN7_ROW_CHICKEN2,
5357                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5358         else
5359                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
5360                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5361
5362
5363         /* WaForceL3Serialization:ivb */
5364         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
5365                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
5366
5367         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
5368          * gating disable must be set.  Failure to set it results in
5369          * flickering pixels due to Z write ordering failures after
5370          * some amount of runtime in the Mesa "fire" demo, and Unigine
5371          * Sanctuary and Tropics, and apparently anything else with
5372          * alpha test or pixel discard.
5373          *
5374          * According to the spec, bit 11 (RCCUNIT) must also be set,
5375          * but we didn't debug actual testcases to find it out.
5376          *
5377          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
5378          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
5379          */
5380         I915_WRITE(GEN6_UCGCTL2,
5381                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE |
5382                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
5383
5384         /* This is required by WaCatErrorRejectionIssue:ivb */
5385         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5386                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5387                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5388
5389         g4x_disable_trickle_feed(dev);
5390
5391         /* WaVSRefCountFullforceMissDisable:ivb */
5392         gen7_setup_fixed_func_scheduler(dev_priv);
5393
5394         /* WaDisable4x2SubspanOptimization:ivb */
5395         I915_WRITE(CACHE_MODE_1,
5396                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
5397
5398         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
5399         snpcr &= ~GEN6_MBC_SNPCR_MASK;
5400         snpcr |= GEN6_MBC_SNPCR_MED;
5401         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
5402
5403         if (!HAS_PCH_NOP(dev))
5404                 cpt_init_clock_gating(dev);
5405
5406         gen6_check_mch_setup(dev);
5407 }
5408
5409 static void valleyview_init_clock_gating(struct drm_device *dev)
5410 {
5411         struct drm_i915_private *dev_priv = dev->dev_private;
5412         u32 val;
5413
5414         mutex_lock(&dev_priv->rps.hw_lock);
5415         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5416         mutex_unlock(&dev_priv->rps.hw_lock);
5417         switch ((val >> 6) & 3) {
5418         case 0:
5419                 dev_priv->mem_freq = 800;
5420                 break;
5421         case 1:
5422                 dev_priv->mem_freq = 1066;
5423                 break;
5424         case 2:
5425                 dev_priv->mem_freq = 1333;
5426                 break;
5427         case 3:
5428                 dev_priv->mem_freq = 1333;
5429                 break;
5430         }
5431         DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
5432
5433         I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
5434
5435         /* WaDisableEarlyCull:vlv */
5436         I915_WRITE(_3D_CHICKEN3,
5437                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
5438
5439         /* WaDisableBackToBackFlipFix:vlv */
5440         I915_WRITE(IVB_CHICKEN3,
5441                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
5442                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
5443
5444         /* WaDisablePSDDualDispatchEnable:vlv */
5445         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
5446                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
5447                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
5448
5449         /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
5450         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
5451                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
5452
5453         /* WaApplyL3ControlAndL3ChickenMode:vlv */
5454         I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS);
5455         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
5456
5457         /* WaForceL3Serialization:vlv */
5458         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
5459                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
5460
5461         /* WaDisableDopClockGating:vlv */
5462         I915_WRITE(GEN7_ROW_CHICKEN2,
5463                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5464
5465         /* This is required by WaCatErrorRejectionIssue:vlv */
5466         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
5467                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
5468                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
5469
5470         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
5471          * gating disable must be set.  Failure to set it results in
5472          * flickering pixels due to Z write ordering failures after
5473          * some amount of runtime in the Mesa "fire" demo, and Unigine
5474          * Sanctuary and Tropics, and apparently anything else with
5475          * alpha test or pixel discard.
5476          *
5477          * According to the spec, bit 11 (RCCUNIT) must also be set,
5478          * but we didn't debug actual testcases to find it out.
5479          *
5480          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
5481          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
5482          *
5483          * Also apply WaDisableVDSUnitClockGating:vlv and
5484          * WaDisableRCPBUnitClockGating:vlv.
5485          */
5486         I915_WRITE(GEN6_UCGCTL2,
5487                    GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
5488                    GEN7_TDLUNIT_CLOCK_GATE_DISABLE |
5489                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE |
5490                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
5491                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
5492
5493         I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
5494
5495         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
5496
5497         I915_WRITE(CACHE_MODE_1,
5498                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
5499
5500         /*
5501          * WaDisableVLVClockGating_VBIIssue:vlv
5502          * Disable clock gating on th GCFG unit to prevent a delay
5503          * in the reporting of vblank events.
5504          */
5505         I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff);
5506
5507         /* Conservative clock gating settings for now */
5508         I915_WRITE(0x9400, 0xffffffff);
5509         I915_WRITE(0x9404, 0xffffffff);
5510         I915_WRITE(0x9408, 0xffffffff);
5511         I915_WRITE(0x940c, 0xffffffff);
5512         I915_WRITE(0x9410, 0xffffffff);
5513         I915_WRITE(0x9414, 0xffffffff);
5514         I915_WRITE(0x9418, 0xffffffff);
5515 }
5516
5517 static void g4x_init_clock_gating(struct drm_device *dev)
5518 {
5519         struct drm_i915_private *dev_priv = dev->dev_private;
5520         uint32_t dspclk_gate;
5521
5522         I915_WRITE(RENCLK_GATE_D1, 0);
5523         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
5524                    GS_UNIT_CLOCK_GATE_DISABLE |
5525                    CL_UNIT_CLOCK_GATE_DISABLE);
5526         I915_WRITE(RAMCLK_GATE_D, 0);
5527         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
5528                 OVRUNIT_CLOCK_GATE_DISABLE |
5529                 OVCUNIT_CLOCK_GATE_DISABLE;
5530         if (IS_GM45(dev))
5531                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
5532         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
5533
5534         /* WaDisableRenderCachePipelinedFlush */
5535         I915_WRITE(CACHE_MODE_0,
5536                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
5537
5538         g4x_disable_trickle_feed(dev);
5539 }
5540
5541 static void crestline_init_clock_gating(struct drm_device *dev)
5542 {
5543         struct drm_i915_private *dev_priv = dev->dev_private;
5544
5545         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
5546         I915_WRITE(RENCLK_GATE_D2, 0);
5547         I915_WRITE(DSPCLK_GATE_D, 0);
5548         I915_WRITE(RAMCLK_GATE_D, 0);
5549         I915_WRITE16(DEUC, 0);
5550         I915_WRITE(MI_ARB_STATE,
5551                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5552 }
5553
5554 static void broadwater_init_clock_gating(struct drm_device *dev)
5555 {
5556         struct drm_i915_private *dev_priv = dev->dev_private;
5557
5558         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
5559                    I965_RCC_CLOCK_GATE_DISABLE |
5560                    I965_RCPB_CLOCK_GATE_DISABLE |
5561                    I965_ISC_CLOCK_GATE_DISABLE |
5562                    I965_FBC_CLOCK_GATE_DISABLE);
5563         I915_WRITE(RENCLK_GATE_D2, 0);
5564         I915_WRITE(MI_ARB_STATE,
5565                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5566 }
5567
5568 static void gen3_init_clock_gating(struct drm_device *dev)
5569 {
5570         struct drm_i915_private *dev_priv = dev->dev_private;
5571         u32 dstate = I915_READ(D_STATE);
5572
5573         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
5574                 DSTATE_DOT_CLOCK_GATING;
5575         I915_WRITE(D_STATE, dstate);
5576
5577         if (IS_PINEVIEW(dev))
5578                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
5579
5580         /* IIR "flip pending" means done if this bit is set */
5581         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5582 }
5583
5584 static void i85x_init_clock_gating(struct drm_device *dev)
5585 {
5586         struct drm_i915_private *dev_priv = dev->dev_private;
5587
5588         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5589 }
5590
5591 static void i830_init_clock_gating(struct drm_device *dev)
5592 {
5593         struct drm_i915_private *dev_priv = dev->dev_private;
5594
5595         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5596 }
5597
5598 void intel_init_clock_gating(struct drm_device *dev)
5599 {
5600         struct drm_i915_private *dev_priv = dev->dev_private;
5601
5602         dev_priv->display.init_clock_gating(dev);
5603 }
5604
5605 void intel_suspend_hw(struct drm_device *dev)
5606 {
5607         if (HAS_PCH_LPT(dev))
5608                 lpt_suspend_hw(dev);
5609 }
5610
5611 static bool is_always_on_power_domain(struct drm_device *dev,
5612                                       enum intel_display_power_domain domain)
5613 {
5614         unsigned long always_on_domains;
5615
5616         BUG_ON(BIT(domain) & ~POWER_DOMAIN_MASK);
5617
5618         if (IS_BROADWELL(dev)) {
5619                 always_on_domains = BDW_ALWAYS_ON_POWER_DOMAINS;
5620         } else if (IS_HASWELL(dev)) {
5621                 always_on_domains = HSW_ALWAYS_ON_POWER_DOMAINS;
5622         } else {
5623                 WARN_ON(1);
5624                 return true;
5625         }
5626
5627         return BIT(domain) & always_on_domains;
5628 }
5629
5630 /**
5631  * We should only use the power well if we explicitly asked the hardware to
5632  * enable it, so check if it's enabled and also check if we've requested it to
5633  * be enabled.
5634  */
5635 bool intel_display_power_enabled(struct drm_device *dev,
5636                                  enum intel_display_power_domain domain)
5637 {
5638         struct drm_i915_private *dev_priv = dev->dev_private;
5639
5640         if (!HAS_POWER_WELL(dev))
5641                 return true;
5642
5643         if (is_always_on_power_domain(dev, domain))
5644                 return true;
5645
5646         return I915_READ(HSW_PWR_WELL_DRIVER) ==
5647                      (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
5648 }
5649
5650 static void __intel_set_power_well(struct drm_device *dev, bool enable)
5651 {
5652         struct drm_i915_private *dev_priv = dev->dev_private;
5653         bool is_enabled, enable_requested;
5654         unsigned long irqflags;
5655         uint32_t tmp;
5656
5657         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
5658         is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
5659         enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
5660
5661         if (enable) {
5662                 if (!enable_requested)
5663                         I915_WRITE(HSW_PWR_WELL_DRIVER,
5664                                    HSW_PWR_WELL_ENABLE_REQUEST);
5665
5666                 if (!is_enabled) {
5667                         DRM_DEBUG_KMS("Enabling power well\n");
5668                         if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
5669                                       HSW_PWR_WELL_STATE_ENABLED), 20))
5670                                 DRM_ERROR("Timeout enabling power well\n");
5671                 }
5672
5673                 if (IS_BROADWELL(dev)) {
5674                         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
5675                         I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_B),
5676                                    dev_priv->de_irq_mask[PIPE_B]);
5677                         I915_WRITE(GEN8_DE_PIPE_IER(PIPE_B),
5678                                    ~dev_priv->de_irq_mask[PIPE_B] |
5679                                    GEN8_PIPE_VBLANK);
5680                         I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_C),
5681                                    dev_priv->de_irq_mask[PIPE_C]);
5682                         I915_WRITE(GEN8_DE_PIPE_IER(PIPE_C),
5683                                    ~dev_priv->de_irq_mask[PIPE_C] |
5684                                    GEN8_PIPE_VBLANK);
5685                         POSTING_READ(GEN8_DE_PIPE_IER(PIPE_C));
5686                         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
5687                 }
5688         } else {
5689                 if (enable_requested) {
5690                         enum pipe p;
5691
5692                         I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
5693                         POSTING_READ(HSW_PWR_WELL_DRIVER);
5694                         DRM_DEBUG_KMS("Requesting to disable the power well\n");
5695
5696                         /*
5697                          * After this, the registers on the pipes that are part
5698                          * of the power well will become zero, so we have to
5699                          * adjust our counters according to that.
5700                          *
5701                          * FIXME: Should we do this in general in
5702                          * drm_vblank_post_modeset?
5703                          */
5704                         spin_lock_irqsave(&dev->vbl_lock, irqflags);
5705                         for_each_pipe(p)
5706                                 if (p != PIPE_A)
5707                                         dev->vblank[p].last = 0;
5708                         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
5709                 }
5710         }
5711 }
5712
5713 static void __intel_power_well_get(struct drm_device *dev,
5714                                    struct i915_power_well *power_well)
5715 {
5716         if (!power_well->count++)
5717                 __intel_set_power_well(dev, true);
5718 }
5719
5720 static void __intel_power_well_put(struct drm_device *dev,
5721                                    struct i915_power_well *power_well)
5722 {
5723         WARN_ON(!power_well->count);
5724         if (!--power_well->count && i915_disable_power_well)
5725                 __intel_set_power_well(dev, false);
5726 }
5727
5728 void intel_display_power_get(struct drm_device *dev,
5729                              enum intel_display_power_domain domain)
5730 {
5731         struct drm_i915_private *dev_priv = dev->dev_private;
5732         struct i915_power_domains *power_domains;
5733
5734         if (!HAS_POWER_WELL(dev))
5735                 return;
5736
5737         if (is_always_on_power_domain(dev, domain))
5738                 return;
5739
5740         power_domains = &dev_priv->power_domains;
5741
5742         mutex_lock(&power_domains->lock);
5743         __intel_power_well_get(dev, &power_domains->power_wells[0]);
5744         mutex_unlock(&power_domains->lock);
5745 }
5746
5747 void intel_display_power_put(struct drm_device *dev,
5748                              enum intel_display_power_domain domain)
5749 {
5750         struct drm_i915_private *dev_priv = dev->dev_private;
5751         struct i915_power_domains *power_domains;
5752
5753         if (!HAS_POWER_WELL(dev))
5754                 return;
5755
5756         if (is_always_on_power_domain(dev, domain))
5757                 return;
5758
5759         power_domains = &dev_priv->power_domains;
5760
5761         mutex_lock(&power_domains->lock);
5762         __intel_power_well_put(dev, &power_domains->power_wells[0]);
5763         mutex_unlock(&power_domains->lock);
5764 }
5765
5766 static struct i915_power_domains *hsw_pwr;
5767
5768 /* Display audio driver power well request */
5769 void i915_request_power_well(void)
5770 {
5771         struct drm_i915_private *dev_priv;
5772
5773         if (WARN_ON(!hsw_pwr))
5774                 return;
5775
5776         dev_priv = container_of(hsw_pwr, struct drm_i915_private,
5777                                 power_domains);
5778         intel_display_power_get(dev_priv->dev, POWER_DOMAIN_AUDIO);
5779 }
5780 EXPORT_SYMBOL_GPL(i915_request_power_well);
5781
5782 /* Display audio driver power well release */
5783 void i915_release_power_well(void)
5784 {
5785         struct drm_i915_private *dev_priv;
5786
5787         if (WARN_ON(!hsw_pwr))
5788                 return;
5789
5790         dev_priv = container_of(hsw_pwr, struct drm_i915_private,
5791                                 power_domains);
5792         intel_display_power_put(dev_priv->dev, POWER_DOMAIN_AUDIO);
5793 }
5794 EXPORT_SYMBOL_GPL(i915_release_power_well);
5795
5796 int intel_power_domains_init(struct drm_device *dev)
5797 {
5798         struct drm_i915_private *dev_priv = dev->dev_private;
5799         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5800         struct i915_power_well *power_well;
5801
5802         mutex_init(&power_domains->lock);
5803         hsw_pwr = power_domains;
5804
5805         power_well = &power_domains->power_wells[0];
5806         power_well->count = 0;
5807
5808         return 0;
5809 }
5810
5811 void intel_power_domains_remove(struct drm_device *dev)
5812 {
5813         hsw_pwr = NULL;
5814 }
5815
5816 static void intel_power_domains_resume(struct drm_device *dev)
5817 {
5818         struct drm_i915_private *dev_priv = dev->dev_private;
5819         struct i915_power_domains *power_domains = &dev_priv->power_domains;
5820         struct i915_power_well *power_well;
5821
5822         if (!HAS_POWER_WELL(dev))
5823                 return;
5824
5825         mutex_lock(&power_domains->lock);
5826
5827         power_well = &power_domains->power_wells[0];
5828         __intel_set_power_well(dev, power_well->count > 0);
5829
5830         mutex_unlock(&power_domains->lock);
5831 }
5832
5833 /*
5834  * Starting with Haswell, we have a "Power Down Well" that can be turned off
5835  * when not needed anymore. We have 4 registers that can request the power well
5836  * to be enabled, and it will only be disabled if none of the registers is
5837  * requesting it to be enabled.
5838  */
5839 void intel_power_domains_init_hw(struct drm_device *dev)
5840 {
5841         struct drm_i915_private *dev_priv = dev->dev_private;
5842
5843         if (!HAS_POWER_WELL(dev))
5844                 return;
5845
5846         /* For now, we need the power well to be always enabled. */
5847         intel_display_set_init_power(dev, true);
5848         intel_power_domains_resume(dev);
5849
5850         /* We're taking over the BIOS, so clear any requests made by it since
5851          * the driver is in charge now. */
5852         if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
5853                 I915_WRITE(HSW_PWR_WELL_BIOS, 0);
5854 }
5855
5856 /* Disables PC8 so we can use the GMBUS and DP AUX interrupts. */
5857 void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
5858 {
5859         hsw_disable_package_c8(dev_priv);
5860 }
5861
5862 void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
5863 {
5864         hsw_enable_package_c8(dev_priv);
5865 }
5866
5867 /* Set up chip specific power management-related functions */
5868 void intel_init_pm(struct drm_device *dev)
5869 {
5870         struct drm_i915_private *dev_priv = dev->dev_private;
5871
5872         if (I915_HAS_FBC(dev)) {
5873                 if (HAS_PCH_SPLIT(dev)) {
5874                         dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
5875                         if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
5876                                 dev_priv->display.enable_fbc =
5877                                         gen7_enable_fbc;
5878                         else
5879                                 dev_priv->display.enable_fbc =
5880                                         ironlake_enable_fbc;
5881                         dev_priv->display.disable_fbc = ironlake_disable_fbc;
5882                 } else if (IS_GM45(dev)) {
5883                         dev_priv->display.fbc_enabled = g4x_fbc_enabled;
5884                         dev_priv->display.enable_fbc = g4x_enable_fbc;
5885                         dev_priv->display.disable_fbc = g4x_disable_fbc;
5886                 } else if (IS_CRESTLINE(dev)) {
5887                         dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
5888                         dev_priv->display.enable_fbc = i8xx_enable_fbc;
5889                         dev_priv->display.disable_fbc = i8xx_disable_fbc;
5890                 }
5891                 /* 855GM needs testing */
5892         }
5893
5894         /* For cxsr */
5895         if (IS_PINEVIEW(dev))
5896                 i915_pineview_get_mem_freq(dev);
5897         else if (IS_GEN5(dev))
5898                 i915_ironlake_get_mem_freq(dev);
5899
5900         /* For FIFO watermark updates */
5901         if (HAS_PCH_SPLIT(dev)) {
5902                 intel_setup_wm_latency(dev);
5903
5904                 if (IS_GEN5(dev)) {
5905                         if (dev_priv->wm.pri_latency[1] &&
5906                             dev_priv->wm.spr_latency[1] &&
5907                             dev_priv->wm.cur_latency[1])
5908                                 dev_priv->display.update_wm = ironlake_update_wm;
5909                         else {
5910                                 DRM_DEBUG_KMS("Failed to get proper latency. "
5911                                               "Disable CxSR\n");
5912                                 dev_priv->display.update_wm = NULL;
5913                         }
5914                         dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
5915                 } else if (IS_GEN6(dev)) {
5916                         if (dev_priv->wm.pri_latency[0] &&
5917                             dev_priv->wm.spr_latency[0] &&
5918                             dev_priv->wm.cur_latency[0]) {
5919                                 dev_priv->display.update_wm = sandybridge_update_wm;
5920                                 dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
5921                         } else {
5922                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
5923                                               "Disable CxSR\n");
5924                                 dev_priv->display.update_wm = NULL;
5925                         }
5926                         dev_priv->display.init_clock_gating = gen6_init_clock_gating;
5927                 } else if (IS_IVYBRIDGE(dev)) {
5928                         if (dev_priv->wm.pri_latency[0] &&
5929                             dev_priv->wm.spr_latency[0] &&
5930                             dev_priv->wm.cur_latency[0]) {
5931                                 dev_priv->display.update_wm = ivybridge_update_wm;
5932                                 dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
5933                         } else {
5934                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
5935                                               "Disable CxSR\n");
5936                                 dev_priv->display.update_wm = NULL;
5937                         }
5938                         dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
5939                 } else if (IS_HASWELL(dev)) {
5940                         if (dev_priv->wm.pri_latency[0] &&
5941                             dev_priv->wm.spr_latency[0] &&
5942                             dev_priv->wm.cur_latency[0]) {
5943                                 dev_priv->display.update_wm = haswell_update_wm;
5944                                 dev_priv->display.update_sprite_wm =
5945                                         haswell_update_sprite_wm;
5946                         } else {
5947                                 DRM_DEBUG_KMS("Failed to read display plane latency. "
5948                                               "Disable CxSR\n");
5949                                 dev_priv->display.update_wm = NULL;
5950                         }
5951                         dev_priv->display.init_clock_gating = haswell_init_clock_gating;
5952                 } else if (INTEL_INFO(dev)->gen == 8) {
5953                         dev_priv->display.init_clock_gating = gen8_init_clock_gating;
5954                 } else
5955                         dev_priv->display.update_wm = NULL;
5956         } else if (IS_VALLEYVIEW(dev)) {
5957                 dev_priv->display.update_wm = valleyview_update_wm;
5958                 dev_priv->display.init_clock_gating =
5959                         valleyview_init_clock_gating;
5960         } else if (IS_PINEVIEW(dev)) {
5961                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
5962                                             dev_priv->is_ddr3,
5963                                             dev_priv->fsb_freq,
5964                                             dev_priv->mem_freq)) {
5965                         DRM_INFO("failed to find known CxSR latency "
5966                                  "(found ddr%s fsb freq %d, mem freq %d), "
5967                                  "disabling CxSR\n",
5968                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
5969                                  dev_priv->fsb_freq, dev_priv->mem_freq);
5970                         /* Disable CxSR and never update its watermark again */
5971                         pineview_disable_cxsr(dev);
5972                         dev_priv->display.update_wm = NULL;
5973                 } else
5974                         dev_priv->display.update_wm = pineview_update_wm;
5975                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
5976         } else if (IS_G4X(dev)) {
5977                 dev_priv->display.update_wm = g4x_update_wm;
5978                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
5979         } else if (IS_GEN4(dev)) {
5980                 dev_priv->display.update_wm = i965_update_wm;
5981                 if (IS_CRESTLINE(dev))
5982                         dev_priv->display.init_clock_gating = crestline_init_clock_gating;
5983                 else if (IS_BROADWATER(dev))
5984                         dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
5985         } else if (IS_GEN3(dev)) {
5986                 dev_priv->display.update_wm = i9xx_update_wm;
5987                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
5988                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
5989         } else if (IS_I865G(dev)) {
5990                 dev_priv->display.update_wm = i830_update_wm;
5991                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
5992                 dev_priv->display.get_fifo_size = i830_get_fifo_size;
5993         } else if (IS_I85X(dev)) {
5994                 dev_priv->display.update_wm = i9xx_update_wm;
5995                 dev_priv->display.get_fifo_size = i85x_get_fifo_size;
5996                 dev_priv->display.init_clock_gating = i85x_init_clock_gating;
5997         } else {
5998                 dev_priv->display.update_wm = i830_update_wm;
5999                 dev_priv->display.init_clock_gating = i830_init_clock_gating;
6000                 if (IS_845G(dev))
6001                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
6002                 else
6003                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
6004         }
6005 }
6006
6007 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val)
6008 {
6009         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6010
6011         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6012                 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
6013                 return -EAGAIN;
6014         }
6015
6016         I915_WRITE(GEN6_PCODE_DATA, *val);
6017         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6018
6019         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6020                      500)) {
6021                 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
6022                 return -ETIMEDOUT;
6023         }
6024
6025         *val = I915_READ(GEN6_PCODE_DATA);
6026         I915_WRITE(GEN6_PCODE_DATA, 0);
6027
6028         return 0;
6029 }
6030
6031 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
6032 {
6033         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6034
6035         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6036                 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
6037                 return -EAGAIN;
6038         }
6039
6040         I915_WRITE(GEN6_PCODE_DATA, val);
6041         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6042
6043         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6044                      500)) {
6045                 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
6046                 return -ETIMEDOUT;
6047         }
6048
6049         I915_WRITE(GEN6_PCODE_DATA, 0);
6050
6051         return 0;
6052 }
6053
6054 int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val)
6055 {
6056         int div;
6057
6058         /* 4 x czclk */
6059         switch (dev_priv->mem_freq) {
6060         case 800:
6061                 div = 10;
6062                 break;
6063         case 1066:
6064                 div = 12;
6065                 break;
6066         case 1333:
6067                 div = 16;
6068                 break;
6069         default:
6070                 return -1;
6071         }
6072
6073         return DIV_ROUND_CLOSEST(dev_priv->mem_freq * (val + 6 - 0xbd), 4 * div);
6074 }
6075
6076 int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val)
6077 {
6078         int mul;
6079
6080         /* 4 x czclk */
6081         switch (dev_priv->mem_freq) {
6082         case 800:
6083                 mul = 10;
6084                 break;
6085         case 1066:
6086                 mul = 12;
6087                 break;
6088         case 1333:
6089                 mul = 16;
6090                 break;
6091         default:
6092                 return -1;
6093         }
6094
6095         return DIV_ROUND_CLOSEST(4 * mul * val, dev_priv->mem_freq) + 0xbd - 6;
6096 }
6097
6098 void intel_pm_init(struct drm_device *dev)
6099 {
6100         struct drm_i915_private *dev_priv = dev->dev_private;
6101
6102         INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
6103                           intel_gen6_powersave_work);
6104 }