drm: Sync vblank handling code with Linux 3.14
[dragonfly.git] / sys / dev / drm / i915 / intel_pm.c
<
CommitLineData
87f57070
FT
1/*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
25 *
26 */
27
87f57070 28#include "i915_drv.h"
a2fdbec6
FT
29#include "intel_drv.h"
30#include <linux/module.h>
c0bdd5d9 31#include <machine/clock.h>
9edbd4a0 32#include <drm/i915_powerwell.h>
87f57070 33
9edbd4a0
FT
34/**
35 * RC6 is a special power stage which allows the GPU to enter an very
36 * low-voltage mode when idle, using down to 0V while at this stage. This
37 * stage is entered automatically when the GPU is idle when RC6 support is
38 * enabled, and as soon as new workload arises GPU wakes up automatically as well.
39 *
40 * There are different RC6 modes available in Intel GPU, which differentiate
41 * among each other with the latency required to enter and leave RC6 and
42 * voltage consumed by the GPU in different states.
43 *
44 * The combination of the following flags define which states GPU is allowed
45 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
46 * RC6pp is deepest RC6. Their support by hardware varies according to the
47 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
48 * which brings the most power savings; deeper states save more power, but
49 * require higher latency to switch to and wake up.
50 */
51#define INTEL_RC6_ENABLE (1<<0)
52#define INTEL_RC6p_ENABLE (1<<1)
53#define INTEL_RC6pp_ENABLE (1<<2)
e9243325 54
0db36e9f
FT
55/* FBC, or Frame Buffer Compression, is a technique employed to compress the
56 * framebuffer contents in-memory, aiming at reducing the required bandwidth
57 * during in-memory transfers and, therefore, reduce the power packet.
58 *
59 * The benefits of FBC are mostly visible with solid backgrounds and
60 * variation-less patterns.
61 *
62 * FBC-related functionality can be enabled by the means of the
63 * i915.i915_enable_fbc parameter
64 */
65
c0bdd5d9 66static void i8xx_disable_fbc(struct drm_device *dev)
87f57070
FT
67{
68 struct drm_i915_private *dev_priv = dev->dev_private;
69 u32 fbc_ctl;
70
71 /* Disable compression */
72 fbc_ctl = I915_READ(FBC_CONTROL);
73 if ((fbc_ctl & FBC_CTL_EN) == 0)
74 return;
75
76 fbc_ctl &= ~FBC_CTL_EN;
77 I915_WRITE(FBC_CONTROL, fbc_ctl);
78
79 /* Wait for compressing bit to clear */
c0bdd5d9 80 if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
87f57070
FT
81 DRM_DEBUG_KMS("FBC idle timed out\n");
82 return;
83 }
84
85 DRM_DEBUG_KMS("disabled FBC\n");
86}
87
9edbd4a0 88static void i8xx_enable_fbc(struct drm_crtc *crtc)
87f57070
FT
89{
90 struct drm_device *dev = crtc->dev;
91 struct drm_i915_private *dev_priv = dev->dev_private;
92 struct drm_framebuffer *fb = crtc->fb;
93 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
94 struct drm_i915_gem_object *obj = intel_fb->obj;
95 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
96 int cfb_pitch;
97 int plane, i;
9edbd4a0 98 u32 fbc_ctl;
87f57070 99
9edbd4a0 100 cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
87f57070
FT
101 if (fb->pitches[0] < cfb_pitch)
102 cfb_pitch = fb->pitches[0];
103
9edbd4a0
FT
104 /* FBC_CTL wants 32B or 64B units */
105 if (IS_GEN2(dev))
106 cfb_pitch = (cfb_pitch / 32) - 1;
107 else
108 cfb_pitch = (cfb_pitch / 64) - 1;
87f57070
FT
109 plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
110
111 /* Clear old tags */
112 for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
113 I915_WRITE(FBC_TAG + (i * 4), 0);
114
9edbd4a0
FT
115 if (IS_GEN4(dev)) {
116 u32 fbc_ctl2;
117
118 /* Set it up... */
119 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
120 fbc_ctl2 |= plane;
121 I915_WRITE(FBC_CONTROL2, fbc_ctl2);
122 I915_WRITE(FBC_FENCE_OFF, crtc->y);
123 }
87f57070
FT
124
125 /* enable it... */
9edbd4a0
FT
126 fbc_ctl = I915_READ(FBC_CONTROL);
127 fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT;
128 fbc_ctl |= FBC_CTL_EN | FBC_CTL_PERIODIC;
87f57070
FT
129 if (IS_I945GM(dev))
130 fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
131 fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
87f57070
FT
132 fbc_ctl |= obj->fence_reg;
133 I915_WRITE(FBC_CONTROL, fbc_ctl);
134
5d0b1887
FT
135 DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
136 cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
87f57070
FT
137}
138
c0bdd5d9 139static bool i8xx_fbc_enabled(struct drm_device *dev)
87f57070
FT
140{
141 struct drm_i915_private *dev_priv = dev->dev_private;
142
143 return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
144}
145
9edbd4a0 146static void g4x_enable_fbc(struct drm_crtc *crtc)
87f57070
FT
147{
148 struct drm_device *dev = crtc->dev;
149 struct drm_i915_private *dev_priv = dev->dev_private;
150 struct drm_framebuffer *fb = crtc->fb;
151 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
152 struct drm_i915_gem_object *obj = intel_fb->obj;
153 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
154 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
87f57070
FT
155 u32 dpfc_ctl;
156
157 dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
158 dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
159 I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
160
87f57070
FT
161 I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
162
163 /* enable it... */
164 I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
165
5d0b1887 166 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
87f57070
FT
167}
168
c0bdd5d9 169static void g4x_disable_fbc(struct drm_device *dev)
87f57070
FT
170{
171 struct drm_i915_private *dev_priv = dev->dev_private;
172 u32 dpfc_ctl;
173
174 /* Disable compression */
175 dpfc_ctl = I915_READ(DPFC_CONTROL);
176 if (dpfc_ctl & DPFC_CTL_EN) {
177 dpfc_ctl &= ~DPFC_CTL_EN;
178 I915_WRITE(DPFC_CONTROL, dpfc_ctl);
179
180 DRM_DEBUG_KMS("disabled FBC\n");
181 }
182}
183
c0bdd5d9 184static bool g4x_fbc_enabled(struct drm_device *dev)
87f57070
FT
185{
186 struct drm_i915_private *dev_priv = dev->dev_private;
187
188 return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
189}
190
191static void sandybridge_blit_fbc_update(struct drm_device *dev)
192{
193 struct drm_i915_private *dev_priv = dev->dev_private;
194 u32 blt_ecoskpd;
195
196 /* Make sure blitter notifies FBC of writes */
9edbd4a0
FT
197
198 /* Blitter is part of Media powerwell on VLV. No impact of
199 * his param in other platforms for now */
200 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA);
201
87f57070
FT
202 blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
203 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
204 GEN6_BLITTER_LOCK_SHIFT;
205 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
206 blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
207 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
208 blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
209 GEN6_BLITTER_LOCK_SHIFT);
210 I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
211 POSTING_READ(GEN6_BLITTER_ECOSKPD);
9edbd4a0
FT
212
213 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA);
87f57070
FT
214}
215
9edbd4a0 216static void ironlake_enable_fbc(struct drm_crtc *crtc)
87f57070
FT
217{
218 struct drm_device *dev = crtc->dev;
219 struct drm_i915_private *dev_priv = dev->dev_private;
220 struct drm_framebuffer *fb = crtc->fb;
221 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
222 struct drm_i915_gem_object *obj = intel_fb->obj;
223 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
224 int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
87f57070
FT
225 u32 dpfc_ctl;
226
227 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
228 dpfc_ctl &= DPFC_RESERVED;
229 dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
230 /* Set persistent mode for front-buffer rendering, ala X. */
231 dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
9edbd4a0
FT
232 dpfc_ctl |= DPFC_CTL_FENCE_EN;
233 if (IS_GEN5(dev))
234 dpfc_ctl |= obj->fence_reg;
87f57070
FT
235 I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
236
87f57070 237 I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
9edbd4a0 238 I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID);
87f57070
FT
239 /* enable it... */
240 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
241
242 if (IS_GEN6(dev)) {
243 I915_WRITE(SNB_DPFC_CTL_SA,
244 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
245 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
246 sandybridge_blit_fbc_update(dev);
247 }
248
5d0b1887 249 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
87f57070
FT
250}
251
c0bdd5d9 252static void ironlake_disable_fbc(struct drm_device *dev)
87f57070
FT
253{
254 struct drm_i915_private *dev_priv = dev->dev_private;
255 u32 dpfc_ctl;
256
257 /* Disable compression */
258 dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
259 if (dpfc_ctl & DPFC_CTL_EN) {
260 dpfc_ctl &= ~DPFC_CTL_EN;
261 I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
262
263 DRM_DEBUG_KMS("disabled FBC\n");
264 }
265}
266
c0bdd5d9 267static bool ironlake_fbc_enabled(struct drm_device *dev)
87f57070
FT
268{
269 struct drm_i915_private *dev_priv = dev->dev_private;
270
271 return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
272}
273
9edbd4a0 274static void gen7_enable_fbc(struct drm_crtc *crtc)
5d0b1887
FT
275{
276 struct drm_device *dev = crtc->dev;
277 struct drm_i915_private *dev_priv = dev->dev_private;
278 struct drm_framebuffer *fb = crtc->fb;
279 struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
280 struct drm_i915_gem_object *obj = intel_fb->obj;
281 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
282
9edbd4a0 283 I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj));
5d0b1887
FT
284
285 I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X |
286 IVB_DPFC_CTL_FENCE_EN |
287 intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
288
289 if (IS_IVYBRIDGE(dev)) {
290 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
291 I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
5d0b1887
FT
292 } else {
293 /* WaFbcAsynchFlipDisableFbcQueue:hsw */
294 I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
295 HSW_BYPASS_FBC_QUEUE);
5d0b1887
FT
296 }
297
298 I915_WRITE(SNB_DPFC_CTL_SA,
299 SNB_CPU_FENCE_ENABLE | obj->fence_reg);
300 I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
301
302 sandybridge_blit_fbc_update(dev);
303
9edbd4a0 304 DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
5d0b1887
FT
305}
306
87f57070
FT
307bool intel_fbc_enabled(struct drm_device *dev)
308{
309 struct drm_i915_private *dev_priv = dev->dev_private;
310
311 if (!dev_priv->display.fbc_enabled)
312 return false;
313
314 return dev_priv->display.fbc_enabled(dev);
315}
316
abf1f4f4 317static void intel_fbc_work_fn(struct work_struct *__work)
87f57070 318{
abf1f4f4
FT
319 struct intel_fbc_work *work =
320 container_of(to_delayed_work(__work),
321 struct intel_fbc_work, work);
87f57070
FT
322 struct drm_device *dev = work->crtc->dev;
323 struct drm_i915_private *dev_priv = dev->dev_private;
324
a2fdbec6 325 mutex_lock(&dev->struct_mutex);
9edbd4a0 326 if (work == dev_priv->fbc.fbc_work) {
87f57070
FT
327 /* Double check that we haven't switched fb without cancelling
328 * the prior work.
329 */
330 if (work->crtc->fb == work->fb) {
9edbd4a0 331 dev_priv->display.enable_fbc(work->crtc);
87f57070 332
9edbd4a0
FT
333 dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
334 dev_priv->fbc.fb_id = work->crtc->fb->base.id;
335 dev_priv->fbc.y = work->crtc->y;
87f57070
FT
336 }
337
9edbd4a0 338 dev_priv->fbc.fbc_work = NULL;
87f57070 339 }
a2fdbec6 340 mutex_unlock(&dev->struct_mutex);
87f57070 341
158486a6 342 kfree(work);
87f57070
FT
343}
344
345static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
346{
9edbd4a0 347 if (dev_priv->fbc.fbc_work == NULL)
87f57070
FT
348 return;
349
350 DRM_DEBUG_KMS("cancelling pending FBC enable\n");
351
352 /* Synchronisation is provided by struct_mutex and checking of
9edbd4a0 353 * dev_priv->fbc.fbc_work, so we can perform the cancellation
87f57070
FT
354 * entirely asynchronously.
355 */
9edbd4a0 356 if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
87f57070 357 /* tasklet was killed before being run, clean up */
9edbd4a0 358 kfree(dev_priv->fbc.fbc_work);
87f57070
FT
359
360 /* Mark the work as no longer wanted so that if it does
361 * wake-up (because the work was already running and waiting
362 * for our mutex), it will discover that is no longer
363 * necessary to run.
364 */
9edbd4a0 365 dev_priv->fbc.fbc_work = NULL;
87f57070
FT
366}
367
9edbd4a0 368static void intel_enable_fbc(struct drm_crtc *crtc)
87f57070
FT
369{
370 struct intel_fbc_work *work;
371 struct drm_device *dev = crtc->dev;
372 struct drm_i915_private *dev_priv = dev->dev_private;
373
374 if (!dev_priv->display.enable_fbc)
375 return;
376
377 intel_cancel_fbc_work(dev_priv);
378
9edbd4a0 379 work = kzalloc(sizeof(*work), GFP_KERNEL);
c0bdd5d9 380 if (work == NULL) {
9edbd4a0
FT
381 DRM_ERROR("Failed to allocate FBC work structure\n");
382 dev_priv->display.enable_fbc(crtc);
c0bdd5d9
FT
383 return;
384 }
385
87f57070
FT
386 work->crtc = crtc;
387 work->fb = crtc->fb;
abf1f4f4 388 INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
87f57070 389
9edbd4a0 390 dev_priv->fbc.fbc_work = work;
87f57070
FT
391
392 /* Delay the actual enabling to let pageflipping cease and the
393 * display to settle before starting the compression. Note that
394 * this delay also serves a second purpose: it allows for a
395 * vblank to pass after disabling the FBC before we attempt
396 * to modify the control registers.
397 *
398 * A more complicated solution would involve tracking vblanks
399 * following the termination of the page-flipping sequence
400 * and indeed performing the enable as a co-routine and not
401 * waiting synchronously upon the vblank.
9edbd4a0
FT
402 *
403 * WaFbcWaitForVBlankBeforeEnable:ilk,snb
87f57070 404 */
abf1f4f4 405 schedule_delayed_work(&work->work, msecs_to_jiffies(50));
87f57070
FT
406}
407
408void intel_disable_fbc(struct drm_device *dev)
409{
410 struct drm_i915_private *dev_priv = dev->dev_private;
411
412 intel_cancel_fbc_work(dev_priv);
413
414 if (!dev_priv->display.disable_fbc)
415 return;
416
417 dev_priv->display.disable_fbc(dev);
9edbd4a0
FT
418 dev_priv->fbc.plane = -1;
419}
420
421static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
422 enum no_fbc_reason reason)
423{
424 if (dev_priv->fbc.no_fbc_reason == reason)
425 return false;
426
427 dev_priv->fbc.no_fbc_reason = reason;
428 return true;
87f57070
FT
429}
430
431/**
432 * intel_update_fbc - enable/disable FBC as needed
433 * @dev: the drm_device
434 *
435 * Set up the framebuffer compression hardware at mode set time. We
436 * enable it if possible:
437 * - plane A only (on pre-965)
438 * - no pixel mulitply/line duplication
439 * - no alpha buffer discard
440 * - no dual wide
5d0b1887 441 * - framebuffer <= max_hdisplay in width, max_vdisplay in height
87f57070
FT
442 *
443 * We can't assume that any compression will take place (worst case),
444 * so the compressed buffer has to be the same size as the uncompressed
445 * one. It also must reside (along with the line length buffer) in
446 * stolen memory.
447 *
448 * We need to enable/disable FBC on a global basis.
449 */
450void intel_update_fbc(struct drm_device *dev)
451{
452 struct drm_i915_private *dev_priv = dev->dev_private;
453 struct drm_crtc *crtc = NULL, *tmp_crtc;
454 struct intel_crtc *intel_crtc;
455 struct drm_framebuffer *fb;
456 struct intel_framebuffer *intel_fb;
457 struct drm_i915_gem_object *obj;
9edbd4a0
FT
458 const struct drm_display_mode *adjusted_mode;
459 unsigned int max_width, max_height;
87f57070 460
9edbd4a0
FT
461 if (!HAS_FBC(dev)) {
462 set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
87f57070 463 return;
9edbd4a0 464 }
87f57070 465
9edbd4a0
FT
466 if (!i915_powersave) {
467 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
468 DRM_DEBUG_KMS("fbc disabled per module param\n");
87f57070 469 return;
9edbd4a0 470 }
87f57070
FT
471
472 /*
473 * If FBC is already on, we just have to verify that we can
474 * keep it that way...
475 * Need to disable if:
476 * - more than one pipe is active
477 * - changing FBC params (stride, fence, mode)
478 * - new fb is too large to fit in compressed buffer
479 * - going to an unsupported config (interlace, pixel multiply, etc.)
480 */
481 list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
c0bdd5d9 482 if (intel_crtc_active(tmp_crtc) &&
9edbd4a0 483 to_intel_crtc(tmp_crtc)->primary_enabled) {
87f57070 484 if (crtc) {
9edbd4a0
FT
485 if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
486 DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
87f57070
FT
487 goto out_disable;
488 }
489 crtc = tmp_crtc;
490 }
491 }
492
493 if (!crtc || crtc->fb == NULL) {
9edbd4a0
FT
494 if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
495 DRM_DEBUG_KMS("no output, disabling\n");
87f57070
FT
496 goto out_disable;
497 }
498
499 intel_crtc = to_intel_crtc(crtc);
500 fb = crtc->fb;
501 intel_fb = to_intel_framebuffer(fb);
502 obj = intel_fb->obj;
9edbd4a0 503 adjusted_mode = &intel_crtc->config.adjusted_mode;
87f57070 504
9edbd4a0
FT
505 if (i915_enable_fbc < 0 &&
506 INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) {
507 if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
508 DRM_DEBUG_KMS("disabled per chip default\n");
509 goto out_disable;
87f57070 510 }
9edbd4a0
FT
511 if (!i915_enable_fbc) {
512 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
513 DRM_DEBUG_KMS("fbc disabled per module param\n");
87f57070
FT
514 goto out_disable;
515 }
9edbd4a0
FT
516 if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) ||
517 (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
518 if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
519 DRM_DEBUG_KMS("mode incompatible with compression, "
520 "disabling\n");
87f57070
FT
521 goto out_disable;
522 }
5d0b1887
FT
523
524 if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
9edbd4a0
FT
525 max_width = 4096;
526 max_height = 2048;
5d0b1887 527 } else {
9edbd4a0
FT
528 max_width = 2048;
529 max_height = 1536;
5d0b1887 530 }
9edbd4a0
FT
531 if (intel_crtc->config.pipe_src_w > max_width ||
532 intel_crtc->config.pipe_src_h > max_height) {
533 if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
534 DRM_DEBUG_KMS("mode too large for compression, disabling\n");
87f57070
FT
535 goto out_disable;
536 }
9edbd4a0
FT
537 if ((INTEL_INFO(dev)->gen < 4 || IS_HASWELL(dev)) &&
538 intel_crtc->plane != PLANE_A) {
539 if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
540 DRM_DEBUG_KMS("plane not A, disabling compression\n");
87f57070
FT
541 goto out_disable;
542 }
c0bdd5d9
FT
543
544 /* The use of a CPU fence is mandatory in order to detect writes
545 * by the CPU to the scanout and trigger updates to the FBC.
546 */
87f57070
FT
547 if (obj->tiling_mode != I915_TILING_X ||
548 obj->fence_reg == I915_FENCE_REG_NONE) {
9edbd4a0
FT
549 if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
550 DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
87f57070
FT
551 goto out_disable;
552 }
553
87f57070 554 /* If the kernel debugger is active, always disable compression */
9edbd4a0
FT
555#ifdef DDB
556 if (in_dbg_master())
87f57070
FT
557 goto out_disable;
558#endif
559
a2fdbec6 560 if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
9edbd4a0
FT
561 if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
562 DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
a2fdbec6
FT
563 goto out_disable;
564 }
565
87f57070
FT
566 /* If the scanout has not changed, don't modify the FBC settings.
567 * Note that we make the fundamental assumption that the fb->obj
568 * cannot be unpinned (and have its GTT offset and fence revoked)
569 * without first being decoupled from the scanout and FBC disabled.
570 */
9edbd4a0
FT
571 if (dev_priv->fbc.plane == intel_crtc->plane &&
572 dev_priv->fbc.fb_id == fb->base.id &&
573 dev_priv->fbc.y == crtc->y)
87f57070
FT
574 return;
575
576 if (intel_fbc_enabled(dev)) {
577 /* We update FBC along two paths, after changing fb/crtc
578 * configuration (modeswitching) and after page-flipping
579 * finishes. For the latter, we know that not only did
580 * we disable the FBC at the start of the page-flip
581 * sequence, but also more than one vblank has passed.
582 *
583 * For the former case of modeswitching, it is possible
584 * to switch between two FBC valid configurations
585 * instantaneously so we do need to disable the FBC
586 * before we can modify its control registers. We also
587 * have to wait for the next vblank for that to take
588 * effect. However, since we delay enabling FBC we can
589 * assume that a vblank has passed since disabling and
590 * that we can safely alter the registers in the deferred
591 * callback.
592 *
593 * In the scenario that we go from a valid to invalid
594 * and then back to valid FBC configuration we have
595 * no strict enforcement that a vblank occurred since
596 * disabling the FBC. However, along all current pipe
597 * disabling paths we do need to wait for a vblank at
598 * some point. And we wait before enabling FBC anyway.
599 */
600 DRM_DEBUG_KMS("disabling active FBC for update\n");
601 intel_disable_fbc(dev);
602 }
603
9edbd4a0
FT
604 intel_enable_fbc(crtc);
605 dev_priv->fbc.no_fbc_reason = FBC_OK;
87f57070
FT
606 return;
607
608out_disable:
609 /* Multiple disables should be harmless */
610 if (intel_fbc_enabled(dev)) {
611 DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
612 intel_disable_fbc(dev);
613 }
a2fdbec6 614 i915_gem_stolen_cleanup_compression(dev);
87f57070
FT
615}
616
0db36e9f 617static void i915_pineview_get_mem_freq(struct drm_device *dev)
bc9b2a29
FT
618{
619 drm_i915_private_t *dev_priv = dev->dev_private;
620 u32 tmp;
621
622 tmp = I915_READ(CLKCFG);
623
624 switch (tmp & CLKCFG_FSB_MASK) {
625 case CLKCFG_FSB_533:
626 dev_priv->fsb_freq = 533; /* 133*4 */
627 break;
628 case CLKCFG_FSB_800:
629 dev_priv->fsb_freq = 800; /* 200*4 */
630 break;
631 case CLKCFG_FSB_667:
632 dev_priv->fsb_freq = 667; /* 167*4 */
633 break;
634 case CLKCFG_FSB_400:
635 dev_priv->fsb_freq = 400; /* 100*4 */
636 break;
637 }
638
639 switch (tmp & CLKCFG_MEM_MASK) {
640 case CLKCFG_MEM_533:
641 dev_priv->mem_freq = 533;
642 break;
643 case CLKCFG_MEM_667:
644 dev_priv->mem_freq = 667;
645 break;
646 case CLKCFG_MEM_800:
647 dev_priv->mem_freq = 800;
648 break;
649 }
650
651 /* detect pineview DDR3 setting */
652 tmp = I915_READ(CSHRDDR3CTL);
653 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
654}
655
0db36e9f 656static void i915_ironlake_get_mem_freq(struct drm_device *dev)
bc9b2a29
FT
657{
658 drm_i915_private_t *dev_priv = dev->dev_private;
659 u16 ddrpll, csipll;
660
661 ddrpll = I915_READ16(DDRMPLL1);
662 csipll = I915_READ16(CSIPLL0);
663
664 switch (ddrpll & 0xff) {
665 case 0xc:
666 dev_priv->mem_freq = 800;
667 break;
668 case 0x10:
669 dev_priv->mem_freq = 1066;
670 break;
671 case 0x14:
672 dev_priv->mem_freq = 1333;
673 break;
674 case 0x18:
675 dev_priv->mem_freq = 1600;
676 break;
677 default:
0db36e9f 678 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
bc9b2a29
FT
679 ddrpll & 0xff);
680 dev_priv->mem_freq = 0;
681 break;
682 }
683
0db36e9f 684 dev_priv->ips.r_t = dev_priv->mem_freq;
bc9b2a29
FT
685
686 switch (csipll & 0x3ff) {
687 case 0x00c:
688 dev_priv->fsb_freq = 3200;
689 break;
690 case 0x00e:
691 dev_priv->fsb_freq = 3733;
692 break;
693 case 0x010:
694 dev_priv->fsb_freq = 4266;
695 break;
696 case 0x012:
697 dev_priv->fsb_freq = 4800;
698 break;
699 case 0x014:
700 dev_priv->fsb_freq = 5333;
701 break;
702 case 0x016:
703 dev_priv->fsb_freq = 5866;
704 break;
705 case 0x018:
706 dev_priv->fsb_freq = 6400;
707 break;
708 default:
0db36e9f 709 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
bc9b2a29
FT
710 csipll & 0x3ff);
711 dev_priv->fsb_freq = 0;
712 break;
713 }
714
715 if (dev_priv->fsb_freq == 3200) {
0db36e9f 716 dev_priv->ips.c_m = 0;
bc9b2a29 717 } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
0db36e9f 718 dev_priv->ips.c_m = 1;
bc9b2a29 719 } else {
0db36e9f 720 dev_priv->ips.c_m = 2;
bc9b2a29
FT
721 }
722}
723
c0bdd5d9
FT
724static const struct cxsr_latency cxsr_latency_table[] = {
725 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
726 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
727 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
728 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
729 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
730
731 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
732 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
733 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
734 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
735 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
736
737 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
738 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
739 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
740 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
741 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
742
743 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
744 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
745 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
746 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
747 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
748
749 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
750 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
751 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
752 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
753 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
754
755 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
756 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
757 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
758 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
759 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
760};
761
762static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
763 int is_ddr3,
764 int fsb,
765 int mem)
766{
767 const struct cxsr_latency *latency;
768 int i;
769
770 if (fsb == 0 || mem == 0)
771 return NULL;
772
773 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
774 latency = &cxsr_latency_table[i];
775 if (is_desktop == latency->is_desktop &&
776 is_ddr3 == latency->is_ddr3 &&
777 fsb == latency->fsb_freq && mem == latency->mem_freq)
778 return latency;
779 }
780
781 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
782
783 return NULL;
784}
785
786static void pineview_disable_cxsr(struct drm_device *dev)
787{
788 struct drm_i915_private *dev_priv = dev->dev_private;
789
790 /* deactivate cxsr */
791 I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
792}
793
794/*
795 * Latency for FIFO fetches is dependent on several factors:
796 * - memory configuration (speed, channels)
797 * - chipset
798 * - current MCH state
799 * It can be fairly high in some situations, so here we assume a fairly
800 * pessimal value. It's a tradeoff between extra memory fetches (if we
801 * set this value too high, the FIFO will fetch frequently to stay full)
802 * and power consumption (set it too low to save power and we might see
803 * FIFO underruns and display "flicker").
804 *
805 * A value of 5us seems to be a good balance; safe for very low end
806 * platforms but not overly aggressive on lower latency configs.
807 */
808static const int latency_ns = 5000;
809
810static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
811{
812 struct drm_i915_private *dev_priv = dev->dev_private;
813 uint32_t dsparb = I915_READ(DSPARB);
814 int size;
815
816 size = dsparb & 0x7f;
817 if (plane)
818 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
819
820 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
821 plane ? "B" : "A", size);
822
823 return size;
824}
825
9edbd4a0 826static int i830_get_fifo_size(struct drm_device *dev, int plane)
c0bdd5d9
FT
827{
828 struct drm_i915_private *dev_priv = dev->dev_private;
829 uint32_t dsparb = I915_READ(DSPARB);
830 int size;
831
832 size = dsparb & 0x1ff;
833 if (plane)
834 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
835 size >>= 1; /* Convert to cachelines */
836
837 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
838 plane ? "B" : "A", size);
839
840 return size;
841}
842
843static int i845_get_fifo_size(struct drm_device *dev, int plane)
844{
845 struct drm_i915_private *dev_priv = dev->dev_private;
846 uint32_t dsparb = I915_READ(DSPARB);
847 int size;
848
849 size = dsparb & 0x7f;
850 size >>= 2; /* Convert to cachelines */
851
852 DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
853 plane ? "B" : "A",
854 size);
855
856 return size;
857}
858
87f57070
FT
859/* Pineview has different values for various configs */
860static const struct intel_watermark_params pineview_display_wm = {
861 PINEVIEW_DISPLAY_FIFO,
862 PINEVIEW_MAX_WM,
863 PINEVIEW_DFT_WM,
864 PINEVIEW_GUARD_WM,
865 PINEVIEW_FIFO_LINE_SIZE
866};
867static const struct intel_watermark_params pineview_display_hplloff_wm = {
868 PINEVIEW_DISPLAY_FIFO,
869 PINEVIEW_MAX_WM,
870 PINEVIEW_DFT_HPLLOFF_WM,
871 PINEVIEW_GUARD_WM,
872 PINEVIEW_FIFO_LINE_SIZE
873};
874static const struct intel_watermark_params pineview_cursor_wm = {
875 PINEVIEW_CURSOR_FIFO,
876 PINEVIEW_CURSOR_MAX_WM,
877 PINEVIEW_CURSOR_DFT_WM,
878 PINEVIEW_CURSOR_GUARD_WM,
879 PINEVIEW_FIFO_LINE_SIZE,
880};
881static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
882 PINEVIEW_CURSOR_FIFO,
883 PINEVIEW_CURSOR_MAX_WM,
884 PINEVIEW_CURSOR_DFT_WM,
885 PINEVIEW_CURSOR_GUARD_WM,
886 PINEVIEW_FIFO_LINE_SIZE
887};
888static const struct intel_watermark_params g4x_wm_info = {
889 G4X_FIFO_SIZE,
890 G4X_MAX_WM,
891 G4X_MAX_WM,
892 2,
893 G4X_FIFO_LINE_SIZE,
894};
895static const struct intel_watermark_params g4x_cursor_wm_info = {
896 I965_CURSOR_FIFO,
897 I965_CURSOR_MAX_WM,
898 I965_CURSOR_DFT_WM,
899 2,
900 G4X_FIFO_LINE_SIZE,
901};
0db36e9f
FT
902static const struct intel_watermark_params valleyview_wm_info = {
903 VALLEYVIEW_FIFO_SIZE,
904 VALLEYVIEW_MAX_WM,
905 VALLEYVIEW_MAX_WM,
906 2,
907 G4X_FIFO_LINE_SIZE,
908};
909static const struct intel_watermark_params valleyview_cursor_wm_info = {
910 I965_CURSOR_FIFO,
911 VALLEYVIEW_CURSOR_MAX_WM,
912 I965_CURSOR_DFT_WM,
913 2,
914 G4X_FIFO_LINE_SIZE,
915};
87f57070
FT
916static const struct intel_watermark_params i965_cursor_wm_info = {
917 I965_CURSOR_FIFO,
918 I965_CURSOR_MAX_WM,
919 I965_CURSOR_DFT_WM,
920 2,
921 I915_FIFO_LINE_SIZE,
922};
923static const struct intel_watermark_params i945_wm_info = {
924 I945_FIFO_SIZE,
925 I915_MAX_WM,
926 1,
927 2,
928 I915_FIFO_LINE_SIZE
929};
930static const struct intel_watermark_params i915_wm_info = {
931 I915_FIFO_SIZE,
932 I915_MAX_WM,
933 1,
934 2,
935 I915_FIFO_LINE_SIZE
936};
9edbd4a0 937static const struct intel_watermark_params i830_wm_info = {
87f57070
FT
938 I855GM_FIFO_SIZE,
939 I915_MAX_WM,
940 1,
941 2,
942 I830_FIFO_LINE_SIZE
943};
9edbd4a0 944static const struct intel_watermark_params i845_wm_info = {
87f57070
FT
945 I830_FIFO_SIZE,
946 I915_MAX_WM,
947 1,
948 2,
949 I830_FIFO_LINE_SIZE
950};
951
87f57070
FT
952/**
953 * intel_calculate_wm - calculate watermark level
954 * @clock_in_khz: pixel clock
955 * @wm: chip FIFO params
956 * @pixel_size: display pixel size
957 * @latency_ns: memory latency for the platform
958 *
959 * Calculate the watermark level (the level at which the display plane will
960 * start fetching from memory again). Each chip has a different display
961 * FIFO size and allocation, so the caller needs to figure that out and pass
962 * in the correct intel_watermark_params structure.
963 *
964 * As the pixel clock runs, the FIFO will be drained at a rate that depends
965 * on the pixel size. When it reaches the watermark level, it'll start
966 * fetching FIFO line sized based chunks from memory until the FIFO fills
967 * past the watermark point. If the FIFO drains completely, a FIFO underrun
968 * will occur, and a display engine hang could result.
969 */
970static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
971 const struct intel_watermark_params *wm,
972 int fifo_size,
973 int pixel_size,
974 unsigned long latency_ns)
975{
976 long entries_required, wm_size;
977
978 /*
979 * Note: we need to make sure we don't overflow for various clock &
980 * latency values.
981 * clocks go from a few thousand to several hundred thousand.
982 * latency is usually a few thousand
983 */
984 entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
985 1000;
c0bdd5d9 986 entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
87f57070
FT
987
988 DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
989
990 wm_size = fifo_size - (entries_required + wm->guard_size);
991
992 DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
993
994 /* Don't promote wm_size to unsigned... */
995 if (wm_size > (long)wm->max_wm)
996 wm_size = wm->max_wm;
997 if (wm_size <= 0)
998 wm_size = wm->default_wm;
999 return wm_size;
1000}
1001
c0bdd5d9
FT
1002static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
1003{
1004 struct drm_crtc *crtc, *enabled = NULL;
87f57070 1005
c0bdd5d9
FT
1006 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1007 if (intel_crtc_active(crtc)) {
1008 if (enabled)
1009 return NULL;
1010 enabled = crtc;
1011 }
1012 }
87f57070 1013
c0bdd5d9
FT
1014 return enabled;
1015}
87f57070 1016
9edbd4a0 1017static void pineview_update_wm(struct drm_crtc *unused_crtc)
87f57070 1018{
9edbd4a0 1019 struct drm_device *dev = unused_crtc->dev;
c0bdd5d9
FT
1020 struct drm_i915_private *dev_priv = dev->dev_private;
1021 struct drm_crtc *crtc;
87f57070 1022 const struct cxsr_latency *latency;
c0bdd5d9
FT
1023 u32 reg;
1024 unsigned long wm;
87f57070 1025
c0bdd5d9
FT
1026 latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
1027 dev_priv->fsb_freq, dev_priv->mem_freq);
1028 if (!latency) {
1029 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
1030 pineview_disable_cxsr(dev);
1031 return;
87f57070
FT
1032 }
1033
c0bdd5d9
FT
1034 crtc = single_enabled_crtc(dev);
1035 if (crtc) {
9edbd4a0 1036 const struct drm_display_mode *adjusted_mode;
c0bdd5d9 1037 int pixel_size = crtc->fb->bits_per_pixel / 8;
9edbd4a0
FT
1038 int clock;
1039
1040 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1041 clock = adjusted_mode->crtc_clock;
87f57070 1042
c0bdd5d9
FT
1043 /* Display SR */
1044 wm = intel_calculate_wm(clock, &pineview_display_wm,
1045 pineview_display_wm.fifo_size,
1046 pixel_size, latency->display_sr);
1047 reg = I915_READ(DSPFW1);
1048 reg &= ~DSPFW_SR_MASK;
1049 reg |= wm << DSPFW_SR_SHIFT;
1050 I915_WRITE(DSPFW1, reg);
1051 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
87f57070 1052
c0bdd5d9
FT
1053 /* cursor SR */
1054 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
1055 pineview_display_wm.fifo_size,
1056 pixel_size, latency->cursor_sr);
1057 reg = I915_READ(DSPFW3);
1058 reg &= ~DSPFW_CURSOR_SR_MASK;
1059 reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
1060 I915_WRITE(DSPFW3, reg);
87f57070
FT
1061
1062 /* Display HPLL off SR */
1063 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
1064 pineview_display_hplloff_wm.fifo_size,
1065 pixel_size, latency->display_hpll_disable);
1066 reg = I915_READ(DSPFW3);
1067 reg &= ~DSPFW_HPLL_SR_MASK;
1068 reg |= wm & DSPFW_HPLL_SR_MASK;
1069 I915_WRITE(DSPFW3, reg);
1070
1071 /* cursor HPLL off SR */
1072 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
1073 pineview_display_hplloff_wm.fifo_size,
1074 pixel_size, latency->cursor_hpll_disable);
1075 reg = I915_READ(DSPFW3);
1076 reg &= ~DSPFW_HPLL_CURSOR_MASK;
1077 reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
1078 I915_WRITE(DSPFW3, reg);
1079 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
1080
1081 /* activate cxsr */
1082 I915_WRITE(DSPFW3,
1083 I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
1084 DRM_DEBUG_KMS("Self-refresh is enabled\n");
1085 } else {
1086 pineview_disable_cxsr(dev);
1087 DRM_DEBUG_KMS("Self-refresh is disabled\n");
1088 }
1089}
1090
1091static bool g4x_compute_wm0(struct drm_device *dev,
1092 int plane,
1093 const struct intel_watermark_params *display,
1094 int display_latency_ns,
1095 const struct intel_watermark_params *cursor,
1096 int cursor_latency_ns,
1097 int *plane_wm,
1098 int *cursor_wm)
1099{
1100 struct drm_crtc *crtc;
9edbd4a0 1101 const struct drm_display_mode *adjusted_mode;
87f57070
FT
1102 int htotal, hdisplay, clock, pixel_size;
1103 int line_time_us, line_count;
1104 int entries, tlb_miss;
1105
1106 crtc = intel_get_crtc_for_plane(dev, plane);
c0bdd5d9 1107 if (!intel_crtc_active(crtc)) {
87f57070
FT
1108 *cursor_wm = cursor->guard_size;
1109 *plane_wm = display->guard_size;
1110 return false;
1111 }
1112
9edbd4a0
FT
1113 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1114 clock = adjusted_mode->crtc_clock;
1115 htotal = adjusted_mode->crtc_htotal;
1116 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
87f57070
FT
1117 pixel_size = crtc->fb->bits_per_pixel / 8;
1118
1119 /* Use the small buffer method to calculate plane watermark */
1120 entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
1121 tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
1122 if (tlb_miss > 0)
1123 entries += tlb_miss;
c0bdd5d9 1124 entries = DIV_ROUND_UP(entries, display->cacheline_size);
87f57070
FT
1125 *plane_wm = entries + display->guard_size;
1126 if (*plane_wm > (int)display->max_wm)
1127 *plane_wm = display->max_wm;
1128
1129 /* Use the large buffer method to calculate cursor watermark */
1130 line_time_us = ((htotal * 1000) / clock);
1131 line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
1132 entries = line_count * 64 * pixel_size;
1133 tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
1134 if (tlb_miss > 0)
1135 entries += tlb_miss;
c0bdd5d9 1136 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
87f57070
FT
1137 *cursor_wm = entries + cursor->guard_size;
1138 if (*cursor_wm > (int)cursor->max_wm)
1139 *cursor_wm = (int)cursor->max_wm;
1140
1141 return true;
1142}
1143
1144/*
1145 * Check the wm result.
1146 *
1147 * If any calculated watermark values is larger than the maximum value that
1148 * can be programmed into the associated watermark register, that watermark
1149 * must be disabled.
1150 */
1151static bool g4x_check_srwm(struct drm_device *dev,
1152 int display_wm, int cursor_wm,
1153 const struct intel_watermark_params *display,
1154 const struct intel_watermark_params *cursor)
1155{
1156 DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
1157 display_wm, cursor_wm);
1158
1159 if (display_wm > display->max_wm) {
1160 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
1161 display_wm, display->max_wm);
1162 return false;
1163 }
1164
1165 if (cursor_wm > cursor->max_wm) {
1166 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
1167 cursor_wm, cursor->max_wm);
1168 return false;
1169 }
1170
1171 if (!(display_wm || cursor_wm)) {
1172 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
1173 return false;
1174 }
1175
1176 return true;
1177}
1178
1179static bool g4x_compute_srwm(struct drm_device *dev,
1180 int plane,
1181 int latency_ns,
1182 const struct intel_watermark_params *display,
1183 const struct intel_watermark_params *cursor,
1184 int *display_wm, int *cursor_wm)
1185{
1186 struct drm_crtc *crtc;
9edbd4a0 1187 const struct drm_display_mode *adjusted_mode;
87f57070
FT
1188 int hdisplay, htotal, pixel_size, clock;
1189 unsigned long line_time_us;
1190 int line_count, line_size;
1191 int small, large;
1192 int entries;
1193
1194 if (!latency_ns) {
1195 *display_wm = *cursor_wm = 0;
1196 return false;
1197 }
1198
1199 crtc = intel_get_crtc_for_plane(dev, plane);
9edbd4a0
FT
1200 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1201 clock = adjusted_mode->crtc_clock;
1202 htotal = adjusted_mode->crtc_htotal;
1203 hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
87f57070
FT
1204 pixel_size = crtc->fb->bits_per_pixel / 8;
1205
1206 line_time_us = (htotal * 1000) / clock;
1207 line_count = (latency_ns / line_time_us + 1000) / 1000;
1208 line_size = hdisplay * pixel_size;
1209
1210 /* Use the minimum of the small and large buffer method for primary */
1211 small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
1212 large = line_count * line_size;
1213
c0bdd5d9 1214 entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
87f57070
FT
1215 *display_wm = entries + display->guard_size;
1216
1217 /* calculate the self-refresh watermark for display cursor */
1218 entries = line_count * pixel_size * 64;
c0bdd5d9 1219 entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
87f57070
FT
1220 *cursor_wm = entries + cursor->guard_size;
1221
1222 return g4x_check_srwm(dev,
1223 *display_wm, *cursor_wm,
1224 display, cursor);
1225}
1226
0db36e9f
FT
1227static bool vlv_compute_drain_latency(struct drm_device *dev,
1228 int plane,
1229 int *plane_prec_mult,
1230 int *plane_dl,
1231 int *cursor_prec_mult,
1232 int *cursor_dl)
1233{
1234 struct drm_crtc *crtc;
1235 int clock, pixel_size;
1236 int entries;
1237
1238 crtc = intel_get_crtc_for_plane(dev, plane);
1239 if (!intel_crtc_active(crtc))
1240 return false;
1241
9edbd4a0 1242 clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
0db36e9f
FT
1243 pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
1244
1245 entries = (clock / 1000) * pixel_size;
1246 *plane_prec_mult = (entries > 256) ?
1247 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1248 *plane_dl = (64 * (*plane_prec_mult) * 4) / ((clock / 1000) *
1249 pixel_size);
1250
1251 entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
1252 *cursor_prec_mult = (entries > 256) ?
1253 DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
1254 *cursor_dl = (64 * (*cursor_prec_mult) * 4) / ((clock / 1000) * 4);
1255
1256 return true;
1257}
1258
1259/*
1260 * Update drain latency registers of memory arbiter
1261 *
1262 * Valleyview SoC has a new memory arbiter and needs drain latency registers
1263 * to be programmed. Each plane has a drain latency multiplier and a drain
1264 * latency value.
1265 */
1266
1267static void vlv_update_drain_latency(struct drm_device *dev)
1268{
1269 struct drm_i915_private *dev_priv = dev->dev_private;
1270 int planea_prec, planea_dl, planeb_prec, planeb_dl;
1271 int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
1272 int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
1273 either 16 or 32 */
1274
1275 /* For plane A, Cursor A */
1276 if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
1277 &cursor_prec_mult, &cursora_dl)) {
1278 cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1279 DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
1280 planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1281 DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
1282
1283 I915_WRITE(VLV_DDL1, cursora_prec |
1284 (cursora_dl << DDL_CURSORA_SHIFT) |
1285 planea_prec | planea_dl);
1286 }
1287
1288 /* For plane B, Cursor B */
1289 if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
1290 &cursor_prec_mult, &cursorb_dl)) {
1291 cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1292 DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
1293 planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
1294 DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
1295
1296 I915_WRITE(VLV_DDL2, cursorb_prec |
1297 (cursorb_dl << DDL_CURSORB_SHIFT) |
1298 planeb_prec | planeb_dl);
1299 }
1300}
1301
a2fdbec6 1302#define single_plane_enabled(mask) is_power_of_2(mask)
87f57070 1303
9edbd4a0 1304static void valleyview_update_wm(struct drm_crtc *crtc)
0db36e9f 1305{
9edbd4a0 1306 struct drm_device *dev = crtc->dev;
0db36e9f
FT
1307 static const int sr_latency_ns = 12000;
1308 struct drm_i915_private *dev_priv = dev->dev_private;
1309 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1310 int plane_sr, cursor_sr;
1311 int ignore_plane_sr, ignore_cursor_sr;
1312 unsigned int enabled = 0;
1313
1314 vlv_update_drain_latency(dev);
1315
8e26cdf6 1316 if (g4x_compute_wm0(dev, PIPE_A,
0db36e9f
FT
1317 &valleyview_wm_info, latency_ns,
1318 &valleyview_cursor_wm_info, latency_ns,
1319 &planea_wm, &cursora_wm))
8e26cdf6 1320 enabled |= 1 << PIPE_A;
0db36e9f 1321
8e26cdf6 1322 if (g4x_compute_wm0(dev, PIPE_B,
0db36e9f
FT
1323 &valleyview_wm_info, latency_ns,
1324 &valleyview_cursor_wm_info, latency_ns,
1325 &planeb_wm, &cursorb_wm))
8e26cdf6 1326 enabled |= 1 << PIPE_B;
0db36e9f
FT
1327
1328 if (single_plane_enabled(enabled) &&
1329 g4x_compute_srwm(dev, ffs(enabled) - 1,
1330 sr_latency_ns,
1331 &valleyview_wm_info,
1332 &valleyview_cursor_wm_info,
1333 &plane_sr, &ignore_cursor_sr) &&
1334 g4x_compute_srwm(dev, ffs(enabled) - 1,
1335 2*sr_latency_ns,
1336 &valleyview_wm_info,
1337 &valleyview_cursor_wm_info,
1338 &ignore_plane_sr, &cursor_sr)) {
1339 I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
1340 } else {
1341 I915_WRITE(FW_BLC_SELF_VLV,
1342 I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
1343 plane_sr = cursor_sr = 0;
1344 }
1345
1346 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1347 planea_wm, cursora_wm,
1348 planeb_wm, cursorb_wm,
1349 plane_sr, cursor_sr);
1350
1351 I915_WRITE(DSPFW1,
1352 (plane_sr << DSPFW_SR_SHIFT) |
1353 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1354 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1355 planea_wm);
1356 I915_WRITE(DSPFW2,
1357 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1358 (cursora_wm << DSPFW_CURSORA_SHIFT));
1359 I915_WRITE(DSPFW3,
1360 (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
1361 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1362}
1363
9edbd4a0 1364static void g4x_update_wm(struct drm_crtc *crtc)
87f57070 1365{
9edbd4a0 1366 struct drm_device *dev = crtc->dev;
87f57070
FT
1367 static const int sr_latency_ns = 12000;
1368 struct drm_i915_private *dev_priv = dev->dev_private;
1369 int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1370 int plane_sr, cursor_sr;
1371 unsigned int enabled = 0;
1372
8e26cdf6 1373 if (g4x_compute_wm0(dev, PIPE_A,
87f57070
FT
1374 &g4x_wm_info, latency_ns,
1375 &g4x_cursor_wm_info, latency_ns,
1376 &planea_wm, &cursora_wm))
8e26cdf6 1377 enabled |= 1 << PIPE_A;
87f57070 1378
8e26cdf6 1379 if (g4x_compute_wm0(dev, PIPE_B,
87f57070
FT
1380 &g4x_wm_info, latency_ns,
1381 &g4x_cursor_wm_info, latency_ns,
1382 &planeb_wm, &cursorb_wm))
8e26cdf6 1383 enabled |= 1 << PIPE_B;
87f57070 1384
87f57070
FT
1385 if (single_plane_enabled(enabled) &&
1386 g4x_compute_srwm(dev, ffs(enabled) - 1,
1387 sr_latency_ns,
1388 &g4x_wm_info,
1389 &g4x_cursor_wm_info,
c0bdd5d9 1390 &plane_sr, &cursor_sr)) {
87f57070 1391 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
c0bdd5d9 1392 } else {
87f57070
FT
1393 I915_WRITE(FW_BLC_SELF,
1394 I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
c0bdd5d9
FT
1395 plane_sr = cursor_sr = 0;
1396 }
87f57070
FT
1397
1398 DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1399 planea_wm, cursora_wm,
1400 planeb_wm, cursorb_wm,
1401 plane_sr, cursor_sr);
1402
1403 I915_WRITE(DSPFW1,
1404 (plane_sr << DSPFW_SR_SHIFT) |
1405 (cursorb_wm << DSPFW_CURSORB_SHIFT) |
1406 (planeb_wm << DSPFW_PLANEB_SHIFT) |
1407 planea_wm);
1408 I915_WRITE(DSPFW2,
c0bdd5d9 1409 (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
87f57070
FT
1410 (cursora_wm << DSPFW_CURSORA_SHIFT));
1411 /* HPLL off in SR has some issues on G4x... disable it */
1412 I915_WRITE(DSPFW3,
c0bdd5d9 1413 (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
87f57070
FT
1414 (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1415}
1416
9edbd4a0 1417static void i965_update_wm(struct drm_crtc *unused_crtc)
87f57070 1418{
9edbd4a0 1419 struct drm_device *dev = unused_crtc->dev;
87f57070
FT
1420 struct drm_i915_private *dev_priv = dev->dev_private;
1421 struct drm_crtc *crtc;
1422 int srwm = 1;
1423 int cursor_sr = 16;
1424
1425 /* Calc sr entries for one plane configs */
1426 crtc = single_enabled_crtc(dev);
1427 if (crtc) {
1428 /* self-refresh has much higher latency */
1429 static const int sr_latency_ns = 12000;
9edbd4a0
FT
1430 const struct drm_display_mode *adjusted_mode =
1431 &to_intel_crtc(crtc)->config.adjusted_mode;
1432 int clock = adjusted_mode->crtc_clock;
1433 int htotal = adjusted_mode->crtc_htotal;
1434 int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
87f57070
FT
1435 int pixel_size = crtc->fb->bits_per_pixel / 8;
1436 unsigned long line_time_us;
1437 int entries;
1438
1439 line_time_us = ((htotal * 1000) / clock);
1440
1441 /* Use ns/us then divide to preserve precision */
1442 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1443 pixel_size * hdisplay;
c0bdd5d9 1444 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
87f57070
FT
1445 srwm = I965_FIFO_SIZE - entries;
1446 if (srwm < 0)
1447 srwm = 1;
1448 srwm &= 0x1ff;
1449 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1450 entries, srwm);
1451
1452 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1453 pixel_size * 64;
c0bdd5d9
FT
1454 entries = DIV_ROUND_UP(entries,
1455 i965_cursor_wm_info.cacheline_size);
87f57070
FT
1456 cursor_sr = i965_cursor_wm_info.fifo_size -
1457 (entries + i965_cursor_wm_info.guard_size);
1458
1459 if (cursor_sr > i965_cursor_wm_info.max_wm)
1460 cursor_sr = i965_cursor_wm_info.max_wm;
1461
1462 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1463 "cursor %d\n", srwm, cursor_sr);
1464
1465 if (IS_CRESTLINE(dev))
1466 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
1467 } else {
1468 /* Turn off self refresh if both pipes are enabled */
1469 if (IS_CRESTLINE(dev))
1470 I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
1471 & ~FW_BLC_SELF_EN);
1472 }
1473
1474 DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1475 srwm);
1476
1477 /* 965 has limitations... */
1478 I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
1479 (8 << 16) | (8 << 8) | (8 << 0));
1480 I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
1481 /* update cursor SR watermark */
1482 I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
1483}
1484
9edbd4a0 1485static void i9xx_update_wm(struct drm_crtc *unused_crtc)
87f57070 1486{
9edbd4a0 1487 struct drm_device *dev = unused_crtc->dev;
87f57070
FT
1488 struct drm_i915_private *dev_priv = dev->dev_private;
1489 const struct intel_watermark_params *wm_info;
1490 uint32_t fwater_lo;
1491 uint32_t fwater_hi;
1492 int cwm, srwm = 1;
1493 int fifo_size;
1494 int planea_wm, planeb_wm;
1495 struct drm_crtc *crtc, *enabled = NULL;
1496
1497 if (IS_I945GM(dev))
1498 wm_info = &i945_wm_info;
1499 else if (!IS_GEN2(dev))
1500 wm_info = &i915_wm_info;
1501 else
9edbd4a0 1502 wm_info = &i830_wm_info;
87f57070
FT
1503
1504 fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1505 crtc = intel_get_crtc_for_plane(dev, 0);
c0bdd5d9 1506 if (intel_crtc_active(crtc)) {
9edbd4a0 1507 const struct drm_display_mode *adjusted_mode;
c0bdd5d9
FT
1508 int cpp = crtc->fb->bits_per_pixel / 8;
1509 if (IS_GEN2(dev))
1510 cpp = 4;
1511
9edbd4a0
FT
1512 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1513 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
c0bdd5d9 1514 wm_info, fifo_size, cpp,
87f57070
FT
1515 latency_ns);
1516 enabled = crtc;
1517 } else
1518 planea_wm = fifo_size - wm_info->guard_size;
1519
1520 fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1521 crtc = intel_get_crtc_for_plane(dev, 1);
c0bdd5d9 1522 if (intel_crtc_active(crtc)) {
9edbd4a0 1523 const struct drm_display_mode *adjusted_mode;
c0bdd5d9
FT
1524 int cpp = crtc->fb->bits_per_pixel / 8;
1525 if (IS_GEN2(dev))
1526 cpp = 4;
1527
9edbd4a0
FT
1528 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1529 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
c0bdd5d9 1530 wm_info, fifo_size, cpp,
87f57070
FT
1531 latency_ns);
1532 if (enabled == NULL)
1533 enabled = crtc;
1534 else
1535 enabled = NULL;
1536 } else
1537 planeb_wm = fifo_size - wm_info->guard_size;
1538
1539 DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1540
1541 /*
1542 * Overlay gets an aggressive default since video jitter is bad.
1543 */
1544 cwm = 2;
1545
1546 /* Play safe and disable self-refresh before adjusting watermarks. */
1547 if (IS_I945G(dev) || IS_I945GM(dev))
1548 I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
1549 else if (IS_I915GM(dev))
9edbd4a0 1550 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_SELF_EN));
87f57070
FT
1551
1552 /* Calc sr entries for one plane configs */
1553 if (HAS_FW_BLC(dev) && enabled) {
1554 /* self-refresh has much higher latency */
1555 static const int sr_latency_ns = 6000;
9edbd4a0
FT
1556 const struct drm_display_mode *adjusted_mode =
1557 &to_intel_crtc(enabled)->config.adjusted_mode;
1558 int clock = adjusted_mode->crtc_clock;
1559 int htotal = adjusted_mode->crtc_htotal;
1560 int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w;
87f57070
FT
1561 int pixel_size = enabled->fb->bits_per_pixel / 8;
1562 unsigned long line_time_us;
1563 int entries;
1564
1565 line_time_us = (htotal * 1000) / clock;
1566
1567 /* Use ns/us then divide to preserve precision */
1568 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1569 pixel_size * hdisplay;
c0bdd5d9 1570 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
87f57070
FT
1571 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1572 srwm = wm_info->fifo_size - entries;
1573 if (srwm < 0)
1574 srwm = 1;
1575
1576 if (IS_I945G(dev) || IS_I945GM(dev))
1577 I915_WRITE(FW_BLC_SELF,
1578 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1579 else if (IS_I915GM(dev))
1580 I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1581 }
1582
1583 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1584 planea_wm, planeb_wm, cwm, srwm);
1585
1586 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1587 fwater_hi = (cwm & 0x1f);
1588
1589 /* Set request length to 8 cachelines per fetch */
1590 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1591 fwater_hi = fwater_hi | (1 << 8);
1592
1593 I915_WRITE(FW_BLC, fwater_lo);
1594 I915_WRITE(FW_BLC2, fwater_hi);
1595
1596 if (HAS_FW_BLC(dev)) {
1597 if (enabled) {
1598 if (IS_I945G(dev) || IS_I945GM(dev))
1599 I915_WRITE(FW_BLC_SELF,
1600 FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
1601 else if (IS_I915GM(dev))
9edbd4a0 1602 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_SELF_EN));
87f57070
FT
1603 DRM_DEBUG_KMS("memory self refresh enabled\n");
1604 } else
1605 DRM_DEBUG_KMS("memory self refresh disabled\n");
1606 }
1607}
1608
9edbd4a0 1609static void i845_update_wm(struct drm_crtc *unused_crtc)
87f57070 1610{
9edbd4a0 1611 struct drm_device *dev = unused_crtc->dev;
87f57070
FT
1612 struct drm_i915_private *dev_priv = dev->dev_private;
1613 struct drm_crtc *crtc;
9edbd4a0 1614 const struct drm_display_mode *adjusted_mode;
87f57070
FT
1615 uint32_t fwater_lo;
1616 int planea_wm;
1617
1618 crtc = single_enabled_crtc(dev);
1619 if (crtc == NULL)
1620 return;
1621
9edbd4a0
FT
1622 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
1623 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1624 &i845_wm_info,
87f57070 1625 dev_priv->display.get_fifo_size(dev, 0),
c0bdd5d9 1626 4, latency_ns);
87f57070
FT
1627 fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1628 fwater_lo |= (3<<8) | planea_wm;
1629
1630 DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1631
1632 I915_WRITE(FW_BLC, fwater_lo);
1633}
1634
9edbd4a0
FT
1635static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
1636 struct drm_crtc *crtc)
5d0b1887
FT
1637{
1638 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
9edbd4a0 1639 uint32_t pixel_rate;
5d0b1887 1640
9edbd4a0 1641 pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
5d0b1887
FT
1642
1643 /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
1644 * adjust the pixel_rate here. */
1645
9edbd4a0 1646 if (intel_crtc->config.pch_pfit.enabled) {
5d0b1887 1647 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
9edbd4a0 1648 uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
5d0b1887 1649
9edbd4a0
FT
1650 pipe_w = intel_crtc->config.pipe_src_w;
1651 pipe_h = intel_crtc->config.pipe_src_h;
5d0b1887
FT
1652 pfit_w = (pfit_size >> 16) & 0xFFFF;
1653 pfit_h = pfit_size & 0xFFFF;
1654 if (pipe_w < pfit_w)
1655 pipe_w = pfit_w;
1656 if (pipe_h < pfit_h)
1657 pipe_h = pfit_h;
1658
1659 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1660 pfit_w * pfit_h);
1661 }
1662
1663 return pixel_rate;
1664}
1665
9edbd4a0
FT
1666/* latency must be in 0.1us units. */
1667static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
5d0b1887
FT
1668 uint32_t latency)
1669{
1670 uint64_t ret;
1671
9edbd4a0
FT
1672 if (WARN(latency == 0, "Latency value missing\n"))
1673 return UINT_MAX;
1674
5d0b1887
FT
1675 ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
1676 ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1677
1678 return ret;
1679}
1680
9edbd4a0
FT
1681/* latency must be in 0.1us units. */
1682static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
5d0b1887
FT
1683 uint32_t horiz_pixels, uint8_t bytes_per_pixel,
1684 uint32_t latency)
1685{
1686 uint32_t ret;
1687
9edbd4a0
FT
1688 if (WARN(latency == 0, "Latency value missing\n"))
1689 return UINT_MAX;
1690
5d0b1887
FT
1691 ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1692 ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
1693 ret = DIV_ROUND_UP(ret, 64) + 2;
1694 return ret;
1695}
1696
9edbd4a0 1697static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
5d0b1887
FT
1698 uint8_t bytes_per_pixel)
1699{
1700 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
1701}
1702
9edbd4a0 1703struct ilk_pipe_wm_parameters {
5d0b1887 1704 bool active;
5d0b1887
FT
1705 uint32_t pipe_htotal;
1706 uint32_t pixel_rate;
9edbd4a0
FT
1707 struct intel_plane_wm_parameters pri;
1708 struct intel_plane_wm_parameters spr;
1709 struct intel_plane_wm_parameters cur;
5d0b1887
FT
1710};
1711
9edbd4a0 1712struct ilk_wm_maximums {
5d0b1887
FT
1713 uint16_t pri;
1714 uint16_t spr;
1715 uint16_t cur;
1716 uint16_t fbc;
1717};
1718
9edbd4a0
FT
1719/* used in computing the new watermarks state */
1720struct intel_wm_config {
1721 unsigned int num_pipes_active;
1722 bool sprites_enabled;
1723 bool sprites_scaled;
5d0b1887
FT
1724};
1725
9edbd4a0
FT
1726/*
1727 * For both WM_PIPE and WM_LP.
1728 * mem_value must be in 0.1us units.
1729 */
1730static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
5d0b1887
FT
1731 uint32_t mem_value,
1732 bool is_lp)
1733{
1734 uint32_t method1, method2;
1735
9edbd4a0 1736 if (!params->active || !params->pri.enabled)
5d0b1887
FT
1737 return 0;
1738
9edbd4a0
FT
1739 method1 = ilk_wm_method1(params->pixel_rate,
1740 params->pri.bytes_per_pixel,
5d0b1887
FT
1741 mem_value);
1742
1743 if (!is_lp)
1744 return method1;
1745
9edbd4a0 1746 method2 = ilk_wm_method2(params->pixel_rate,
5d0b1887 1747 params->pipe_htotal,
9edbd4a0
FT
1748 params->pri.horiz_pixels,
1749 params->pri.bytes_per_pixel,
5d0b1887
FT
1750 mem_value);
1751
1752 return min(method1, method2);
1753}
1754
9edbd4a0
FT
1755/*
1756 * For both WM_PIPE and WM_LP.
1757 * mem_value must be in 0.1us units.
1758 */
1759static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
5d0b1887
FT
1760 uint32_t mem_value)
1761{
1762 uint32_t method1, method2;
1763
9edbd4a0 1764 if (!params->active || !params->spr.enabled)
5d0b1887
FT
1765 return 0;
1766
9edbd4a0
FT
1767 method1 = ilk_wm_method1(params->pixel_rate,
1768 params->spr.bytes_per_pixel,
5d0b1887 1769 mem_value);
9edbd4a0 1770 method2 = ilk_wm_method2(params->pixel_rate,
5d0b1887 1771 params->pipe_htotal,
9edbd4a0
FT
1772 params->spr.horiz_pixels,
1773 params->spr.bytes_per_pixel,
5d0b1887
FT
1774 mem_value);
1775 return min(method1, method2);
1776}
1777
9edbd4a0
FT
1778/*
1779 * For both WM_PIPE and WM_LP.
1780 * mem_value must be in 0.1us units.
1781 */
1782static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
5d0b1887
FT
1783 uint32_t mem_value)
1784{
9edbd4a0 1785 if (!params->active || !params->cur.enabled)
5d0b1887
FT
1786 return 0;
1787
9edbd4a0 1788 return ilk_wm_method2(params->pixel_rate,
5d0b1887 1789 params->pipe_htotal,
9edbd4a0
FT
1790 params->cur.horiz_pixels,
1791 params->cur.bytes_per_pixel,
5d0b1887
FT
1792 mem_value);
1793}
1794
1795/* Only for WM_LP. */
9edbd4a0
FT
1796static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
1797 uint32_t pri_val)
5d0b1887 1798{
9edbd4a0 1799 if (!params->active || !params->pri.enabled)
5d0b1887
FT
1800 return 0;
1801
9edbd4a0
FT
1802 return ilk_wm_fbc(pri_val,
1803 params->pri.horiz_pixels,
1804 params->pri.bytes_per_pixel);
5d0b1887
FT
1805}
1806
9edbd4a0 1807static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
5d0b1887 1808{
9edbd4a0
FT
1809 if (INTEL_INFO(dev)->gen >= 8)
1810 return 3072;
1811 else if (INTEL_INFO(dev)->gen >= 7)
1812 return 768;
1813 else
1814 return 512;
1815}
5d0b1887 1816
9edbd4a0
FT
1817/* Calculate the maximum primary/sprite plane watermark */
1818static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1819 int level,
1820 const struct intel_wm_config *config,
1821 enum intel_ddb_partitioning ddb_partitioning,
1822 bool is_sprite)
1823{
1824 unsigned int fifo_size = ilk_display_fifo_size(dev);
1825 unsigned int max;
5d0b1887 1826
9edbd4a0
FT
1827 /* if sprites aren't enabled, sprites get nothing */
1828 if (is_sprite && !config->sprites_enabled)
1829 return 0;
5d0b1887 1830
9edbd4a0
FT
1831 /* HSW allows LP1+ watermarks even with multiple pipes */
1832 if (level == 0 || config->num_pipes_active > 1) {
1833 fifo_size /= INTEL_INFO(dev)->num_pipes;
5d0b1887 1834
9edbd4a0
FT
1835 /*
1836 * For some reason the non self refresh
1837 * FIFO size is only half of the self
1838 * refresh FIFO size on ILK/SNB.
1839 */
1840 if (INTEL_INFO(dev)->gen <= 6)
1841 fifo_size /= 2;
5d0b1887
FT
1842 }
1843
9edbd4a0
FT
1844 if (config->sprites_enabled) {
1845 /* level 0 is always calculated with 1:1 split */
1846 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1847 if (is_sprite)
1848 fifo_size *= 5;
1849 fifo_size /= 6;
1850 } else {
1851 fifo_size /= 2;
1852 }
1853 }
1854
1855 /* clamp to max that the registers can hold */
1856 if (INTEL_INFO(dev)->gen >= 8)
1857 max = level == 0 ? 255 : 2047;
1858 else if (INTEL_INFO(dev)->gen >= 7)
1859 /* IVB/HSW primary/sprite plane watermarks */
1860 max = level == 0 ? 127 : 1023;
1861 else if (!is_sprite)
1862 /* ILK/SNB primary plane watermarks */
1863 max = level == 0 ? 127 : 511;
1864 else
1865 /* ILK/SNB sprite plane watermarks */
1866 max = level == 0 ? 63 : 255;
1867
1868 return min(fifo_size, max);
5d0b1887
FT
1869}
1870
9edbd4a0
FT
1871/* Calculate the maximum cursor plane watermark */
1872static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
1873 int level,
1874 const struct intel_wm_config *config)
5d0b1887 1875{
9edbd4a0
FT
1876 /* HSW LP1+ watermarks w/ multiple pipes */
1877 if (level > 0 && config->num_pipes_active > 1)
1878 return 64;
5d0b1887 1879
9edbd4a0
FT
1880 /* otherwise just report max that registers can hold */
1881 if (INTEL_INFO(dev)->gen >= 7)
1882 return level == 0 ? 63 : 255;
1883 else
1884 return level == 0 ? 31 : 63;
1885}
5d0b1887 1886
9edbd4a0
FT
1887/* Calculate the maximum FBC watermark */
1888static unsigned int ilk_fbc_wm_max(struct drm_device *dev)
1889{
1890 /* max that registers can hold */
1891 if (INTEL_INFO(dev)->gen >= 8)
1892 return 31;
1893 else
1894 return 15;
1895}
5d0b1887 1896
9edbd4a0
FT
1897static void ilk_compute_wm_maximums(struct drm_device *dev,
1898 int level,
1899 const struct intel_wm_config *config,
1900 enum intel_ddb_partitioning ddb_partitioning,
1901 struct ilk_wm_maximums *max)
1902{
1903 max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1904 max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1905 max->cur = ilk_cursor_wm_max(dev, level, config);
1906 max->fbc = ilk_fbc_wm_max(dev);
1907}
1908
1909static bool ilk_validate_wm_level(int level,
1910 const struct ilk_wm_maximums *max,
1911 struct intel_wm_level *result)
1912{
1913 bool ret;
1914
1915 /* already determined to be invalid? */
1916 if (!result->enable)
1917 return false;
1918
1919 result->enable = result->pri_val <= max->pri &&
1920 result->spr_val <= max->spr &&
1921 result->cur_val <= max->cur;
1922
1923 ret = result->enable;
1924
1925 /*
1926 * HACK until we can pre-compute everything,
1927 * and thus fail gracefully if LP0 watermarks
1928 * are exceeded...
1929 */
1930 if (level == 0 && !result->enable) {
1931 if (result->pri_val > max->pri)
1932 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
1933 level, result->pri_val, max->pri);
1934 if (result->spr_val > max->spr)
1935 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
1936 level, result->spr_val, max->spr);
1937 if (result->cur_val > max->cur)
1938 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
1939 level, result->cur_val, max->cur);
1940
1941 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
1942 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
1943 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
1944 result->enable = true;
1945 }
1946
1947 return ret;
1948}
1949
1950static void ilk_compute_wm_level(struct drm_i915_private *dev_priv,
1951 int level,
1952 const struct ilk_pipe_wm_parameters *p,
1953 struct intel_wm_level *result)
1954{
1955 uint16_t pri_latency = dev_priv->wm.pri_latency[level];
1956 uint16_t spr_latency = dev_priv->wm.spr_latency[level];
1957 uint16_t cur_latency = dev_priv->wm.cur_latency[level];
1958
1959 /* WM1+ latency values stored in 0.5us units */
1960 if (level > 0) {
1961 pri_latency *= 5;
1962 spr_latency *= 5;
1963 cur_latency *= 5;
1964 }
1965
1966 result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
1967 result->spr_val = ilk_compute_spr_wm(p, spr_latency);
1968 result->cur_val = ilk_compute_cur_wm(p, cur_latency);
1969 result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
1970 result->enable = true;
5d0b1887
FT
1971}
1972
1973static uint32_t
1974hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
0db36e9f
FT
1975{
1976 struct drm_i915_private *dev_priv = dev->dev_private;
5d0b1887
FT
1977 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1978 struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
1979 u32 linetime, ips_linetime;
0db36e9f 1980
5d0b1887
FT
1981 if (!intel_crtc_active(crtc))
1982 return 0;
0db36e9f
FT
1983
1984 /* The WM are computed with base on how long it takes to fill a single
1985 * row at the given clock rate, multiplied by 8.
1986 * */
9edbd4a0
FT
1987 linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
1988 mode->crtc_clock);
1989 ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
5d0b1887 1990 intel_ddi_get_cdclk_freq(dev_priv));
0db36e9f 1991
5d0b1887
FT
1992 return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
1993 PIPE_WM_LINETIME_TIME(linetime);
1994}
1995
9edbd4a0 1996static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
5d0b1887
FT
1997{
1998 struct drm_i915_private *dev_priv = dev->dev_private;
5d0b1887 1999
9edbd4a0
FT
2000 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2001 uint64_t sskpd = I915_READ64(MCH_SSKPD);
2002
5d0b1887 2003 wm[0] = (sskpd >> 56) & 0xFF;
9edbd4a0
FT
2004 if (wm[0] == 0)
2005 wm[0] = sskpd & 0xF;
2006 wm[1] = (sskpd >> 4) & 0xFF;
2007 wm[2] = (sskpd >> 12) & 0xFF;
2008 wm[3] = (sskpd >> 20) & 0x1FF;
2009 wm[4] = (sskpd >> 32) & 0x1FF;
2010 } else if (INTEL_INFO(dev)->gen >= 6) {
2011 uint32_t sskpd = I915_READ(MCH_SSKPD);
2012
2013 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
2014 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
2015 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
2016 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
2017 } else if (INTEL_INFO(dev)->gen >= 5) {
2018 uint32_t mltr = I915_READ(MLTR_ILK);
2019
2020 /* ILK primary LP0 latency is 700 ns */
2021 wm[0] = 7;
2022 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
2023 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
2024 }
2025}
2026
2027static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
2028{
2029 /* ILK sprite LP0 latency is 1300 ns */
2030 if (INTEL_INFO(dev)->gen == 5)
2031 wm[0] = 13;
2032}
2033
2034static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
2035{
2036 /* ILK cursor LP0 latency is 1300 ns */
2037 if (INTEL_INFO(dev)->gen == 5)
2038 wm[0] = 13;
2039
2040 /* WaDoubleCursorLP3Latency:ivb */
2041 if (IS_IVYBRIDGE(dev))
2042 wm[3] *= 2;
2043}
2044
2045static int ilk_wm_max_level(const struct drm_device *dev)
2046{
2047 /* how many WM levels are we expecting */
2048 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2049 return 4;
2050 else if (INTEL_INFO(dev)->gen >= 6)
2051 return 3;
5d0b1887 2052 else
9edbd4a0
FT
2053 return 2;
2054}
5d0b1887 2055
9edbd4a0
FT
2056static void intel_print_wm_latency(struct drm_device *dev,
2057 const char *name,
2058 const uint16_t wm[5])
2059{
2060 int level, max_level = ilk_wm_max_level(dev);
5d0b1887 2061
9edbd4a0
FT
2062 for (level = 0; level <= max_level; level++) {
2063 unsigned int latency = wm[level];
5d0b1887 2064
9edbd4a0
FT
2065 if (latency == 0) {
2066 DRM_ERROR("%s WM%d latency not provided\n",
2067 name, level);
5d0b1887 2068 continue;
9edbd4a0 2069 }
5d0b1887 2070
9edbd4a0
FT
2071 /* WM1+ latency values in 0.5us units */
2072 if (level > 0)
2073 latency *= 5;
5d0b1887 2074
9edbd4a0
FT
2075 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
2076 name, level, wm[level],
2077 latency / 10, latency % 10);
5d0b1887 2078 }
9edbd4a0 2079}
5d0b1887 2080
9edbd4a0
FT
2081static void intel_setup_wm_latency(struct drm_device *dev)
2082{
2083 struct drm_i915_private *dev_priv = dev->dev_private;
2084
2085 intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
5d0b1887 2086
9edbd4a0
FT
2087 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2088 sizeof(dev_priv->wm.pri_latency));
2089 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2090 sizeof(dev_priv->wm.pri_latency));
5d0b1887 2091
9edbd4a0
FT
2092 intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
2093 intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
5d0b1887 2094
9edbd4a0
FT
2095 intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2096 intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2097 intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2098}
2099
2100static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
2101 struct ilk_pipe_wm_parameters *p,
2102 struct intel_wm_config *config)
2103{
2104 struct drm_device *dev = crtc->dev;
2105 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2106 enum i915_pipe pipe = intel_crtc->pipe;
2107 struct drm_plane *plane;
2108
2109 p->active = intel_crtc_active(crtc);
2110 if (p->active) {
2111 p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
2112 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
2113 p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8;
2114 p->cur.bytes_per_pixel = 4;
2115 p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
2116 p->cur.horiz_pixels = 64;
2117 /* TODO: for now, assume primary and cursor planes are always enabled. */
2118 p->pri.enabled = true;
2119 p->cur.enabled = true;
5d0b1887
FT
2120 }
2121
9edbd4a0
FT
2122 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
2123 config->num_pipes_active += intel_crtc_active(crtc);
2124
2125 list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2126 struct intel_plane *intel_plane = to_intel_plane(plane);
2127
2128 if (intel_plane->pipe == pipe)
2129 p->spr = intel_plane->wm;
2130
2131 config->sprites_enabled |= intel_plane->wm.enabled;
2132 config->sprites_scaled |= intel_plane->wm.scaled;
5d0b1887 2133 }
5d0b1887
FT
2134}
2135
9edbd4a0
FT
2136/* Compute new watermarks for the pipe */
2137static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
2138 const struct ilk_pipe_wm_parameters *params,
2139 struct intel_pipe_wm *pipe_wm)
5d0b1887 2140{
9edbd4a0 2141 struct drm_device *dev = crtc->dev;
5d0b1887 2142 struct drm_i915_private *dev_priv = dev->dev_private;
9edbd4a0
FT
2143 int level, max_level = ilk_wm_max_level(dev);
2144 /* LP0 watermark maximums depend on this pipe alone */
2145 struct intel_wm_config config = {
2146 .num_pipes_active = 1,
2147 .sprites_enabled = params->spr.enabled,
2148 .sprites_scaled = params->spr.scaled,
2149 };
2150 struct ilk_wm_maximums max;
5d0b1887 2151
9edbd4a0
FT
2152 /* LP0 watermarks always use 1/2 DDB partitioning */
2153 ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2154
2155 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2156 if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
2157 max_level = 1;
2158
2159 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2160 if (params->spr.scaled)
2161 max_level = 0;
2162
2163 for (level = 0; level <= max_level; level++)
2164 ilk_compute_wm_level(dev_priv, level, params,
2165 &pipe_wm->wm[level]);
5d0b1887 2166
9edbd4a0
FT
2167 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2168 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
2169
2170 /* At least LP0 must be valid */
2171 return ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]);
2172}
2173
2174/*
2175 * Merge the watermarks from all active pipes for a specific level.
2176 */
2177static void ilk_merge_wm_level(struct drm_device *dev,
2178 int level,
2179 struct intel_wm_level *ret_wm)
2180{
2181 struct intel_crtc *intel_crtc;
2182
2183 list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
2184 const struct intel_wm_level *wm =
2185 &intel_crtc->wm.active.wm[level];
2186
2187 if (!wm->enable)
2188 return;
2189
2190 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2191 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2192 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2193 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2194 }
2195
2196 ret_wm->enable = true;
2197}
2198
2199/*
2200 * Merge all low power watermarks for all active pipes.
2201 */
2202static void ilk_wm_merge(struct drm_device *dev,
2203 const struct intel_wm_config *config,
2204 const struct ilk_wm_maximums *max,
2205 struct intel_pipe_wm *merged)
2206{
2207 int level, max_level = ilk_wm_max_level(dev);
2208
2209 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2210 if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2211 config->num_pipes_active > 1)
2212 return;
2213
2214 /* ILK: FBC WM must be disabled always */
2215 merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
2216
2217 /* merge each WM1+ level */
5d0b1887 2218 for (level = 1; level <= max_level; level++) {
9edbd4a0
FT
2219 struct intel_wm_level *wm = &merged->wm[level];
2220
2221 ilk_merge_wm_level(dev, level, wm);
2222
2223 if (!ilk_validate_wm_level(level, max, wm))
5d0b1887 2224 break;
9edbd4a0
FT
2225
2226 /*
2227 * The spec says it is preferred to disable
2228 * FBC WMs instead of disabling a WM level.
2229 */
2230 if (wm->fbc_val > max->fbc) {
2231 merged->fbc_wm_enabled = false;
2232 wm->fbc_val = 0;
5d0b1887
FT
2233 }
2234 }
2235
9edbd4a0
FT
2236 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2237 /*
2238 * FIXME this is racy. FBC might get enabled later.
2239 * What we should check here is whether FBC can be
2240 * enabled sometime later.
2241 */
2242 if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
2243 for (level = 2; level <= max_level; level++) {
2244 struct intel_wm_level *wm = &merged->wm[level];
2245
2246 wm->enable = false;
2247 }
2248 }
2249}
2250
2251static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2252{
2253 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2254 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2255}
2256
2257/* The value we need to program into the WM_LPx latency field */
2258static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2259{
2260 struct drm_i915_private *dev_priv = dev->dev_private;
2261
2262 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2263 return 2 * level;
2264 else
2265 return dev_priv->wm.pri_latency[level];
2266}
2267
2268static void ilk_compute_wm_results(struct drm_device *dev,
2269 const struct intel_pipe_wm *merged,
2270 enum intel_ddb_partitioning partitioning,
2271 struct ilk_wm_values *results)
2272{
2273 struct intel_crtc *intel_crtc;
2274 int level, wm_lp;
2275
2276 results->enable_fbc_wm = merged->fbc_wm_enabled;
2277 results->partitioning = partitioning;
2278
2279 /* LP1+ register values */
5d0b1887 2280 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
9edbd4a0 2281 const struct intel_wm_level *r;
5d0b1887 2282
9edbd4a0
FT
2283 level = ilk_wm_lp_to_level(wm_lp, merged);
2284
2285 r = &merged->wm[level];
2286 if (!r->enable)
5d0b1887
FT
2287 break;
2288
9edbd4a0
FT
2289 results->wm_lp[wm_lp - 1] = WM3_LP_EN |
2290 (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
2291 (r->pri_val << WM1_LP_SR_SHIFT) |
2292 r->cur_val;
2293
2294 if (INTEL_INFO(dev)->gen >= 8)
2295 results->wm_lp[wm_lp - 1] |=
2296 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2297 else
2298 results->wm_lp[wm_lp - 1] |=
2299 r->fbc_val << WM1_LP_FBC_SHIFT;
2300
2301 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2302 WARN_ON(wm_lp != 1);
2303 results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2304 } else
2305 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
5d0b1887
FT
2306 }
2307
9edbd4a0
FT
2308 /* LP0 register values */
2309 list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
2310 enum i915_pipe pipe = intel_crtc->pipe;
2311 const struct intel_wm_level *r =
2312 &intel_crtc->wm.active.wm[0];
5d0b1887 2313
9edbd4a0
FT
2314 if (WARN_ON(!r->enable))
2315 continue;
2316
2317 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
2318
2319 results->wm_pipe[pipe] =
2320 (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2321 (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2322 r->cur_val;
5d0b1887
FT
2323 }
2324}
2325
2326/* Find the result with the highest level enabled. Check for enable_fbc_wm in
2327 * case both are at the same level. Prefer r1 in case they're the same. */
9edbd4a0
FT
2328static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
2329 struct intel_pipe_wm *r1,
2330 struct intel_pipe_wm *r2)
5d0b1887 2331{
9edbd4a0
FT
2332 int level, max_level = ilk_wm_max_level(dev);
2333 int level1 = 0, level2 = 0;
5d0b1887 2334
9edbd4a0
FT
2335 for (level = 1; level <= max_level; level++) {
2336 if (r1->wm[level].enable)
2337 level1 = level;
2338 if (r2->wm[level].enable)
2339 level2 = level;
5d0b1887
FT
2340 }
2341
9edbd4a0
FT
2342 if (level1 == level2) {
2343 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
5d0b1887
FT
2344 return r2;
2345 else
2346 return r1;
9edbd4a0 2347 } else if (level1 > level2) {
5d0b1887
FT
2348 return r1;
2349 } else {
2350 return r2;
2351 }
2352}
2353
9edbd4a0
FT
2354/* dirty bits used to track which watermarks need changes */
2355#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2356#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2357#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2358#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2359#define WM_DIRTY_FBC (1 << 24)
2360#define WM_DIRTY_DDB (1 << 25)
2361
2362static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
2363 const struct ilk_wm_values *old,
2364 const struct ilk_wm_values *new)
2365{
2366 unsigned int dirty = 0;
2367 enum i915_pipe pipe;
2368 int wm_lp;
2369
2370 for_each_pipe(pipe) {
2371 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2372 dirty |= WM_DIRTY_LINETIME(pipe);
2373 /* Must disable LP1+ watermarks too */
2374 dirty |= WM_DIRTY_LP_ALL;
2375 }
2376
2377 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2378 dirty |= WM_DIRTY_PIPE(pipe);
2379 /* Must disable LP1+ watermarks too */
2380 dirty |= WM_DIRTY_LP_ALL;
2381 }
2382 }
2383
2384 if (old->enable_fbc_wm != new->enable_fbc_wm) {
2385 dirty |= WM_DIRTY_FBC;
2386 /* Must disable LP1+ watermarks too */
2387 dirty |= WM_DIRTY_LP_ALL;
2388 }
2389
2390 if (old->partitioning != new->partitioning) {
2391 dirty |= WM_DIRTY_DDB;
2392 /* Must disable LP1+ watermarks too */
2393 dirty |= WM_DIRTY_LP_ALL;
2394 }
2395
2396 /* LP1+ watermarks already deemed dirty, no need to continue */
2397 if (dirty & WM_DIRTY_LP_ALL)
2398 return dirty;
2399
2400 /* Find the lowest numbered LP1+ watermark in need of an update... */
2401 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2402 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2403 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2404 break;
2405 }
2406
2407 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2408 for (; wm_lp <= 3; wm_lp++)
2409 dirty |= WM_DIRTY_LP(wm_lp);
2410
2411 return dirty;
2412}
2413
2414static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2415 unsigned int dirty)
2416{
2417 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2418 bool changed = false;
2419
2420 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2421 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2422 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
2423 changed = true;
2424 }
2425 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2426 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2427 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
2428 changed = true;
2429 }
2430 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2431 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2432 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
2433 changed = true;
2434 }
2435
2436 /*
2437 * Don't touch WM1S_LP_EN here.
2438 * Doing so could cause underruns.
2439 */
2440
2441 return changed;
2442}
2443
5d0b1887
FT
2444/*
2445 * The spec says we shouldn't write when we don't need, because every write
2446 * causes WMs to be re-evaluated, expending some power.
2447 */
9edbd4a0
FT
2448static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2449 struct ilk_wm_values *results)
5d0b1887 2450{
9edbd4a0
FT
2451 struct drm_device *dev = dev_priv->dev;
2452 struct ilk_wm_values *previous = &dev_priv->wm.hw;
2453 unsigned int dirty;
5d0b1887 2454 uint32_t val;
9edbd4a0
FT
2455
2456 dirty = ilk_compute_wm_dirty(dev, previous, results);
2457 if (!dirty)
5d0b1887
FT
2458 return;
2459
9edbd4a0 2460 _ilk_disable_lp_wm(dev_priv, dirty);
5d0b1887 2461
9edbd4a0 2462 if (dirty & WM_DIRTY_PIPE(PIPE_A))
5d0b1887 2463 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
9edbd4a0 2464 if (dirty & WM_DIRTY_PIPE(PIPE_B))
5d0b1887 2465 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
9edbd4a0 2466 if (dirty & WM_DIRTY_PIPE(PIPE_C))
5d0b1887
FT
2467 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2468
9edbd4a0 2469 if (dirty & WM_DIRTY_LINETIME(PIPE_A))
5d0b1887 2470 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
9edbd4a0 2471 if (dirty & WM_DIRTY_LINETIME(PIPE_B))
5d0b1887 2472 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
9edbd4a0 2473 if (dirty & WM_DIRTY_LINETIME(PIPE_C))
5d0b1887
FT
2474 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2475
9edbd4a0
FT
2476 if (dirty & WM_DIRTY_DDB) {
2477 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2478 val = I915_READ(WM_MISC);
2479 if (results->partitioning == INTEL_DDB_PART_1_2)
2480 val &= ~WM_MISC_DATA_PARTITION_5_6;
2481 else
2482 val |= WM_MISC_DATA_PARTITION_5_6;
2483 I915_WRITE(WM_MISC, val);
2484 } else {
2485 val = I915_READ(DISP_ARB_CTL2);
2486 if (results->partitioning == INTEL_DDB_PART_1_2)
2487 val &= ~DISP_DATA_PARTITION_5_6;
2488 else
2489 val |= DISP_DATA_PARTITION_5_6;
2490 I915_WRITE(DISP_ARB_CTL2, val);
2491 }
5d0b1887
FT
2492 }
2493
9edbd4a0 2494 if (dirty & WM_DIRTY_FBC) {
5d0b1887
FT
2495 val = I915_READ(DISP_ARB_CTL);
2496 if (results->enable_fbc_wm)
2497 val &= ~DISP_FBC_WM_DIS;
2498 else
2499 val |= DISP_FBC_WM_DIS;
2500 I915_WRITE(DISP_ARB_CTL, val);
2501 }
2502
9edbd4a0
FT
2503 if (dirty & WM_DIRTY_LP(1) &&
2504 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
5d0b1887 2505 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
5d0b1887 2506
9edbd4a0
FT
2507 if (INTEL_INFO(dev)->gen >= 7) {
2508 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2509 I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2510 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2511 I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2512 }
2513
2514 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
5d0b1887 2515 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
9edbd4a0 2516 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
5d0b1887 2517 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
9edbd4a0 2518 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
5d0b1887 2519 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
9edbd4a0
FT
2520
2521 dev_priv->wm.hw = *results;
5d0b1887
FT
2522}
2523
9edbd4a0 2524static bool ilk_disable_lp_wm(struct drm_device *dev)
5d0b1887
FT
2525{
2526 struct drm_i915_private *dev_priv = dev->dev_private;
5d0b1887 2527
9edbd4a0 2528 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
5d0b1887
FT
2529}
2530
9edbd4a0 2531static void ilk_update_wm(struct drm_crtc *crtc)
5d0b1887 2532{
9edbd4a0
FT
2533 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2534 struct drm_device *dev = crtc->dev;
2535 struct drm_i915_private *dev_priv = dev->dev_private;
2536 struct ilk_wm_maximums max;
2537 struct ilk_pipe_wm_parameters params = {};
2538 struct ilk_wm_values results = {};
2539 enum intel_ddb_partitioning partitioning;
2540 struct intel_pipe_wm pipe_wm = {};
2541 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
2542 struct intel_wm_config config = {};
5d0b1887 2543
9edbd4a0 2544 ilk_compute_wm_parameters(crtc, &params, &config);
5d0b1887 2545
9edbd4a0 2546 intel_compute_pipe_wm(crtc, &params, &pipe_wm);
0db36e9f 2547
9edbd4a0
FT
2548 if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
2549 return;
0db36e9f 2550
9edbd4a0 2551 intel_crtc->wm.active = pipe_wm;
87f57070 2552
9edbd4a0
FT
2553 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
2554 ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
2555
2556 /* 5/6 split only in single pipe config on IVB+ */
2557 if (INTEL_INFO(dev)->gen >= 7 &&
2558 config.num_pipes_active == 1 && config.sprites_enabled) {
2559 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
2560 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
2561
2562 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
2563 } else {
2564 best_lp_wm = &lp_wm_1_2;
87f57070
FT
2565 }
2566
9edbd4a0
FT
2567 partitioning = (best_lp_wm == &lp_wm_1_2) ?
2568 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
87f57070 2569
9edbd4a0 2570 ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
87f57070 2571
9edbd4a0 2572 ilk_write_wm_values(dev_priv, &results);
87f57070
FT
2573}
2574
9edbd4a0
FT
2575static void ilk_update_sprite_wm(struct drm_plane *plane,
2576 struct drm_crtc *crtc,
2577 uint32_t sprite_width, int pixel_size,
2578 bool enabled, bool scaled)
87f57070 2579{
9edbd4a0
FT
2580 struct drm_device *dev = plane->dev;
2581 struct intel_plane *intel_plane = to_intel_plane(plane);
87f57070 2582
9edbd4a0
FT
2583 intel_plane->wm.enabled = enabled;
2584 intel_plane->wm.scaled = scaled;
2585 intel_plane->wm.horiz_pixels = sprite_width;
2586 intel_plane->wm.bytes_per_pixel = pixel_size;
87f57070 2587
9edbd4a0
FT
2588 /*
2589 * IVB workaround: must disable low power watermarks for at least
2590 * one frame before enabling scaling. LP watermarks can be re-enabled
2591 * when scaling is disabled.
2592 *
2593 * WaCxSRDisabledForSpriteScaling:ivb
2594 */
2595 if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
2596 intel_wait_for_vblank(dev, intel_plane->pipe);
87f57070 2597
9edbd4a0 2598 ilk_update_wm(crtc);
87f57070
FT
2599}
2600
9edbd4a0 2601static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
87f57070 2602{
9edbd4a0 2603 struct drm_device *dev = crtc->dev;
87f57070 2604 struct drm_i915_private *dev_priv = dev->dev_private;
9edbd4a0
FT
2605 struct ilk_wm_values *hw = &dev_priv->wm.hw;
2606 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2607 struct intel_pipe_wm *active = &intel_crtc->wm.active;
2608 enum i915_pipe pipe = intel_crtc->pipe;
2609 static const unsigned int wm0_pipe_reg[] = {
2610 [PIPE_A] = WM0_PIPEA_ILK,
2611 [PIPE_B] = WM0_PIPEB_ILK,
2612 [PIPE_C] = WM0_PIPEC_IVB,
2613 };
87f57070 2614
9edbd4a0
FT
2615 hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
2616 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2617 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
5d0b1887 2618
9edbd4a0
FT
2619 if (intel_crtc_active(crtc)) {
2620 u32 tmp = hw->wm_pipe[pipe];
87f57070 2621
9edbd4a0
FT
2622 /*
2623 * For active pipes LP0 watermark is marked as
2624 * enabled, and LP1+ watermaks as disabled since
2625 * we can't really reverse compute them in case
2626 * multiple pipes are active.
2627 */
2628 active->wm[0].enable = true;
2629 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
2630 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
2631 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
2632 active->linetime = hw->wm_linetime[pipe];
2633 } else {
2634 int level, max_level = ilk_wm_max_level(dev);
2635
2636 /*
2637 * For inactive pipes, all watermark levels
2638 * should be marked as enabled but zeroed,
2639 * which is what we'd compute them to.
2640 */
2641 for (level = 0; level <= max_level; level++)
2642 active->wm[level].enable = true;
87f57070 2643 }
9edbd4a0 2644}
87f57070 2645
9edbd4a0
FT
2646void ilk_wm_get_hw_state(struct drm_device *dev)
2647{
2648 struct drm_i915_private *dev_priv = dev->dev_private;
2649 struct ilk_wm_values *hw = &dev_priv->wm.hw;
2650 struct drm_crtc *crtc;
87f57070 2651
9edbd4a0
FT
2652 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
2653 ilk_pipe_wm_get_hw_state(crtc);
87f57070 2654
9edbd4a0
FT
2655 hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
2656 hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
2657 hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
87f57070 2658
9edbd4a0
FT
2659 hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
2660 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
2661 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
87f57070 2662
9edbd4a0
FT
2663 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2664 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
2665 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
2666 else if (IS_IVYBRIDGE(dev))
2667 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
2668 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
87f57070 2669
9edbd4a0
FT
2670 hw->enable_fbc_wm =
2671 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
87f57070
FT
2672}
2673
2674/**
2675 * intel_update_watermarks - update FIFO watermark values based on current modes
2676 *
2677 * Calculate watermark values for the various WM regs based on current mode
2678 * and plane configuration.
2679 *
2680 * There are several cases to deal with here:
2681 * - normal (i.e. non-self-refresh)
2682 * - self-refresh (SR) mode
2683 * - lines are large relative to FIFO size (buffer can hold up to 2)
2684 * - lines are small relative to FIFO size (buffer can hold more than 2
2685 * lines), so need to account for TLB latency
2686 *
2687 * The normal calculation is:
2688 * watermark = dotclock * bytes per pixel * latency
2689 * where latency is platform & configuration dependent (we assume pessimal
2690 * values here).
2691 *
2692 * The SR calculation is:
2693 * watermark = (trunc(latency/line time)+1) * surface width *
2694 * bytes per pixel
2695 * where
2696 * line time = htotal / dotclock
2697 * surface width = hdisplay for normal plane and 64 for cursor
2698 * and latency is assumed to be high, as above.
2699 *
2700 * The final value programmed to the register should always be rounded up,
2701 * and include an extra 2 entries to account for clock crossings.
2702 *
2703 * We don't use the sprite, so we can ignore that. And on Crestline we have
2704 * to set the non-SR watermarks to 8.
2705 */
9edbd4a0 2706void intel_update_watermarks(struct drm_crtc *crtc)
87f57070 2707{
9edbd4a0 2708 struct drm_i915_private *dev_priv = crtc->dev->dev_private;
87f57070
FT
2709
2710 if (dev_priv->display.update_wm)
9edbd4a0 2711 dev_priv->display.update_wm(crtc);
87f57070
FT
2712}
2713
9edbd4a0
FT
2714void intel_update_sprite_watermarks(struct drm_plane *plane,
2715 struct drm_crtc *crtc,
5d0b1887 2716 uint32_t sprite_width, int pixel_size,
9edbd4a0 2717 bool enabled, bool scaled)
87f57070 2718{
9edbd4a0 2719 struct drm_i915_private *dev_priv = plane->dev->dev_private;
87f57070
FT
2720
2721 if (dev_priv->display.update_sprite_wm)
9edbd4a0
FT
2722 dev_priv->display.update_sprite_wm(plane, crtc, sprite_width,
2723 pixel_size, enabled, scaled);
87f57070
FT
2724}
2725
2726static struct drm_i915_gem_object *
2727intel_alloc_context_page(struct drm_device *dev)
2728{
2729 struct drm_i915_gem_object *ctx;
2730 int ret;
2731
a2fdbec6 2732 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
87f57070
FT
2733
2734 ctx = i915_gem_alloc_object(dev, 4096);
2735 if (!ctx) {
2736 DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
2737 return NULL;
2738 }
2739
9edbd4a0 2740 ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false);
87f57070
FT
2741 if (ret) {
2742 DRM_ERROR("failed to pin power context: %d\n", ret);
2743 goto err_unref;
2744 }
2745
2746 ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
2747 if (ret) {
2748 DRM_ERROR("failed to set-domain on power context: %d\n", ret);
2749 goto err_unpin;
2750 }
2751
2752 return ctx;
2753
2754err_unpin:
2755 i915_gem_object_unpin(ctx);
2756err_unref:
2757 drm_gem_object_unreference(&ctx->base);
87f57070
FT
2758 return NULL;
2759}
2760
bc9b2a29
FT
2761/**
2762 * Lock protecting IPS related data structures
2763 */
2764struct lock mchdev_lock;
2765LOCK_SYSINIT(mchdev, &mchdev_lock, "mchdev", LK_CANRECURSE);
2766
2767/* Global for IPS driver to get at the current i915 device. Protected by
2768 * mchdev_lock. */
a2fdbec6 2769static struct drm_i915_private *i915_mch_dev;
bc9b2a29 2770
87f57070
FT
2771bool ironlake_set_drps(struct drm_device *dev, u8 val)
2772{
2773 struct drm_i915_private *dev_priv = dev->dev_private;
2774 u16 rgvswctl;
2775
9edbd4a0
FT
2776 assert_spin_locked(&mchdev_lock);
2777
87f57070
FT
2778 rgvswctl = I915_READ16(MEMSWCTL);
2779 if (rgvswctl & MEMCTL_CMD_STS) {
2780 DRM_DEBUG("gpu busy, RCS change rejected\n");
2781 return false; /* still busy with another command */
2782 }
2783
2784 rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
2785 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
2786 I915_WRITE16(MEMSWCTL, rgvswctl);
2787 POSTING_READ16(MEMSWCTL);
2788
2789 rgvswctl |= MEMCTL_CMD_STS;
2790 I915_WRITE16(MEMSWCTL, rgvswctl);
2791
2792 return true;
2793}
2794
c0bdd5d9 2795static void ironlake_enable_drps(struct drm_device *dev)
87f57070
FT
2796{
2797 struct drm_i915_private *dev_priv = dev->dev_private;
2798 u32 rgvmodectl = I915_READ(MEMMODECTL);
2799 u8 fmax, fmin, fstart, vstart;
2800
c0bdd5d9
FT
2801 lockmgr(&mchdev_lock, LK_EXCLUSIVE);
2802
87f57070
FT
2803 /* Enable temp reporting */
2804 I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
2805 I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
2806
2807 /* 100ms RC evaluation intervals */
2808 I915_WRITE(RCUPEI, 100000);
2809 I915_WRITE(RCDNEI, 100000);
2810
2811 /* Set max/min thresholds to 90ms and 80ms respectively */
2812 I915_WRITE(RCBMAXAVG, 90000);
2813 I915_WRITE(RCBMINAVG, 80000);
2814
2815 I915_WRITE(MEMIHYST, 1);
2816
2817 /* Set up min, max, and cur for interrupt handling */
2818 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
2819 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
2820 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
2821 MEMMODE_FSTART_SHIFT;
2822
2823 vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
2824 PXVFREQ_PX_SHIFT;
2825
e77a37f5
FT
2826 dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
2827 dev_priv->ips.fstart = fstart;
87f57070 2828
c0bdd5d9
FT
2829 dev_priv->ips.max_delay = fstart;
2830 dev_priv->ips.min_delay = fmin;
2831 dev_priv->ips.cur_delay = fstart;
87f57070 2832
c0bdd5d9 2833 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
87f57070
FT
2834 fmax, fmin, fstart);
2835
2836 I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
2837
2838 /*
2839 * Interrupts will be enabled in ironlake_irq_postinstall
2840 */
2841
2842 I915_WRITE(VIDSTART, vstart);
2843 POSTING_READ(VIDSTART);
2844
2845 rgvmodectl |= MEMMODE_SWMODE_EN;
2846 I915_WRITE(MEMMODECTL, rgvmodectl);
2847
c0bdd5d9 2848 if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
87f57070 2849 DRM_ERROR("stuck trying to change perf mode\n");
a2fdbec6 2850 mdelay(1);
87f57070
FT
2851
2852 ironlake_set_drps(dev, fstart);
2853
e77a37f5 2854 dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
87f57070 2855 I915_READ(0x112e0);
e77a37f5
FT
2856 dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
2857 dev_priv->ips.last_count2 = I915_READ(0x112f4);
a2fdbec6 2858 getrawmonotonic(&dev_priv->ips.last_time2);
c0bdd5d9
FT
2859
2860 lockmgr(&mchdev_lock, LK_RELEASE);
87f57070
FT
2861}
2862
c0bdd5d9 2863static void ironlake_disable_drps(struct drm_device *dev)
87f57070
FT
2864{
2865 struct drm_i915_private *dev_priv = dev->dev_private;
c0bdd5d9
FT
2866 u16 rgvswctl;
2867
a2fdbec6 2868 lockmgr(&mchdev_lock, LK_EXCLUSIVE);
c0bdd5d9
FT
2869
2870 rgvswctl = I915_READ16(MEMSWCTL);
87f57070
FT
2871
2872 /* Ack interrupts, disable EFC interrupt */
2873 I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
2874 I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
2875 I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
2876 I915_WRITE(DEIIR, DE_PCU_EVENT);
2877 I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
2878
2879 /* Go back to the starting frequency */
e77a37f5 2880 ironlake_set_drps(dev, dev_priv->ips.fstart);
a2fdbec6 2881 mdelay(1);
87f57070
FT
2882 rgvswctl |= MEMCTL_CMD_STS;
2883 I915_WRITE(MEMSWCTL, rgvswctl);
a2fdbec6 2884 mdelay(1);
87f57070 2885
c0bdd5d9
FT
2886 lockmgr(&mchdev_lock, LK_RELEASE);
2887}
2888
2889/* There's a funny hw issue where the hw returns all 0 when reading from
2890 * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
2891 * ourselves, instead of doing a rmw cycle (which might result in us clearing
2892 * all limits and the gpu stuck at whatever frequency it is at atm).
2893 */
9edbd4a0 2894static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
c0bdd5d9
FT
2895{
2896 u32 limits;
2897
c0bdd5d9
FT
2898 /* Only set the down limit when we've reached the lowest level to avoid
2899 * getting more interrupts, otherwise leave this clear. This prevents a
2900 * race in the hw when coming out of rc6: There's a tiny window where
2901 * the hw runs at the minimal clock before selecting the desired
2902 * frequency, if the down threshold expires in that window we will not
2903 * receive a down interrupt. */
9edbd4a0
FT
2904 limits = dev_priv->rps.max_delay << 24;
2905 if (val <= dev_priv->rps.min_delay)
c0bdd5d9 2906 limits |= dev_priv->rps.min_delay << 16;
c0bdd5d9
FT
2907
2908 return limits;
87f57070
FT
2909}
2910
9edbd4a0
FT
2911static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
2912{
2913 int new_power;
2914
2915 new_power = dev_priv->rps.power;
2916 switch (dev_priv->rps.power) {
2917 case LOW_POWER:
2918 if (val > dev_priv->rps.rpe_delay + 1 && val > dev_priv->rps.cur_delay)
2919 new_power = BETWEEN;
2920 break;
2921
2922 case BETWEEN:
2923 if (val <= dev_priv->rps.rpe_delay && val < dev_priv->rps.cur_delay)
2924 new_power = LOW_POWER;
2925 else if (val >= dev_priv->rps.rp0_delay && val > dev_priv->rps.cur_delay)
2926 new_power = HIGH_POWER;
2927 break;
2928
2929 case HIGH_POWER:
2930 if (val < (dev_priv->rps.rp1_delay + dev_priv->rps.rp0_delay) >> 1 && val < dev_priv->rps.cur_delay)
2931 new_power = BETWEEN;
2932 break;
2933 }
2934 /* Max/min bins are special */
2935 if (val == dev_priv->rps.min_delay)
2936 new_power = LOW_POWER;
2937 if (val == dev_priv->rps.max_delay)
2938 new_power = HIGH_POWER;
2939 if (new_power == dev_priv->rps.power)
2940 return;
2941
2942 /* Note the units here are not exactly 1us, but 1280ns. */
2943 switch (new_power) {
2944 case LOW_POWER:
2945 /* Upclock if more than 95% busy over 16ms */
2946 I915_WRITE(GEN6_RP_UP_EI, 12500);
2947 I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
2948
2949 /* Downclock if less than 85% busy over 32ms */
2950 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
2951 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
2952
2953 I915_WRITE(GEN6_RP_CONTROL,
2954 GEN6_RP_MEDIA_TURBO |
2955 GEN6_RP_MEDIA_HW_NORMAL_MODE |
2956 GEN6_RP_MEDIA_IS_GFX |
2957 GEN6_RP_ENABLE |
2958 GEN6_RP_UP_BUSY_AVG |
2959 GEN6_RP_DOWN_IDLE_AVG);
2960 break;
2961
2962 case BETWEEN:
2963 /* Upclock if more than 90% busy over 13ms */
2964 I915_WRITE(GEN6_RP_UP_EI, 10250);
2965 I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
2966
2967 /* Downclock if less than 75% busy over 32ms */
2968 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
2969 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
2970
2971 I915_WRITE(GEN6_RP_CONTROL,
2972 GEN6_RP_MEDIA_TURBO |
2973 GEN6_RP_MEDIA_HW_NORMAL_MODE |
2974 GEN6_RP_MEDIA_IS_GFX |
2975 GEN6_RP_ENABLE |
2976 GEN6_RP_UP_BUSY_AVG |
2977 GEN6_RP_DOWN_IDLE_AVG);
2978 break;
2979
2980 case HIGH_POWER:
2981 /* Upclock if more than 85% busy over 10ms */
2982 I915_WRITE(GEN6_RP_UP_EI, 8000);
2983 I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
2984
2985 /* Downclock if less than 60% busy over 32ms */
2986 I915_WRITE(GEN6_RP_DOWN_EI, 25000);
2987 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
2988
2989 I915_WRITE(GEN6_RP_CONTROL,
2990 GEN6_RP_MEDIA_TURBO |
2991 GEN6_RP_MEDIA_HW_NORMAL_MODE |
2992 GEN6_RP_MEDIA_IS_GFX |
2993 GEN6_RP_ENABLE |
2994 GEN6_RP_UP_BUSY_AVG |
2995 GEN6_RP_DOWN_IDLE_AVG);
2996 break;
2997 }
2998
2999 dev_priv->rps.power = new_power;
3000 dev_priv->rps.last_adj = 0;
3001}
3002
87f57070
FT
3003void gen6_set_rps(struct drm_device *dev, u8 val)
3004{
3005 struct drm_i915_private *dev_priv = dev->dev_private;
c0bdd5d9 3006
a2fdbec6 3007 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
c0bdd5d9
FT
3008 WARN_ON(val > dev_priv->rps.max_delay);
3009 WARN_ON(val < dev_priv->rps.min_delay);
3010
3011 if (val == dev_priv->rps.cur_delay)
3012 return;
3013
9edbd4a0
FT
3014 gen6_set_rps_thresholds(dev_priv, val);
3015
8e26cdf6
FT
3016 if (IS_HASWELL(dev))
3017 I915_WRITE(GEN6_RPNSWREQ,
3018 HSW_FREQUENCY(val));
3019 else
3020 I915_WRITE(GEN6_RPNSWREQ,
3021 GEN6_FREQUENCY(val) |
3022 GEN6_OFFSET(0) |
3023 GEN6_AGGRESSIVE_TURBO);
c0bdd5d9
FT
3024
3025 /* Make sure we continue to get interrupts
3026 * until we hit the minimum or maximum frequencies.
3027 */
9edbd4a0
FT
3028 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
3029 gen6_rps_limits(dev_priv, val));
87f57070 3030
c0bdd5d9
FT
3031 POSTING_READ(GEN6_RPNSWREQ);
3032
3033 dev_priv->rps.cur_delay = val;
a2fdbec6
FT
3034
3035 trace_intel_gpu_freq_change(val * 50);
87f57070
FT
3036}
3037
9edbd4a0 3038void gen6_rps_idle(struct drm_i915_private *dev_priv)
5d0b1887 3039{
9edbd4a0 3040 struct drm_device *dev = dev_priv->dev;
5d0b1887 3041
9edbd4a0
FT
3042 mutex_lock(&dev_priv->rps.hw_lock);
3043 if (dev_priv->rps.enabled) {
3044 if (IS_VALLEYVIEW(dev))
3045 valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
3046 else
3047 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
3048 dev_priv->rps.last_adj = 0;
3049 }
3050 mutex_unlock(&dev_priv->rps.hw_lock);
3051}
5d0b1887 3052
9edbd4a0
FT
3053void gen6_rps_boost(struct drm_i915_private *dev_priv)
3054{
3055 struct drm_device *dev = dev_priv->dev;
5d0b1887 3056
9edbd4a0
FT
3057 mutex_lock(&dev_priv->rps.hw_lock);
3058 if (dev_priv->rps.enabled) {
3059 if (IS_VALLEYVIEW(dev))
3060 valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
3061 else
3062 gen6_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
3063 dev_priv->rps.last_adj = 0;
3064 }
3065 mutex_unlock(&dev_priv->rps.hw_lock);
5d0b1887
FT
3066}
3067
3068void valleyview_set_rps(struct drm_device *dev, u8 val)
3069{
3070 struct drm_i915_private *dev_priv = dev->dev_private;
3071
5d0b1887
FT
3072 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3073 WARN_ON(val > dev_priv->rps.max_delay);
3074 WARN_ON(val < dev_priv->rps.min_delay);
3075
5d0b1887 3076 DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
9edbd4a0 3077 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay),
5d0b1887 3078 dev_priv->rps.cur_delay,
9edbd4a0 3079 vlv_gpu_freq(dev_priv, val), val);
5d0b1887
FT
3080
3081 if (val == dev_priv->rps.cur_delay)
3082 return;
3083
3084 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
3085
3086 dev_priv->rps.cur_delay = val;
3087
9edbd4a0 3088 trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
5d0b1887
FT
3089}
3090
9edbd4a0 3091static void gen6_disable_rps_interrupts(struct drm_device *dev)
87f57070
FT
3092{
3093 struct drm_i915_private *dev_priv = dev->dev_private;
3094
87f57070 3095 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
5d0b1887
FT
3096 I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS);
3097 /* Complete PM interrupt masking here doesn't race with the rps work
3098 * item again unmasking PM interrupts because that is using a different
3099 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
3100 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
3101
9edbd4a0
FT
3102 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE);
3103 dev_priv->rps.pm_iir = 0;
3104 lockmgr(&dev_priv->irq_lock, LK_RELEASE);
3105
3106 I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
3107}
3108
3109static void gen6_disable_rps(struct drm_device *dev)
3110{
3111 struct drm_i915_private *dev_priv = dev->dev_private;
3112
3113 I915_WRITE(GEN6_RC_CONTROL, 0);
3114 I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
5d0b1887 3115
9edbd4a0 3116 gen6_disable_rps_interrupts(dev);
5d0b1887
FT
3117}
3118
3119static void valleyview_disable_rps(struct drm_device *dev)
3120{
3121 struct drm_i915_private *dev_priv = dev->dev_private;
3122
3123 I915_WRITE(GEN6_RC_CONTROL, 0);
87f57070 3124
9edbd4a0 3125 gen6_disable_rps_interrupts(dev);
5d0b1887
FT
3126
3127 if (dev_priv->vlv_pctx) {
3128 drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
3129 dev_priv->vlv_pctx = NULL;
3130 }
87f57070
FT
3131}
3132
9edbd4a0
FT
3133static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
3134{
3135 if (IS_GEN6(dev))
3136 DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
3137
3138 if (IS_HASWELL(dev))
3139 DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
3140
3141 DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
3142 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
3143 (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
3144 (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
3145}
3146
c0bdd5d9 3147int intel_enable_rc6(const struct drm_device *dev)
87f57070 3148{
9edbd4a0
FT
3149 /* No RC6 before Ironlake */
3150 if (INTEL_INFO(dev)->gen < 5)
3151 return 0;
3152
c0bdd5d9
FT
3153 /* Respect the kernel parameter if it is set */
3154 if (i915_enable_rc6 >= 0)
3155 return i915_enable_rc6;
87f57070 3156
c0bdd5d9
FT
3157 /* Disable RC6 on Ironlake */
3158 if (INTEL_INFO(dev)->gen == 5)
87f57070
FT
3159 return 0;
3160
9edbd4a0 3161 if (IS_HASWELL(dev))
c0bdd5d9 3162 return INTEL_RC6_ENABLE;
87f57070 3163
c0bdd5d9 3164 /* snb/ivb have more than one rc6 state. */
9edbd4a0 3165 if (INTEL_INFO(dev)->gen == 6)
c0bdd5d9 3166 return INTEL_RC6_ENABLE;
87f57070 3167
c0bdd5d9
FT
3168 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
3169}
bc9b2a29 3170
9edbd4a0
FT
3171static void gen6_enable_rps_interrupts(struct drm_device *dev)
3172{
3173 struct drm_i915_private *dev_priv = dev->dev_private;
3174 u32 enabled_intrs;
3175
3176 lockmgr(&dev_priv->irq_lock, LK_EXCLUSIVE);
3177 WARN_ON(dev_priv->rps.pm_iir);
3178 snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
3179 I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
3180 lockmgr(&dev_priv->irq_lock, LK_RELEASE);
3181
3182 /* only unmask PM interrupts we need. Mask all others. */
3183 enabled_intrs = GEN6_PM_RPS_EVENTS;
3184
3185 /* IVB and SNB hard hangs on looping batchbuffer
3186 * if GEN6_PM_UP_EI_EXPIRED is masked.
3187 */
3188 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
3189 enabled_intrs |= GEN6_PM_RP_UP_EI_EXPIRED;
3190
3191 I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs);
3192}
3193
3194static void gen8_enable_rps(struct drm_device *dev)
3195{
3196 struct drm_i915_private *dev_priv = dev->dev_private;
3197 struct intel_ring_buffer *ring;
3198 uint32_t rc6_mask = 0, rp_state_cap;
3199 int unused;
3200
3201 /* 1a: Software RC state - RC0 */
3202 I915_WRITE(GEN6_RC_STATE, 0);
3203
3204 /* 1c & 1d: Get forcewake during program sequence. Although the driver
3205 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
3206 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3207
3208 /* 2a: Disable RC states. */
3209 I915_WRITE(GEN6_RC_CONTROL, 0);
3210
3211 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3212
3213 /* 2b: Program RC6 thresholds.*/
3214 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
3215 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
3216 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
3217 for_each_ring(ring, dev_priv, unused)
3218 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3219 I915_WRITE(GEN6_RC_SLEEP, 0);
3220 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
3221
3222 /* 3: Enable RC6 */
3223 if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
3224 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
3225 DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
3226 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
3227 GEN6_RC_CTL_EI_MODE(1) |
3228 rc6_mask);
3229
3230 /* 4 Program defaults and thresholds for RPS*/
3231 I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */
3232 I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(12)); /* Request 600 MHz */
3233 /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
3234 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
3235
3236 /* Docs recommend 900MHz, and 300 MHz respectively */
3237 I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
3238 dev_priv->rps.max_delay << 24 |
3239 dev_priv->rps.min_delay << 16);
3240
3241 I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
3242 I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
3243 I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
3244 I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
3245
3246 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
3247
3248 /* 5: Enable RPS */
3249 I915_WRITE(GEN6_RP_CONTROL,
3250 GEN6_RP_MEDIA_TURBO |
3251 GEN6_RP_MEDIA_HW_NORMAL_MODE |
3252 GEN6_RP_MEDIA_IS_GFX |
3253 GEN6_RP_ENABLE |
3254 GEN6_RP_UP_BUSY_AVG |
3255 GEN6_RP_DOWN_IDLE_AVG);
3256
3257 /* 6: Ring frequency + overclocking (our driver does this later */
3258
3259 gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
3260
3261 gen6_enable_rps_interrupts(dev);
3262
3263 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
3264}
3265
c0bdd5d9 3266static void gen6_enable_rps(struct drm_device *dev)
bc9b2a29 3267{
c0bdd5d9
FT
3268 struct drm_i915_private *dev_priv = dev->dev_private;
3269 struct intel_ring_buffer *ring;
3270 u32 rp_state_cap;
3271 u32 gt_perf_status;
3272 u32 rc6vids, pcu_mbox, rc6_mask = 0;
3273 u32 gtfifodbg;
3274 int rc6_mode;
3275 int i, ret;
bc9b2a29 3276
a2fdbec6
FT
3277 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3278
c0bdd5d9
FT
3279 /* Here begins a magic sequence of register writes to enable
3280 * auto-downclocking.
3281 *
3282 * Perhaps there might be some value in exposing these to
3283 * userspace...
bc9b2a29 3284 */
c0bdd5d9 3285 I915_WRITE(GEN6_RC_STATE, 0);
bc9b2a29 3286
c0bdd5d9
FT
3287 /* Clear the DBG now so we don't confuse earlier errors */
3288 if ((gtfifodbg = I915_READ(GTFIFODBG))) {
3289 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
3290 I915_WRITE(GTFIFODBG, gtfifodbg);
3291 }
bc9b2a29 3292
9edbd4a0 3293 gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
bc9b2a29 3294
c0bdd5d9
FT
3295 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
3296 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
bc9b2a29 3297
8e26cdf6
FT
3298 /* In units of 50MHz */
3299 dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
9edbd4a0
FT
3300 dev_priv->rps.min_delay = (rp_state_cap >> 16) & 0xff;
3301 dev_priv->rps.rp1_delay = (rp_state_cap >> 8) & 0xff;
3302 dev_priv->rps.rp0_delay = (rp_state_cap >> 0) & 0xff;
3303 dev_priv->rps.rpe_delay = dev_priv->rps.rp1_delay;
c0bdd5d9
FT
3304 dev_priv->rps.cur_delay = 0;
3305
3306 /* disable the counters and set deterministic thresholds */
3307 I915_WRITE(GEN6_RC_CONTROL, 0);
3308
3309 I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
3310 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
3311 I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
3312 I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
3313 I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
3314
3315 for_each_ring(ring, dev_priv, i)
3316 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
3317
3318 I915_WRITE(GEN6_RC_SLEEP, 0);
3319 I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
9edbd4a0
FT
3320 if (IS_IVYBRIDGE(dev))
3321 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
3322 else
3323 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
c0bdd5d9
FT
3324 I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3325 I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
3326
3327 /* Check if we are enabling RC6 */
3328 rc6_mode = intel_enable_rc6(dev_priv->dev);
3329 if (rc6_mode & INTEL_RC6_ENABLE)
3330 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
3331
3332 /* We don't use those on Haswell */
3333 if (!IS_HASWELL(dev)) {
3334 if (rc6_mode & INTEL_RC6p_ENABLE)
3335 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
3336
3337 if (rc6_mode & INTEL_RC6pp_ENABLE)
3338 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
3339 }
3340
9edbd4a0 3341 intel_print_rc6_info(dev, rc6_mask);
c0bdd5d9
FT
3342
3343 I915_WRITE(GEN6_RC_CONTROL,
3344 rc6_mask |
3345 GEN6_RC_CTL_EI_MODE(1) |
3346 GEN6_RC_CTL_HW_ENABLE);
3347
9edbd4a0
FT
3348 /* Power down if completely idle for over 50ms */
3349 I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
c0bdd5d9 3350 I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
c0bdd5d9
FT
3351
3352 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
3353 if (!ret) {
3354 pcu_mbox = 0;
3355 ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
8e26cdf6
FT
3356 if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
3357 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
3358 (dev_priv->rps.max_delay & 0xff) * 50,
3359 (pcu_mbox & 0xff) * 50);
3360 dev_priv->rps.hw_max = pcu_mbox & 0xff;
c0bdd5d9
FT
3361 }
3362 } else {
3363 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
3364 }
3365
9edbd4a0
FT
3366 dev_priv->rps.power = HIGH_POWER; /* force a reset */
3367 gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
c0bdd5d9 3368
9edbd4a0 3369 gen6_enable_rps_interrupts(dev);
c0bdd5d9
FT
3370
3371 rc6vids = 0;
3372 ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
3373 if (IS_GEN6(dev) && ret) {
3374 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
3375 } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
3376 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
3377 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
3378 rc6vids &= 0xffff00;
3379 rc6vids |= GEN6_ENCODE_RC6_VID(450);
3380 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
3381 if (ret)
3382 DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
3383 }
3384
9edbd4a0 3385 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
c0bdd5d9
FT
3386}
3387
9edbd4a0 3388void gen6_update_ring_freq(struct drm_device *dev)
c0bdd5d9
FT
3389{
3390 struct drm_i915_private *dev_priv = dev->dev_private;
3391 int min_freq = 15;
8e26cdf6
FT
3392 unsigned int gpu_freq;
3393 unsigned int max_ia_freq, min_ring_freq;
c0bdd5d9
FT
3394 int scaling_factor = 180;
3395
a2fdbec6
FT
3396 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3397
c0bdd5d9 3398#if 0
9edbd4a0
FT
3399 policy = cpufreq_cpu_get(0);
3400 if (policy) {
3401 max_ia_freq = policy->cpuinfo.max_freq;
3402 cpufreq_cpu_put(policy);
3403 } else {
3404 /*
3405 * Default to measured freq if none found, PCU will ensure we
3406 * don't go over
3407 */
c0bdd5d9 3408 max_ia_freq = tsc_khz;
9edbd4a0 3409 }
c0bdd5d9
FT
3410#else
3411 max_ia_freq = tsc_frequency / 1000;
3412#endif
3413
3414 /* Convert from kHz to MHz */
3415 max_ia_freq /= 1000;
3416
9edbd4a0
FT
3417 min_ring_freq = I915_READ(DCLK) & 0xf;
3418 /* convert DDR frequency from units of 266.6MHz to bandwidth */
3419 min_ring_freq = mult_frac(min_ring_freq, 8, 3);
8e26cdf6 3420
c0bdd5d9
FT
3421 /*
3422 * For each potential GPU frequency, load a ring frequency we'd like
3423 * to use for memory access. We do this by specifying the IA frequency
3424 * the PCU should use as a reference to determine the ring frequency.
3425 */
3426 for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
3427 gpu_freq--) {
3428 int diff = dev_priv->rps.max_delay - gpu_freq;
8e26cdf6
FT
3429 unsigned int ia_freq = 0, ring_freq = 0;
3430
9edbd4a0
FT
3431 if (INTEL_INFO(dev)->gen >= 8) {
3432 /* max(2 * GT, DDR). NB: GT is 50MHz units */
3433 ring_freq = max(min_ring_freq, gpu_freq);
3434 } else if (IS_HASWELL(dev)) {
3435 ring_freq = mult_frac(gpu_freq, 5, 4);
8e26cdf6
FT
3436 ring_freq = max(min_ring_freq, ring_freq);
3437 /* leave ia_freq as the default, chosen by cpufreq */
3438 } else {
3439 /* On older processors, there is no separate ring
3440 * clock domain, so in order to boost the bandwidth
3441 * of the ring, we need to upclock the CPU (ia_freq).
3442 *
3443 * For GPU frequencies less than 750MHz,
3444 * just use the lowest ring freq.
3445 */
3446 if (gpu_freq < min_freq)
3447 ia_freq = 800;
3448 else
3449 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
3450 ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3451 }
c0bdd5d9
FT
3452
3453 sandybridge_pcode_write(dev_priv,
3454 GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
8e26cdf6
FT
3455 ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
3456 ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
3457 gpu_freq);
c0bdd5d9
FT
3458 }
3459}
3460
5d0b1887 3461int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
c0bdd5d9 3462{
5d0b1887 3463 u32 val, rp0;
c0bdd5d9 3464
5d0b1887 3465 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
c0bdd5d9 3466
5d0b1887
FT
3467 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
3468 /* Clamp to max */
3469 rp0 = min_t(u32, rp0, 0xea);
3470
3471 return rp0;
3472}
3473
3474static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
3475{
3476 u32 val, rpe;
3477
3478 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
3479 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
3480 val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
3481 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;