[dragonfly.git] / sys / dev / drm / i915 / i915_irq.c

/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
 */
/*-
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#include "dev/drm/drmP.h"
#include "dev/drm/drm.h"
#include "dev/drm/i915_drm.h"
#include "i915_drv.h"

#define MAX_NOPID ((u32)~0)

/**
 * Interrupts that are always left unmasked.
 *
 * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
 * we leave them always unmasked in IMR and then control enabling them through
 * PIPESTAT alone.
 */
#define I915_INTERRUPT_ENABLE_FIX	(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
				   	 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)

/** Interrupts that we mask and unmask at runtime. */
#define I915_INTERRUPT_ENABLE_VAR	(I915_USER_INTERRUPT)

/** These are all of the interrupts used by the driver */
#define I915_INTERRUPT_ENABLE_MASK	(I915_INTERRUPT_ENABLE_FIX | \
				    	 I915_INTERRUPT_ENABLE_VAR)

#define DRM_I915_VBLANK_PIPE_ALL	(DRM_I915_VBLANK_PIPE_A | \
					 DRM_I915_VBLANK_PIPE_B)

static inline void
i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
{
	mask &= I915_INTERRUPT_ENABLE_VAR;
	if ((dev_priv->irq_mask_reg & mask) != 0) {
		dev_priv->irq_mask_reg &= ~mask;
		I915_WRITE(IMR, dev_priv->irq_mask_reg);
		(void) I915_READ(IMR);
	}
}

static inline void
i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
{
	mask &= I915_INTERRUPT_ENABLE_VAR;
	if ((dev_priv->irq_mask_reg & mask) != mask) {
		dev_priv->irq_mask_reg |= mask;
		I915_WRITE(IMR, dev_priv->irq_mask_reg);
		(void) I915_READ(IMR);
	}
}

static inline u32
i915_pipestat(int pipe)
{
	if (pipe == 0)
	    return PIPEASTAT;
	if (pipe == 1)
	    return PIPEBSTAT;
	return -EINVAL;
}

void
i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
{
	if ((dev_priv->pipestat[pipe] & mask) != mask) {
		u32 reg = i915_pipestat(pipe);

		dev_priv->pipestat[pipe] |= mask;
		/* Enable the interrupt, clear any pending status */
		I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
		(void) I915_READ(reg);
	}
}

void
i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
{
	if ((dev_priv->pipestat[pipe] & mask) != 0) {
		u32 reg = i915_pipestat(pipe);

		dev_priv->pipestat[pipe] &= ~mask;
		I915_WRITE(reg, dev_priv->pipestat[pipe]);
		(void) I915_READ(reg);
	}
}

/**
 * i915_pipe_enabled - check if a pipe is enabled
 * @dev: DRM device
 * @pipe: pipe to check
 *
 * Reading certain registers when the pipe is disabled can hang the chip.
 * Use this routine to make sure the PLL is running and the pipe is active
 * before reading such registers if unsure.
 */
static int
i915_pipe_enabled(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;

	if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
		return 1;

	return 0;
}

/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long high_frame;
	unsigned long low_frame;
	u32 high1, high2, low, count;

	high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
	low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;

	if (!i915_pipe_enabled(dev, pipe)) {
		DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe);
		return 0;
	}

	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
		high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
			 PIPE_FRAME_HIGH_SHIFT);
		low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
			PIPE_FRAME_LOW_SHIFT);
		high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
			 PIPE_FRAME_HIGH_SHIFT);
	} while (high1 != high2);

	count = (high1 << 8) | low;

	return count;
}

u32 g45_get_vblank_counter(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45;

	if (!i915_pipe_enabled(dev, pipe)) {
		DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe);
		return 0;
	}

	return I915_READ(reg);
}

irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
{
	struct drm_device *dev = (struct drm_device *) arg;
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	u32 iir, new_iir;
	u32 pipea_stats, pipeb_stats;
	u32 vblank_status;
	int irq_received;

	iir = I915_READ(IIR);

	if (IS_I965G(dev))
		vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS;
	else
		vblank_status = I915_VBLANK_INTERRUPT_STATUS;

	for (;;) {
		irq_received = iir != 0;

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
		DRM_SPINLOCK(&dev_priv->user_irq_lock);
		pipea_stats = I915_READ(PIPEASTAT);
		pipeb_stats = I915_READ(PIPEBSTAT);

		/*
		 * Clear the PIPE(A|B)STAT regs before the IIR
		 */
		if (pipea_stats & 0x8000ffff) {
			I915_WRITE(PIPEASTAT, pipea_stats);
			irq_received = 1;
		}

		if (pipeb_stats & 0x8000ffff) {
			I915_WRITE(PIPEBSTAT, pipeb_stats);
			irq_received = 1;
		}
		DRM_SPINUNLOCK(&dev_priv->user_irq_lock);

		if (!irq_received)
			break;

		I915_WRITE(IIR, iir);
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (dev_priv->sarea_priv)
			dev_priv->sarea_priv->last_dispatch =
			    READ_BREADCRUMB(dev_priv);

		if (iir & I915_USER_INTERRUPT) {
			DRM_WAKEUP(&dev_priv->irq_queue);
		}

		if (pipea_stats & vblank_status)
			drm_handle_vblank(dev, 0);

		if (pipeb_stats & vblank_status)
			drm_handle_vblank(dev, 1);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}
}

static int i915_emit_irq(struct drm_device * dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	RING_LOCALS;

	i915_kernel_lost_context(dev);

	if (++dev_priv->counter > 0x7FFFFFFFUL)
		dev_priv->counter = 0;
	if (dev_priv->sarea_priv)
		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;

	DRM_DEBUG("emitting: %d\n", dev_priv->counter);

	BEGIN_LP_RING(4);
	OUT_RING(MI_STORE_DWORD_INDEX);
	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
	OUT_RING(dev_priv->counter);
	OUT_RING(MI_USER_INTERRUPT);
	ADVANCE_LP_RING();

	return dev_priv->counter;
}

void i915_user_irq_get(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	if (dev->irq_enabled == 0)
		return;

	DRM_DEBUG("\n");
	DRM_SPINLOCK(&dev_priv->user_irq_lock);
	if (++dev_priv->user_irq_refcount == 1)
		i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
}

void i915_user_irq_put(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	if (dev->irq_enabled == 0)
		return;

	DRM_SPINLOCK(&dev_priv->user_irq_lock);
	KASSERT(dev_priv->user_irq_refcount > 0, ("invalid refcount"));
	if (--dev_priv->user_irq_refcount == 0)
		i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
}

static int i915_wait_irq(struct drm_device * dev, int irq_nr)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	int ret = 0;

	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
		if (dev_priv->sarea_priv) {
			dev_priv->sarea_priv->last_dispatch =
				READ_BREADCRUMB(dev_priv);
		}
		return 0;
	}

	if (dev_priv->sarea_priv)
		dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;

	DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
		  READ_BREADCRUMB(dev_priv));

	i915_user_irq_get(dev);
	DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ,
		    READ_BREADCRUMB(dev_priv) >= irq_nr);
	i915_user_irq_put(dev);

	if (ret == -ERESTART)
		DRM_DEBUG("restarting syscall\n");

	if (ret == -EBUSY) {
		DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
	}

	return ret;
}

/* Needs the lock as it touches the ring.
 */
int i915_irq_emit(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	drm_i915_irq_emit_t *emit = data;
	int result;

	if (!dev_priv) {
		DRM_ERROR("called with no initialization\n");
		return -EINVAL;
	}

	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);

	result = i915_emit_irq(dev);

	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
		DRM_ERROR("copy_to_user\n");
		return -EFAULT;
	}

	return 0;
}

/* Doesn't need the hardware lock.
 */
int i915_irq_wait(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	drm_i915_irq_wait_t *irqwait = data;

	if (!dev_priv) {
		DRM_ERROR("called with no initialization\n");
		return -EINVAL;
	}

	return i915_wait_irq(dev, irqwait->irq_seq);
}

/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
int i915_enable_vblank(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	DRM_SPINLOCK(&dev_priv->user_irq_lock);
	if (IS_I965G(dev))
		i915_enable_pipestat(dev_priv, pipe,
				     PIPE_START_VBLANK_INTERRUPT_ENABLE);
	else
		i915_enable_pipestat(dev_priv, pipe,
				     PIPE_VBLANK_INTERRUPT_ENABLE);
	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
	return 0;
}

/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
void i915_disable_vblank(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	DRM_SPINLOCK(&dev_priv->user_irq_lock);
	i915_disable_pipestat(dev_priv, pipe,
			      PIPE_VBLANK_INTERRUPT_ENABLE |
			      PIPE_START_VBLANK_INTERRUPT_ENABLE);
	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
}

/* Set the vblank monitor pipe
 */
int i915_vblank_pipe_set(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
{
	drm_i915_private_t *dev_priv = dev->dev_private;

	if (!dev_priv) {
		DRM_ERROR("called with no initialization\n");
		return -EINVAL;
	}

	return 0;
}

int i915_vblank_pipe_get(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	drm_i915_vblank_pipe_t *pipe = data;

	if (!dev_priv) {
		DRM_ERROR("called with no initialization\n");
		return -EINVAL;
	}

	pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;

	return 0;
}

/**
 * Schedule buffer swap at given vertical blank.
 */
int i915_vblank_swap(struct drm_device *dev, void *data,
		     struct drm_file *file_priv)
{
	/* The delayed swap mechanism was fundamentally racy, and has been
	 * removed.  The model was that the client requested a delayed flip/swap
	 * from the kernel, then waited for vblank before continuing to perform
	 * rendering.  The problem was that the kernel might wake the client
	 * up before it dispatched the vblank swap (since the lock has to be
	 * held while touching the ringbuffer), in which case the client would
	 * clear and start the next frame before the swap occurred, and
	 * flicker would occur in addition to likely missing the vblank.
	 *
	 * In the absence of this ioctl, userland falls back to a correct path
	 * of waiting for a vblank, then dispatching the swap on its own.
	 * Context switching to userland and back is plenty fast enough for
	 * meeting the requirements of vblank swapping.
	 */
	return -EINVAL;
}

/* drm_dma.h hooks
*/
void i915_driver_irq_preinstall(struct drm_device * dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	I915_WRITE(HWSTAM, 0xeffe);
	I915_WRITE(PIPEASTAT, 0);
	I915_WRITE(PIPEBSTAT, 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	(void) I915_READ(IER);
}

int i915_driver_irq_postinstall(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;

	/* Disable pipe interrupt enables, clear pending pipe status */
	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);

	/* Clear pending interrupt status */
	I915_WRITE(IIR, I915_READ(IIR));

	I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
	I915_WRITE(IMR, dev_priv->irq_mask_reg);
	I915_WRITE(PIPEASTAT, dev_priv->pipestat[0] |
	    (dev_priv->pipestat[0] >> 16));
	I915_WRITE(PIPEBSTAT, dev_priv->pipestat[1] |
	    (dev_priv->pipestat[1] >> 16));
	(void) I915_READ(IER);

	return 0;
}

void i915_driver_irq_uninstall(struct drm_device * dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

	if (!dev_priv)
		return;

	dev_priv->vblank_pipe = 0;

	I915_WRITE(HWSTAM, 0xffffffff);
	I915_WRITE(PIPEASTAT, 0);
	I915_WRITE(PIPEBSTAT, 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}
Commit	Line	Data
7f3c3d6f HT	1	/* i915_irq.c -- IRQ support for the I915 -- linux-c --
7f3c3d6f HT	2	*/
b3705d71	3	/*-
7f3c3d6f HT	4	* Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
	5	* All Rights Reserved.
	6	*
	7	* Permission is hereby granted, free of charge, to any person obtaining a
	8	* copy of this software and associated documentation files (the
	9	* "Software"), to deal in the Software without restriction, including
	10	* without limitation the rights to use, copy, modify, merge, publish,
	11	* distribute, sub license, and/or sell copies of the Software, and to
	12	* permit persons to whom the Software is furnished to do so, subject to
	13	* the following conditions:
	14	*
	15	* The above copyright notice and this permission notice (including the
	16	* next paragraph) shall be included in all copies or substantial portions
	17	* of the Software.
	18	*
	19	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
	20	* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	21	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
	22	* IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
	23	* ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
	24	* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
	25	* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	26	*
7f3c3d6f HT	27	*/
7f3c3d6f HT	28
b3705d71 HT	29	#include "dev/drm/drmP.h"
	30	#include "dev/drm/drm.h"
	31	#include "dev/drm/i915_drm.h"
c4a9e910	32	#include "i915_drv.h"
7f3c3d6f HT	33
	34	#define MAX_NOPID ((u32)~0)
	35
	36	/**
b3705d71	37	* Interrupts that are always left unmasked.
7f3c3d6f	38	*
b3705d71 HT	39	* Since pipe events are edge-triggered from the PIPESTAT register to IIR,
	40	* we leave them always unmasked in IMR and then control enabling them through
	41	* PIPESTAT alone.
7f3c3d6f	42	*/
b3705d71 HT	43	#define I915_INTERRUPT_ENABLE_FIX (I915_DISPLAY_PIPE_A_EVENT_INTERRUPT \| \
	44	I915_DISPLAY_PIPE_B_EVENT_INTERRUPT)
	45
	46	/** Interrupts that we mask and unmask at runtime. */
	47	#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT)
	48
	49	/** These are all of the interrupts used by the driver */
	50	#define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX \| \
	51	I915_INTERRUPT_ENABLE_VAR)
	52
b3705d71 HT	53	#define DRM_I915_VBLANK_PIPE_ALL (DRM_I915_VBLANK_PIPE_A \| \
	54	DRM_I915_VBLANK_PIPE_B)
	55
	56	static inline void
	57	i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
7f3c3d6f	58	{
b3705d71 HT	59	mask &= I915_INTERRUPT_ENABLE_VAR;
	60	if ((dev_priv->irq_mask_reg & mask) != 0) {
	61	dev_priv->irq_mask_reg &= ~mask;
	62	I915_WRITE(IMR, dev_priv->irq_mask_reg);
	63	(void) I915_READ(IMR);
	64	}
	65	}
7f3c3d6f	66
b3705d71 HT	67	static inline void
	68	i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
	69	{
	70	mask &= I915_INTERRUPT_ENABLE_VAR;
	71	if ((dev_priv->irq_mask_reg & mask) != mask) {
	72	dev_priv->irq_mask_reg \|= mask;
	73	I915_WRITE(IMR, dev_priv->irq_mask_reg);
	74	(void) I915_READ(IMR);
	75	}
7f3c3d6f HT	76	}
7f3c3d6f HT	77
b3705d71 HT	78	static inline u32
b3705d71 HT	79	i915_pipestat(int pipe)
7f3c3d6f	80	{
b3705d71 HT	81	if (pipe == 0)
	82	return PIPEASTAT;
	83	if (pipe == 1)
	84	return PIPEBSTAT;
	85	return -EINVAL;
	86	}
7f3c3d6f	87
b3705d71 HT	88	void
	89	i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
	90	{
	91	if ((dev_priv->pipestat[pipe] & mask) != mask) {
	92	u32 reg = i915_pipestat(pipe);
7f3c3d6f	93
b3705d71 HT	94	dev_priv->pipestat[pipe] \|= mask;
	95	/* Enable the interrupt, clear any pending status */
	96	I915_WRITE(reg, dev_priv->pipestat[pipe] \| (mask >> 16));
	97	(void) I915_READ(reg);
7f3c3d6f	98	}
b3705d71	99	}
7f3c3d6f	100
b3705d71 HT	101	void
	102	i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
	103	{
	104	if ((dev_priv->pipestat[pipe] & mask) != 0) {
	105	u32 reg = i915_pipestat(pipe);
7f3c3d6f	106
b3705d71 HT	107	dev_priv->pipestat[pipe] &= ~mask;
	108	I915_WRITE(reg, dev_priv->pipestat[pipe]);
	109	(void) I915_READ(reg);
7f3c3d6f	110	}
7f3c3d6f HT	111	}
	112
	113	/**
b3705d71 HT	114	* i915_pipe_enabled - check if a pipe is enabled
	115	* @dev: DRM device
	116	* @pipe: pipe to check
7f3c3d6f	117	*
b3705d71 HT	118	* Reading certain registers when the pipe is disabled can hang the chip.
	119	* Use this routine to make sure the PLL is running and the pipe is active
	120	* before reading such registers if unsure.
7f3c3d6f	121	*/
b3705d71 HT	122	static int
b3705d71 HT	123	i915_pipe_enabled(struct drm_device *dev, int pipe)
7f3c3d6f HT	124	{
7f3c3d6f HT	125	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
b3705d71	126	unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;
7f3c3d6f	127
b3705d71 HT	128	if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
b3705d71 HT	129	return 1;
7f3c3d6f	130
b3705d71 HT	131	return 0;
b3705d71 HT	132	}
7f3c3d6f	133
b3705d71 HT	134	/* Called from drm generic code, passed a 'crtc', which
	135	* we use as a pipe index
	136	*/
	137	u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
	138	{
	139	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
	140	unsigned long high_frame;
	141	unsigned long low_frame;
	142	u32 high1, high2, low, count;
7f3c3d6f	143
b3705d71 HT	144	high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
b3705d71 HT	145	low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
7f3c3d6f	146
b3705d71	147	if (!i915_pipe_enabled(dev, pipe)) {
f77a46dc	148	DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe);
b3705d71	149	return 0;
7f3c3d6f HT	150	}
7f3c3d6f HT	151
b3705d71 HT	152	/*
	153	* High & low register fields aren't synchronized, so make sure
	154	* we get a low value that's stable across two reads of the high
	155	* register.
7f3c3d6f	156	*/
b3705d71 HT	157	do {
	158	high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
	159	PIPE_FRAME_HIGH_SHIFT);
	160	low = ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
	161	PIPE_FRAME_LOW_SHIFT);
	162	high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
	163	PIPE_FRAME_HIGH_SHIFT);
	164	} while (high1 != high2);
	165
	166	count = (high1 << 8) \| low;
	167
	168	return count;
	169	}
7f3c3d6f	170
09c555ec	171	u32 g45_get_vblank_counter(struct drm_device *dev, int pipe)
b3705d71 HT	172	{
	173	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
	174	int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45;
7f3c3d6f	175
b3705d71	176	if (!i915_pipe_enabled(dev, pipe)) {
f77a46dc	177	DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe);
b3705d71	178	return 0;
7f3c3d6f HT	179	}
7f3c3d6f HT	180
b3705d71	181	return I915_READ(reg);
7f3c3d6f HT	182	}
	183
	184	irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
	185	{
	186	struct drm_device dev = (struct drm_device ) arg;
	187	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
b3705d71	188	u32 iir, new_iir;
7f3c3d6f	189	u32 pipea_stats, pipeb_stats;
b3705d71	190	u32 vblank_status;
b3705d71	191	int irq_received;
7f3c3d6f	192
b3705d71	193	iir = I915_READ(IIR);
7f3c3d6f	194
e076c50a	195	if (IS_I965G(dev))
f77a46dc	196	vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS;
e076c50a	197	else
b3705d71	198	vblank_status = I915_VBLANK_INTERRUPT_STATUS;
7f3c3d6f	199
b3705d71 HT	200	for (;;) {
b3705d71 HT	201	irq_received = iir != 0;
7f3c3d6f	202
b3705d71 HT	203	/* Can't rely on pipestat interrupt bit in iir as it might
	204	* have been cleared after the pipestat interrupt was received.
	205	* It doesn't set the bit in iir again, but it still produces
	206	* interrupts (for non-MSI).
	207	*/
	208	DRM_SPINLOCK(&dev_priv->user_irq_lock);
	209	pipea_stats = I915_READ(PIPEASTAT);
	210	pipeb_stats = I915_READ(PIPEBSTAT);
7f3c3d6f	211
b3705d71 HT	212	/*
	213	* Clear the PIPE(A\|B)STAT regs before the IIR
	214	*/
	215	if (pipea_stats & 0x8000ffff) {
	216	I915_WRITE(PIPEASTAT, pipea_stats);
	217	irq_received = 1;
	218	}
7f3c3d6f	219
b3705d71 HT	220	if (pipeb_stats & 0x8000ffff) {
	221	I915_WRITE(PIPEBSTAT, pipeb_stats);
	222	irq_received = 1;
	223	}
	224	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
	225
	226	if (!irq_received)
	227	break;
	228
	229	I915_WRITE(IIR, iir);
	230	new_iir = I915_READ(IIR); /* Flush posted writes */
	231
	232	if (dev_priv->sarea_priv)
	233	dev_priv->sarea_priv->last_dispatch =
	234	READ_BREADCRUMB(dev_priv);
	235
	236	if (iir & I915_USER_INTERRUPT) {
	237	DRM_WAKEUP(&dev_priv->irq_queue);
	238	}
7f3c3d6f	239
b3705d71 HT	240	if (pipea_stats & vblank_status)
	241	drm_handle_vblank(dev, 0);
	242
	243	if (pipeb_stats & vblank_status)
	244	drm_handle_vblank(dev, 1);
	245
	246	/* With MSI, interrupts are only generated when iir
	247	* transitions from zero to nonzero. If another bit got
	248	* set while we were handling the existing iir bits, then
	249	* we would never get another interrupt.
	250	*
	251	* This is fine on non-MSI as well, as if we hit this path
	252	* we avoid exiting the interrupt handler only to generate
	253	* another one.
	254	*
	255	* Note that for MSI this could cause a stray interrupt report
	256	* if an interrupt landed in the time between writing IIR and
	257	* the posting read. This should be rare enough to never
	258	* trigger the 99% of 100,000 interrupts test for disabling
	259	* stray interrupts.
	260	*/
	261	iir = new_iir;
	262	}
7f3c3d6f HT	263	}
7f3c3d6f HT	264
b3705d71	265	static int i915_emit_irq(struct drm_device * dev)
7f3c3d6f HT	266	{
	267	drm_i915_private_t *dev_priv = dev->dev_private;
	268	RING_LOCALS;
	269
	270	i915_kernel_lost_context(dev);
	271
b3705d71 HT	272	if (++dev_priv->counter > 0x7FFFFFFFUL)
	273	dev_priv->counter = 0;
	274	if (dev_priv->sarea_priv)
	275	dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
7f3c3d6f	276
b3705d71	277	DRM_DEBUG("emitting: %d\n", dev_priv->counter);
7f3c3d6f	278
b3705d71 HT	279	BEGIN_LP_RING(4);
	280	OUT_RING(MI_STORE_DWORD_INDEX);
	281	OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
	282	OUT_RING(dev_priv->counter);
	283	OUT_RING(MI_USER_INTERRUPT);
7f3c3d6f HT	284	ADVANCE_LP_RING();
	285
	286	return dev_priv->counter;
	287	}
	288
b3705d71	289	void i915_user_irq_get(struct drm_device *dev)
7f3c3d6f	290	{
b3705d71 HT	291	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
b3705d71 HT	292
f77a46dc HT	293	if (dev->irq_enabled == 0)
	294	return;
	295
b3705d71	296	DRM_DEBUG("\n");
7f3c3d6f	297	DRM_SPINLOCK(&dev_priv->user_irq_lock);
f77a46dc	298	if (++dev_priv->user_irq_refcount == 1)
b3705d71	299	i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
7f3c3d6f	300	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
7f3c3d6f HT	301	}
7f3c3d6f HT	302
b3705d71	303	void i915_user_irq_put(struct drm_device *dev)
7f3c3d6f	304	{
b3705d71 HT	305	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
b3705d71 HT	306
f77a46dc HT	307	if (dev->irq_enabled == 0)
	308	return;
	309
7f3c3d6f	310	DRM_SPINLOCK(&dev_priv->user_irq_lock);
f77a46dc HT	311	KASSERT(dev_priv->user_irq_refcount > 0, ("invalid refcount"));
	312	if (--dev_priv->user_irq_refcount == 0)
	313	i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
7f3c3d6f HT	314	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
	315	}
	316
7f3c3d6f HT	317	static int i915_wait_irq(struct drm_device * dev, int irq_nr)
	318	{
	319	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
	320	int ret = 0;
	321
b3705d71 HT	322	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
	323	if (dev_priv->sarea_priv) {
	324	dev_priv->sarea_priv->last_dispatch =
	325	READ_BREADCRUMB(dev_priv);
	326	}
	327	return 0;
	328	}
	329
	330	if (dev_priv->sarea_priv)
	331	dev_priv->sarea_priv->perf_boxes \|= I915_BOX_WAIT;
	332
7f3c3d6f HT	333	DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
	334	READ_BREADCRUMB(dev_priv));
	335
b3705d71	336	i915_user_irq_get(dev);
7f3c3d6f HT	337	DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ,
7f3c3d6f HT	338	READ_BREADCRUMB(dev_priv) >= irq_nr);
b3705d71 HT	339	i915_user_irq_put(dev);
	340
	341	if (ret == -ERESTART)
	342	DRM_DEBUG("restarting syscall\n");
7f3c3d6f HT	343
	344	if (ret == -EBUSY) {
	345	DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
	346	READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
	347	}
	348
7f3c3d6f HT	349	return ret;
	350	}
	351
7f3c3d6f HT	352	/* Needs the lock as it touches the ring.
	353	*/
	354	int i915_irq_emit(struct drm_device dev, void data,
	355	struct drm_file *file_priv)
	356	{
	357	drm_i915_private_t *dev_priv = dev->dev_private;
	358	drm_i915_irq_emit_t *emit = data;
	359	int result;
	360
7f3c3d6f HT	361	if (!dev_priv) {
	362	DRM_ERROR("called with no initialization\n");
	363	return -EINVAL;
	364	}
	365
b3705d71 HT	366	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
b3705d71 HT	367
7f3c3d6f HT	368	result = i915_emit_irq(dev);
	369
	370	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
	371	DRM_ERROR("copy_to_user\n");
	372	return -EFAULT;
	373	}
	374
	375	return 0;
	376	}
	377
	378	/* Doesn't need the hardware lock.
	379	*/
	380	int i915_irq_wait(struct drm_device dev, void data,
b3705d71	381	struct drm_file *file_priv)
7f3c3d6f HT	382	{
	383	drm_i915_private_t *dev_priv = dev->dev_private;
	384	drm_i915_irq_wait_t *irqwait = data;
	385
	386	if (!dev_priv) {
	387	DRM_ERROR("called with no initialization\n");
	388	return -EINVAL;
	389	}
	390
	391	return i915_wait_irq(dev, irqwait->irq_seq);
	392	}
	393
b3705d71 HT	394	/* Called from drm generic code, passed 'crtc' which
	395	* we use as a pipe index
	396	*/
	397	int i915_enable_vblank(struct drm_device *dev, int pipe)
7f3c3d6f HT	398	{
7f3c3d6f HT	399	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
b3705d71	400
f77a46dc	401	if (!i915_pipe_enabled(dev, pipe))
b3705d71 HT	402	return -EINVAL;
	403
	404	DRM_SPINLOCK(&dev_priv->user_irq_lock);
	405	if (IS_I965G(dev))
	406	i915_enable_pipestat(dev_priv, pipe,
	407	PIPE_START_VBLANK_INTERRUPT_ENABLE);
	408	else
	409	i915_enable_pipestat(dev_priv, pipe,
	410	PIPE_VBLANK_INTERRUPT_ENABLE);
	411	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
	412	return 0;
	413	}
7f3c3d6f	414
b3705d71 HT	415	/* Called from drm generic code, passed 'crtc' which
	416	* we use as a pipe index
	417	*/
	418	void i915_disable_vblank(struct drm_device *dev, int pipe)
	419	{
	420	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
7f3c3d6f	421
b3705d71 HT	422	DRM_SPINLOCK(&dev_priv->user_irq_lock);
	423	i915_disable_pipestat(dev_priv, pipe,
	424	PIPE_VBLANK_INTERRUPT_ENABLE \|
	425	PIPE_START_VBLANK_INTERRUPT_ENABLE);
	426	DRM_SPINUNLOCK(&dev_priv->user_irq_lock);
7f3c3d6f HT	427	}
	428
	429	/* Set the vblank monitor pipe
	430	*/
	431	int i915_vblank_pipe_set(struct drm_device dev, void data,
	432	struct drm_file *file_priv)
	433	{
	434	drm_i915_private_t *dev_priv = dev->dev_private;
7f3c3d6f HT	435
	436	if (!dev_priv) {
	437	DRM_ERROR("called with no initialization\n");
	438	return -EINVAL;
	439	}
	440
7f3c3d6f HT	441	return 0;
	442	}
	443
	444	int i915_vblank_pipe_get(struct drm_device dev, void data,
	445	struct drm_file *file_priv)
	446	{
	447	drm_i915_private_t *dev_priv = dev->dev_private;
	448	drm_i915_vblank_pipe_t *pipe = data;
7f3c3d6f HT	449
	450	if (!dev_priv) {
	451	DRM_ERROR("called with no initialization\n");
	452	return -EINVAL;
	453	}
	454
b3705d71	455	pipe->pipe = DRM_I915_VBLANK_PIPE_A \| DRM_I915_VBLANK_PIPE_B;
7f3c3d6f HT	456
	457	return 0;
	458	}
	459
	460	/**
	461	* Schedule buffer swap at given vertical blank.
	462	*/
	463	int i915_vblank_swap(struct drm_device dev, void data,
	464	struct drm_file *file_priv)
	465	{
b3705d71 HT	466	/* The delayed swap mechanism was fundamentally racy, and has been
	467	* removed. The model was that the client requested a delayed flip/swap
	468	* from the kernel, then waited for vblank before continuing to perform
	469	* rendering. The problem was that the kernel might wake the client
	470	* up before it dispatched the vblank swap (since the lock has to be
	471	* held while touching the ringbuffer), in which case the client would
	472	* clear and start the next frame before the swap occurred, and
	473	* flicker would occur in addition to likely missing the vblank.
	474	*
	475	* In the absence of this ioctl, userland falls back to a correct path
	476	* of waiting for a vblank, then dispatching the swap on its own.
	477	* Context switching to userland and back is plenty fast enough for
	478	* meeting the requirements of vblank swapping.
7f3c3d6f	479	*/
b3705d71	480	return -EINVAL;
7f3c3d6f HT	481	}
	482
	483	/* drm_dma.h hooks
	484	*/
	485	void i915_driver_irq_preinstall(struct drm_device * dev)
	486	{
	487	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
	488
b3705d71 HT	489	I915_WRITE(HWSTAM, 0xeffe);
	490	I915_WRITE(PIPEASTAT, 0);
	491	I915_WRITE(PIPEBSTAT, 0);
	492	I915_WRITE(IMR, 0xffffffff);
	493	I915_WRITE(IER, 0x0);
	494	(void) I915_READ(IER);
7f3c3d6f HT	495	}
7f3c3d6f HT	496
b3705d71	497	int i915_driver_irq_postinstall(struct drm_device *dev)
7f3c3d6f HT	498	{
	499	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
	500
b3705d71	501	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A \| DRM_I915_VBLANK_PIPE_B;
7f3c3d6f	502
b3705d71 HT	503	/* Unmask the interrupts that we always want on. */
b3705d71 HT	504	dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;
7f3c3d6f	505
b3705d71 HT	506	/* Disable pipe interrupt enables, clear pending pipe status */
	507	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
	508	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
	509
	510	/* Clear pending interrupt status */
	511	I915_WRITE(IIR, I915_READ(IIR));
7f3c3d6f	512
b3705d71 HT	513	I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
b3705d71 HT	514	I915_WRITE(IMR, dev_priv->irq_mask_reg);
f77a46dc HT	515	I915_WRITE(PIPEASTAT, dev_priv->pipestat[0] \|
	516	(dev_priv->pipestat[0] >> 16));
	517	I915_WRITE(PIPEBSTAT, dev_priv->pipestat[1] \|
	518	(dev_priv->pipestat[1] >> 16));
b3705d71 HT	519	(void) I915_READ(IER);
	520
	521	return 0;
7f3c3d6f HT	522	}
	523
	524	void i915_driver_irq_uninstall(struct drm_device * dev)
	525	{
	526	drm_i915_private_t dev_priv = (drm_i915_private_t ) dev->dev_private;
7f3c3d6f HT	527
	528	if (!dev_priv)
	529	return;
	530
b3705d71 HT	531	dev_priv->vblank_pipe = 0;
	532
	533	I915_WRITE(HWSTAM, 0xffffffff);
	534	I915_WRITE(PIPEASTAT, 0);
	535	I915_WRITE(PIPEBSTAT, 0);
	536	I915_WRITE(IMR, 0xffffffff);
	537	I915_WRITE(IER, 0x0);
7f3c3d6f	538
b3705d71 HT	539	I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
	540	I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
	541	I915_WRITE(IIR, I915_READ(IIR));
7f3c3d6f	542	}