1 /* mach64_dma.c -- DMA support for mach64 (Rage Pro) driver -*- linux-c -*- */
4 * DMA support for mach64 (Rage Pro) driver
6 * \author Gareth Hughes <gareth@valinux.com>
7 * \author Frank C. Earl <fearl@airmail.net>
8 * \author Leif Delgass <ldelgass@retinalburn.net>
9 * \author José Fonseca <j_r_fonseca@yahoo.co.uk>
13 * Copyright 2000 Gareth Hughes
14 * Copyright 2002 Frank C. Earl
15 * Copyright 2002-2003 Leif Delgass
16 * All Rights Reserved.
18 * Permission is hereby granted, free of charge, to any person obtaining a
19 * copy of this software and associated documentation files (the "Software"),
20 * to deal in the Software without restriction, including without limitation
21 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
22 * and/or sell copies of the Software, and to permit persons to whom the
23 * Software is furnished to do so, subject to the following conditions:
25 * The above copyright notice and this permission notice (including the next
26 * paragraph) shall be included in all copies or substantial portions of the
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
32 * THE COPYRIGHT OWNER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
33 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
34 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 #include "mach64_drm.h"
39 #include "mach64_drv.h"
41 /*******************************************************************/
42 /** \name Engine, FIFO control */
46 * Waits for free entries in the FIFO.
48 * \note Most writes to Mach64 registers are automatically routed through
49 * command FIFO which is 16 entry deep. Prior to writing to any draw engine
50 * register one has to ensure that enough FIFO entries are available by calling
51 * this function. Failure to do so may cause the engine to lock.
53 * \param dev_priv pointer to device private data structure.
54 * \param entries number of free entries in the FIFO to wait for.
56 * \returns zero on success, or -EBUSY if the timeout (specificed by
57 * drm_mach64_private::usec_timeout) occurs.
59 int mach64_do_wait_for_fifo(drm_mach64_private_t *dev_priv, int entries)
63 for (i = 0; i < dev_priv->usec_timeout; i++) {
64 slots = (MACH64_READ(MACH64_FIFO_STAT) & MACH64_FIFO_SLOT_MASK);
65 if (slots <= (0x8000 >> entries))
70 DRM_INFO("failed! slots=%d entries=%d\n", slots, entries);
75 * Wait for the draw engine to be idle.
77 int mach64_do_wait_for_idle(drm_mach64_private_t *dev_priv)
81 ret = mach64_do_wait_for_fifo(dev_priv, 16);
85 for (i = 0; i < dev_priv->usec_timeout; i++) {
86 if (!(MACH64_READ(MACH64_GUI_STAT) & MACH64_GUI_ACTIVE))
91 DRM_INFO("failed! GUI_STAT=0x%08x\n", MACH64_READ(MACH64_GUI_STAT));
92 mach64_dump_ring_info(dev_priv);
97 * Wait for free entries in the ring buffer.
99 * The Mach64 bus master can be configured to act as a virtual FIFO, using a
100 * circular buffer (commonly referred as "ring buffer" in other drivers) with
101 * pointers to engine commands. This allows the CPU to do other things while
102 * the graphics engine is busy, i.e., DMA mode.
104 * This function should be called before writing new entries to the ring
107 * \param dev_priv pointer to device private data structure.
108 * \param n number of free entries in the ring buffer to wait for.
110 * \returns zero on success, or -EBUSY if the timeout (specificed by
111 * drm_mach64_private_t::usec_timeout) occurs.
113 * \sa mach64_dump_ring_info()
115 int mach64_wait_ring(drm_mach64_private_t *dev_priv, int n)
117 drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
120 for (i = 0; i < dev_priv->usec_timeout; i++) {
121 mach64_update_ring_snapshot(dev_priv);
122 if (ring->space >= n) {
124 DRM_DEBUG("%d usecs\n", i);
130 /* FIXME: This is being ignored... */
131 DRM_ERROR("failed!\n");
132 mach64_dump_ring_info(dev_priv);
137 * Wait until all DMA requests have been processed...
139 * \sa mach64_wait_ring()
141 static int mach64_ring_idle(drm_mach64_private_t *dev_priv)
143 drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
149 while (i < dev_priv->usec_timeout) {
150 mach64_update_ring_snapshot(dev_priv);
151 if (ring->head == ring->tail &&
152 !(MACH64_READ(MACH64_GUI_STAT) & MACH64_GUI_ACTIVE)) {
154 DRM_DEBUG("%d usecs\n", i);
157 if (ring->head == head) {
166 DRM_INFO("failed! GUI_STAT=0x%08x\n", MACH64_READ(MACH64_GUI_STAT));
167 mach64_dump_ring_info(dev_priv);
172 * Reset the ring buffer descriptors.
174 * \sa mach64_do_engine_reset()
176 static void mach64_ring_reset(drm_mach64_private_t *dev_priv)
178 drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
180 mach64_do_release_used_buffers(dev_priv);
181 ring->head_addr = ring->start_addr;
182 ring->head = ring->tail = 0;
183 ring->space = ring->size;
185 MACH64_WRITE(MACH64_BM_GUI_TABLE_CMD,
186 ring->head_addr | MACH64_CIRCULAR_BUF_SIZE_16KB);
188 dev_priv->ring_running = 0;
192 * Ensure the all the queued commands will be processed.
194 int mach64_do_dma_flush(drm_mach64_private_t *dev_priv)
196 /* FIXME: It's not necessary to wait for idle when flushing
197 * we just need to ensure the ring will be completely processed
198 * in finite time without another ioctl
200 return mach64_ring_idle(dev_priv);
204 * Stop all DMA activity.
206 int mach64_do_dma_idle(drm_mach64_private_t *dev_priv)
210 /* wait for completion */
211 if ((ret = mach64_ring_idle(dev_priv)) < 0) {
212 DRM_ERROR("failed BM_GUI_TABLE=0x%08x tail: %u\n",
213 MACH64_READ(MACH64_BM_GUI_TABLE),
214 dev_priv->ring.tail);
218 mach64_ring_stop(dev_priv);
220 /* clean up after pass */
221 mach64_do_release_used_buffers(dev_priv);
226 * Reset the engine. This will stop the DMA if it is running.
228 int mach64_do_engine_reset(drm_mach64_private_t *dev_priv)
234 /* Kill off any outstanding DMA transfers.
236 tmp = MACH64_READ(MACH64_BUS_CNTL);
237 MACH64_WRITE(MACH64_BUS_CNTL, tmp | MACH64_BUS_MASTER_DIS);
239 /* Reset the GUI engine (high to low transition).
241 tmp = MACH64_READ(MACH64_GEN_TEST_CNTL);
242 MACH64_WRITE(MACH64_GEN_TEST_CNTL, tmp & ~MACH64_GUI_ENGINE_ENABLE);
243 /* Enable the GUI engine
245 tmp = MACH64_READ(MACH64_GEN_TEST_CNTL);
246 MACH64_WRITE(MACH64_GEN_TEST_CNTL, tmp | MACH64_GUI_ENGINE_ENABLE);
248 /* ensure engine is not locked up by clearing any FIFO or HOST errors
250 tmp = MACH64_READ(MACH64_BUS_CNTL);
251 MACH64_WRITE(MACH64_BUS_CNTL, tmp | 0x00a00000);
253 /* Once GUI engine is restored, disable bus mastering */
254 MACH64_WRITE(MACH64_SRC_CNTL, 0);
256 /* Reset descriptor ring */
257 mach64_ring_reset(dev_priv);
265 /*******************************************************************/
266 /** \name Debugging output */
270 * Dump engine registers values.
272 void mach64_dump_engine_info(drm_mach64_private_t *dev_priv)
275 if (!dev_priv->is_pci) {
276 DRM_INFO(" AGP_BASE = 0x%08x\n",
277 MACH64_READ(MACH64_AGP_BASE));
278 DRM_INFO(" AGP_CNTL = 0x%08x\n",
279 MACH64_READ(MACH64_AGP_CNTL));
281 DRM_INFO(" ALPHA_TST_CNTL = 0x%08x\n",
282 MACH64_READ(MACH64_ALPHA_TST_CNTL));
284 DRM_INFO(" BM_COMMAND = 0x%08x\n",
285 MACH64_READ(MACH64_BM_COMMAND));
286 DRM_INFO("BM_FRAME_BUF_OFFSET = 0x%08x\n",
287 MACH64_READ(MACH64_BM_FRAME_BUF_OFFSET));
288 DRM_INFO(" BM_GUI_TABLE = 0x%08x\n",
289 MACH64_READ(MACH64_BM_GUI_TABLE));
290 DRM_INFO(" BM_STATUS = 0x%08x\n",
291 MACH64_READ(MACH64_BM_STATUS));
292 DRM_INFO(" BM_SYSTEM_MEM_ADDR = 0x%08x\n",
293 MACH64_READ(MACH64_BM_SYSTEM_MEM_ADDR));
294 DRM_INFO(" BM_SYSTEM_TABLE = 0x%08x\n",
295 MACH64_READ(MACH64_BM_SYSTEM_TABLE));
296 DRM_INFO(" BUS_CNTL = 0x%08x\n",
297 MACH64_READ(MACH64_BUS_CNTL));
299 /* DRM_INFO( " CLOCK_CNTL = 0x%08x\n", MACH64_READ( MACH64_CLOCK_CNTL ) ); */
300 DRM_INFO(" CLR_CMP_CLR = 0x%08x\n",
301 MACH64_READ(MACH64_CLR_CMP_CLR));
302 DRM_INFO(" CLR_CMP_CNTL = 0x%08x\n",
303 MACH64_READ(MACH64_CLR_CMP_CNTL));
304 /* DRM_INFO( " CLR_CMP_MSK = 0x%08x\n", MACH64_READ( MACH64_CLR_CMP_MSK ) ); */
305 DRM_INFO(" CONFIG_CHIP_ID = 0x%08x\n",
306 MACH64_READ(MACH64_CONFIG_CHIP_ID));
307 DRM_INFO(" CONFIG_CNTL = 0x%08x\n",
308 MACH64_READ(MACH64_CONFIG_CNTL));
309 DRM_INFO(" CONFIG_STAT0 = 0x%08x\n",
310 MACH64_READ(MACH64_CONFIG_STAT0));
311 DRM_INFO(" CONFIG_STAT1 = 0x%08x\n",
312 MACH64_READ(MACH64_CONFIG_STAT1));
313 DRM_INFO(" CONFIG_STAT2 = 0x%08x\n",
314 MACH64_READ(MACH64_CONFIG_STAT2));
315 DRM_INFO(" CRC_SIG = 0x%08x\n", MACH64_READ(MACH64_CRC_SIG));
316 DRM_INFO(" CUSTOM_MACRO_CNTL = 0x%08x\n",
317 MACH64_READ(MACH64_CUSTOM_MACRO_CNTL));
319 /* DRM_INFO( " DAC_CNTL = 0x%08x\n", MACH64_READ( MACH64_DAC_CNTL ) ); */
320 /* DRM_INFO( " DAC_REGS = 0x%08x\n", MACH64_READ( MACH64_DAC_REGS ) ); */
321 DRM_INFO(" DP_BKGD_CLR = 0x%08x\n",
322 MACH64_READ(MACH64_DP_BKGD_CLR));
323 DRM_INFO(" DP_FRGD_CLR = 0x%08x\n",
324 MACH64_READ(MACH64_DP_FRGD_CLR));
325 DRM_INFO(" DP_MIX = 0x%08x\n", MACH64_READ(MACH64_DP_MIX));
326 DRM_INFO(" DP_PIX_WIDTH = 0x%08x\n",
327 MACH64_READ(MACH64_DP_PIX_WIDTH));
328 DRM_INFO(" DP_SRC = 0x%08x\n", MACH64_READ(MACH64_DP_SRC));
329 DRM_INFO(" DP_WRITE_MASK = 0x%08x\n",
330 MACH64_READ(MACH64_DP_WRITE_MASK));
331 DRM_INFO(" DSP_CONFIG = 0x%08x\n",
332 MACH64_READ(MACH64_DSP_CONFIG));
333 DRM_INFO(" DSP_ON_OFF = 0x%08x\n",
334 MACH64_READ(MACH64_DSP_ON_OFF));
335 DRM_INFO(" DST_CNTL = 0x%08x\n",
336 MACH64_READ(MACH64_DST_CNTL));
337 DRM_INFO(" DST_OFF_PITCH = 0x%08x\n",
338 MACH64_READ(MACH64_DST_OFF_PITCH));
340 /* DRM_INFO( " EXT_DAC_REGS = 0x%08x\n", MACH64_READ( MACH64_EXT_DAC_REGS ) ); */
341 DRM_INFO(" EXT_MEM_CNTL = 0x%08x\n",
342 MACH64_READ(MACH64_EXT_MEM_CNTL));
344 DRM_INFO(" FIFO_STAT = 0x%08x\n",
345 MACH64_READ(MACH64_FIFO_STAT));
347 DRM_INFO(" GEN_TEST_CNTL = 0x%08x\n",
348 MACH64_READ(MACH64_GEN_TEST_CNTL));
349 /* DRM_INFO( " GP_IO = 0x%08x\n", MACH64_READ( MACH64_GP_IO ) ); */
350 DRM_INFO(" GUI_CMDFIFO_DATA = 0x%08x\n",
351 MACH64_READ(MACH64_GUI_CMDFIFO_DATA));
352 DRM_INFO(" GUI_CMDFIFO_DEBUG = 0x%08x\n",
353 MACH64_READ(MACH64_GUI_CMDFIFO_DEBUG));
354 DRM_INFO(" GUI_CNTL = 0x%08x\n",
355 MACH64_READ(MACH64_GUI_CNTL));
356 DRM_INFO(" GUI_STAT = 0x%08x\n",
357 MACH64_READ(MACH64_GUI_STAT));
358 DRM_INFO(" GUI_TRAJ_CNTL = 0x%08x\n",
359 MACH64_READ(MACH64_GUI_TRAJ_CNTL));
361 DRM_INFO(" HOST_CNTL = 0x%08x\n",
362 MACH64_READ(MACH64_HOST_CNTL));
363 DRM_INFO(" HW_DEBUG = 0x%08x\n",
364 MACH64_READ(MACH64_HW_DEBUG));
366 DRM_INFO(" MEM_ADDR_CONFIG = 0x%08x\n",
367 MACH64_READ(MACH64_MEM_ADDR_CONFIG));
368 DRM_INFO(" MEM_BUF_CNTL = 0x%08x\n",
369 MACH64_READ(MACH64_MEM_BUF_CNTL));
371 DRM_INFO(" PAT_REG0 = 0x%08x\n",
372 MACH64_READ(MACH64_PAT_REG0));
373 DRM_INFO(" PAT_REG1 = 0x%08x\n",
374 MACH64_READ(MACH64_PAT_REG1));
376 DRM_INFO(" SC_LEFT = 0x%08x\n", MACH64_READ(MACH64_SC_LEFT));
377 DRM_INFO(" SC_RIGHT = 0x%08x\n",
378 MACH64_READ(MACH64_SC_RIGHT));
379 DRM_INFO(" SC_TOP = 0x%08x\n", MACH64_READ(MACH64_SC_TOP));
380 DRM_INFO(" SC_BOTTOM = 0x%08x\n",
381 MACH64_READ(MACH64_SC_BOTTOM));
383 DRM_INFO(" SCALE_3D_CNTL = 0x%08x\n",
384 MACH64_READ(MACH64_SCALE_3D_CNTL));
385 DRM_INFO(" SCRATCH_REG0 = 0x%08x\n",
386 MACH64_READ(MACH64_SCRATCH_REG0));
387 DRM_INFO(" SCRATCH_REG1 = 0x%08x\n",
388 MACH64_READ(MACH64_SCRATCH_REG1));
389 DRM_INFO(" SETUP_CNTL = 0x%08x\n",
390 MACH64_READ(MACH64_SETUP_CNTL));
391 DRM_INFO(" SRC_CNTL = 0x%08x\n",
392 MACH64_READ(MACH64_SRC_CNTL));
394 DRM_INFO(" TEX_CNTL = 0x%08x\n",
395 MACH64_READ(MACH64_TEX_CNTL));
396 DRM_INFO(" TEX_SIZE_PITCH = 0x%08x\n",
397 MACH64_READ(MACH64_TEX_SIZE_PITCH));
398 DRM_INFO(" TIMER_CONFIG = 0x%08x\n",
399 MACH64_READ(MACH64_TIMER_CONFIG));
401 DRM_INFO(" Z_CNTL = 0x%08x\n", MACH64_READ(MACH64_Z_CNTL));
402 DRM_INFO(" Z_OFF_PITCH = 0x%08x\n",
403 MACH64_READ(MACH64_Z_OFF_PITCH));
407 #define MACH64_DUMP_CONTEXT 3
410 * Used by mach64_dump_ring_info() to dump the contents of the current buffer
411 * pointed by the ring head.
413 static void mach64_dump_buf_info(drm_mach64_private_t *dev_priv,
416 u32 addr = GETBUFADDR(buf);
417 u32 used = buf->used >> 2;
418 u32 sys_addr = MACH64_READ(MACH64_BM_SYSTEM_MEM_ADDR);
419 u32 *p = GETBUFPTR(buf);
422 DRM_INFO("buffer contents:\n");
427 reg = le32_to_cpu(*p++);
428 if (addr <= GETBUFADDR(buf) + MACH64_DUMP_CONTEXT * 4 ||
429 (addr >= sys_addr - MACH64_DUMP_CONTEXT * 4 &&
430 addr <= sys_addr + MACH64_DUMP_CONTEXT * 4) ||
432 GETBUFADDR(buf) + buf->used - MACH64_DUMP_CONTEXT * 4) {
433 DRM_INFO("%08x: 0x%08x\n", addr, reg);
438 count = (reg >> 16) + 1;
441 while (count && used) {
442 if (addr <= GETBUFADDR(buf) + MACH64_DUMP_CONTEXT * 4 ||
443 (addr >= sys_addr - MACH64_DUMP_CONTEXT * 4 &&
444 addr <= sys_addr + MACH64_DUMP_CONTEXT * 4) ||
446 GETBUFADDR(buf) + buf->used -
447 MACH64_DUMP_CONTEXT * 4) {
448 DRM_INFO("%08x: 0x%04x = 0x%08x\n", addr,
449 reg, le32_to_cpu(*p));
470 * Dump the ring state and contents, including the contents of the buffer being
471 * processed by the graphics engine.
473 void mach64_dump_ring_info(drm_mach64_private_t *dev_priv)
475 drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
480 DRM_INFO("ring contents:\n");
481 DRM_INFO(" head_addr: 0x%08x head: %u tail: %u\n\n",
482 ring->head_addr, ring->head, ring->tail);
485 for (i = 0; i < ring->size / sizeof(u32); i += 4) {
486 if (i <= MACH64_DUMP_CONTEXT * 4 ||
487 i >= ring->size / sizeof(u32) - MACH64_DUMP_CONTEXT * 4 ||
488 (i >= ring->tail - MACH64_DUMP_CONTEXT * 4 &&
489 i <= ring->tail + MACH64_DUMP_CONTEXT * 4) ||
490 (i >= ring->head - MACH64_DUMP_CONTEXT * 4 &&
491 i <= ring->head + MACH64_DUMP_CONTEXT * 4)) {
492 DRM_INFO(" 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x%s%s\n",
493 (u32)(ring->start_addr + i * sizeof(u32)),
494 le32_to_cpu(((u32 *) ring->start)[i + 0]),
495 le32_to_cpu(((u32 *) ring->start)[i + 1]),
496 le32_to_cpu(((u32 *) ring->start)[i + 2]),
497 le32_to_cpu(((u32 *) ring->start)[i + 3]),
498 i == ring->head ? " (head)" : "",
499 i == ring->tail ? " (tail)" : "");
511 if (ring->head < ring->size / sizeof(u32)) {
512 struct list_head *ptr;
513 u32 addr = le32_to_cpu(((u32 *) ring->start)[ring->head + 1]);
515 list_for_each(ptr, &dev_priv->pending) {
516 drm_mach64_freelist_t *entry =
517 list_entry(ptr, drm_mach64_freelist_t, list);
518 struct drm_buf *buf = entry->buf;
520 u32 buf_addr = GETBUFADDR(buf);
522 if (buf_addr <= addr && addr < buf_addr + buf->used)
523 mach64_dump_buf_info(dev_priv, buf);
528 DRM_INFO(" BM_GUI_TABLE = 0x%08x\n",
529 MACH64_READ(MACH64_BM_GUI_TABLE));
531 DRM_INFO("BM_FRAME_BUF_OFFSET = 0x%08x\n",
532 MACH64_READ(MACH64_BM_FRAME_BUF_OFFSET));
533 DRM_INFO(" BM_SYSTEM_MEM_ADDR = 0x%08x\n",
534 MACH64_READ(MACH64_BM_SYSTEM_MEM_ADDR));
535 DRM_INFO(" BM_COMMAND = 0x%08x\n",
536 MACH64_READ(MACH64_BM_COMMAND));
538 DRM_INFO(" BM_STATUS = 0x%08x\n",
539 MACH64_READ(MACH64_BM_STATUS));
540 DRM_INFO(" BUS_CNTL = 0x%08x\n",
541 MACH64_READ(MACH64_BUS_CNTL));
542 DRM_INFO(" FIFO_STAT = 0x%08x\n",
543 MACH64_READ(MACH64_FIFO_STAT));
544 DRM_INFO(" GUI_STAT = 0x%08x\n",
545 MACH64_READ(MACH64_GUI_STAT));
546 DRM_INFO(" SRC_CNTL = 0x%08x\n",
547 MACH64_READ(MACH64_SRC_CNTL));
553 /*******************************************************************/
554 /** \name DMA descriptor ring macros */
558 * Add the end mark to the ring's new tail position.
560 * The bus master engine will keep processing the DMA buffers listed in the ring
561 * until it finds this mark, making it stop.
563 * \sa mach64_clear_dma_eol
565 static __inline__ void mach64_set_dma_eol(volatile u32 *addr)
567 #if defined(__i386__)
570 /* Taken from include/asm-i386/bitops.h linux header */
571 __asm__ __volatile__("lock;" "btsl %1,%0":"=m"(*addr)
573 #elif defined(__powerpc__)
575 u32 mask = cpu_to_le32(MACH64_DMA_EOL);
577 /* Taken from the include/asm-ppc/bitops.h linux header */
578 __asm__ __volatile__("\n\
582 bne- 1b":"=&r"(old), "=m"(*addr)
583 :"r"(mask), "r"(addr), "m"(*addr)
585 #elif defined(__alpha__)
587 u32 mask = MACH64_DMA_EOL;
589 /* Taken from the include/asm-alpha/bitops.h linux header */
590 __asm__ __volatile__("1: ldl_l %0,%3\n"
596 ".previous":"=&r"(temp), "=m"(*addr)
597 :"Ir"(mask), "m"(*addr));
599 u32 mask = cpu_to_le32(MACH64_DMA_EOL);
606 * Remove the end mark from the ring's old tail position.
608 * It should be called after calling mach64_set_dma_eol to mark the ring's new
611 * We update the end marks while the bus master engine is in operation. Since
612 * the bus master engine may potentially be reading from the same position
613 * that we write, we must change atomically to avoid having intermediary bad
616 static __inline__ void mach64_clear_dma_eol(volatile u32 *addr)
618 #if defined(__i386__)
621 /* Taken from include/asm-i386/bitops.h linux header */
622 __asm__ __volatile__("lock;" "btrl %1,%0":"=m"(*addr)
624 #elif defined(__powerpc__)
626 u32 mask = cpu_to_le32(MACH64_DMA_EOL);
628 /* Taken from the include/asm-ppc/bitops.h linux header */
629 __asm__ __volatile__("\n\
633 bne- 1b":"=&r"(old), "=m"(*addr)
634 :"r"(mask), "r"(addr), "m"(*addr)
636 #elif defined(__alpha__)
638 u32 mask = ~MACH64_DMA_EOL;
640 /* Taken from the include/asm-alpha/bitops.h linux header */
641 __asm__ __volatile__("1: ldl_l %0,%3\n"
647 ".previous":"=&r"(temp), "=m"(*addr)
648 :"Ir"(mask), "m"(*addr));
650 u32 mask = cpu_to_le32(~MACH64_DMA_EOL);
656 #define RING_LOCALS \
657 int _ring_tail, _ring_write; unsigned int _ring_mask; volatile u32 *_ring
659 #define RING_WRITE_OFS _ring_write
661 #define BEGIN_RING(n) \
663 if (MACH64_VERBOSE) { \
664 DRM_INFO( "BEGIN_RING( %d ) \n", \
667 if (dev_priv->ring.space <= (n) * sizeof(u32)) { \
669 if ((ret = mach64_wait_ring( dev_priv, (n) * sizeof(u32))) < 0 ) { \
670 DRM_ERROR( "wait_ring failed, resetting engine\n"); \
671 mach64_dump_engine_info( dev_priv ); \
672 mach64_do_engine_reset( dev_priv ); \
676 dev_priv->ring.space -= (n) * sizeof(u32); \
677 _ring = (u32 *) dev_priv->ring.start; \
678 _ring_tail = _ring_write = dev_priv->ring.tail; \
679 _ring_mask = dev_priv->ring.tail_mask; \
682 #define OUT_RING( x ) \
684 if (MACH64_VERBOSE) { \
685 DRM_INFO( " OUT_RING( 0x%08x ) at 0x%x\n", \
686 (unsigned int)(x), _ring_write ); \
688 _ring[_ring_write++] = cpu_to_le32( x ); \
689 _ring_write &= _ring_mask; \
692 #define ADVANCE_RING() \
694 if (MACH64_VERBOSE) { \
695 DRM_INFO( "ADVANCE_RING() wr=0x%06x tail=0x%06x\n", \
696 _ring_write, _ring_tail ); \
698 DRM_MEMORYBARRIER(); \
699 mach64_clear_dma_eol( &_ring[(_ring_tail - 2) & _ring_mask] ); \
700 DRM_MEMORYBARRIER(); \
701 dev_priv->ring.tail = _ring_write; \
702 mach64_ring_tick( dev_priv, &(dev_priv)->ring ); \
706 * Queue a DMA buffer of registers writes into the ring buffer.
708 int mach64_add_buf_to_ring(drm_mach64_private_t *dev_priv,
709 drm_mach64_freelist_t *entry)
711 int bytes, pages, remainder;
714 struct drm_buf *buf = entry->buf;
718 address = GETBUFADDR( buf );
719 pages = (bytes + MACH64_DMA_CHUNKSIZE - 1) / MACH64_DMA_CHUNKSIZE;
721 BEGIN_RING( pages * 4 );
723 for ( i = 0 ; i < pages-1 ; i++ ) {
724 page = address + i * MACH64_DMA_CHUNKSIZE;
725 OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );
727 OUT_RING( MACH64_DMA_CHUNKSIZE | MACH64_DMA_HOLD_OFFSET );
731 /* generate the final descriptor for any remaining commands in this buffer */
732 page = address + i * MACH64_DMA_CHUNKSIZE;
733 remainder = bytes - i * MACH64_DMA_CHUNKSIZE;
735 /* Save dword offset of last descriptor for this buffer.
736 * This is needed to check for completion of the buffer in freelist_get
738 entry->ring_ofs = RING_WRITE_OFS;
740 OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );
742 OUT_RING( remainder | MACH64_DMA_HOLD_OFFSET | MACH64_DMA_EOL );
751 * Queue DMA buffer controlling host data tranfers (e.g., blit).
753 * Almost identical to mach64_add_buf_to_ring.
755 int mach64_add_hostdata_buf_to_ring(drm_mach64_private_t *dev_priv,
756 drm_mach64_freelist_t *entry)
758 int bytes, pages, remainder;
761 struct drm_buf *buf = entry->buf;
764 bytes = buf->used - MACH64_HOSTDATA_BLIT_OFFSET;
765 pages = (bytes + MACH64_DMA_CHUNKSIZE - 1) / MACH64_DMA_CHUNKSIZE;
766 address = GETBUFADDR( buf );
768 BEGIN_RING( 4 + pages * 4 );
770 OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_ADDR );
772 OUT_RING( MACH64_HOSTDATA_BLIT_OFFSET | MACH64_DMA_HOLD_OFFSET );
774 address += MACH64_HOSTDATA_BLIT_OFFSET;
776 for ( i = 0 ; i < pages-1 ; i++ ) {
777 page = address + i * MACH64_DMA_CHUNKSIZE;
778 OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_HOSTDATA );
780 OUT_RING( MACH64_DMA_CHUNKSIZE | MACH64_DMA_HOLD_OFFSET );
784 /* generate the final descriptor for any remaining commands in this buffer */
785 page = address + i * MACH64_DMA_CHUNKSIZE;
786 remainder = bytes - i * MACH64_DMA_CHUNKSIZE;
788 /* Save dword offset of last descriptor for this buffer.
789 * This is needed to check for completion of the buffer in freelist_get
791 entry->ring_ofs = RING_WRITE_OFS;
793 OUT_RING( MACH64_APERTURE_OFFSET + MACH64_BM_HOSTDATA );
795 OUT_RING( remainder | MACH64_DMA_HOLD_OFFSET | MACH64_DMA_EOL );
806 /*******************************************************************/
807 /** \name DMA test and initialization */
811 * Perform a simple DMA operation using the pattern registers to test whether
814 * \return zero if successful.
816 * \note This function was the testbed for many experiences regarding Mach64
817 * DMA operation. It is left here since it so tricky to get DMA operating
818 * properly in some architectures and hardware.
820 static int mach64_bm_dma_test(struct drm_device * dev)
822 drm_mach64_private_t *dev_priv = dev->dev_private;
823 drm_dma_handle_t *cpu_addr_dmah;
827 u32 src_cntl, pat_reg0, pat_reg1;
828 int i, count, failed;
832 table = (u32 *) dev_priv->ring.start;
834 /* FIXME: get a dma buffer from the freelist here */
835 DRM_DEBUG("Allocating data memory ...\n");
836 #if defined(__FreeBSD__) || defined(__DragonFly__)
840 drm_pci_alloc(dev, 0x1000, 0x1000, 0xfffffffful);
841 #if defined(__FreeBSD__) || defined(__DragonFly__)
844 if (!cpu_addr_dmah) {
845 DRM_INFO("data-memory allocation failed!\n");
848 data = (u32 *) cpu_addr_dmah->vaddr;
849 data_addr = (u32) cpu_addr_dmah->busaddr;
852 /* Save the X server's value for SRC_CNTL and restore it
853 * in case our test fails. This prevents the X server
854 * from disabling it's cache for this register
856 src_cntl = MACH64_READ(MACH64_SRC_CNTL);
857 pat_reg0 = MACH64_READ(MACH64_PAT_REG0);
858 pat_reg1 = MACH64_READ(MACH64_PAT_REG1);
860 mach64_do_wait_for_fifo(dev_priv, 3);
862 MACH64_WRITE(MACH64_SRC_CNTL, 0);
863 MACH64_WRITE(MACH64_PAT_REG0, 0x11111111);
864 MACH64_WRITE(MACH64_PAT_REG1, 0x11111111);
866 mach64_do_wait_for_idle(dev_priv);
868 for (i = 0; i < 2; i++) {
870 reg = MACH64_READ((MACH64_PAT_REG0 + i * 4));
871 DRM_DEBUG("(Before DMA Transfer) reg %d = 0x%08x\n", i, reg);
872 if (reg != 0x11111111) {
873 DRM_INFO("Error initializing test registers\n");
874 DRM_INFO("resetting engine ...\n");
875 mach64_do_engine_reset(dev_priv);
876 DRM_INFO("freeing data buffer memory.\n");
877 drm_pci_free(dev, cpu_addr_dmah);
882 /* fill up a buffer with sets of 2 consecutive writes starting with PAT_REG0 */
885 data[count++] = cpu_to_le32(DMAREG(MACH64_PAT_REG0) | (1 << 16));
886 data[count++] = expected[0] = 0x22222222;
887 data[count++] = expected[1] = 0xaaaaaaaa;
889 while (count < 1020) {
891 cpu_to_le32(DMAREG(MACH64_PAT_REG0) | (1 << 16));
892 data[count++] = 0x22222222;
893 data[count++] = 0xaaaaaaaa;
895 data[count++] = cpu_to_le32(DMAREG(MACH64_SRC_CNTL) | (0 << 16));
898 DRM_DEBUG("Preparing table ...\n");
899 table[MACH64_DMA_FRAME_BUF_OFFSET] = cpu_to_le32(MACH64_BM_ADDR +
900 MACH64_APERTURE_OFFSET);
901 table[MACH64_DMA_SYS_MEM_ADDR] = cpu_to_le32(data_addr);
902 table[MACH64_DMA_COMMAND] = cpu_to_le32(count * sizeof(u32)
903 | MACH64_DMA_HOLD_OFFSET
905 table[MACH64_DMA_RESERVED] = 0;
907 DRM_DEBUG("table[0] = 0x%08x\n", table[0]);
908 DRM_DEBUG("table[1] = 0x%08x\n", table[1]);
909 DRM_DEBUG("table[2] = 0x%08x\n", table[2]);
910 DRM_DEBUG("table[3] = 0x%08x\n", table[3]);
912 for (i = 0; i < 6; i++) {
913 DRM_DEBUG(" data[%d] = 0x%08x\n", i, data[i]);
916 for (i = count - 5; i < count; i++) {
917 DRM_DEBUG(" data[%d] = 0x%08x\n", i, data[i]);
922 DRM_DEBUG("waiting for idle...\n");
923 if ((i = mach64_do_wait_for_idle(dev_priv))) {
924 DRM_INFO("mach64_do_wait_for_idle failed (result=%d)\n", i);
925 DRM_INFO("resetting engine ...\n");
926 mach64_do_engine_reset(dev_priv);
927 mach64_do_wait_for_fifo(dev_priv, 3);
928 MACH64_WRITE(MACH64_SRC_CNTL, src_cntl);
929 MACH64_WRITE(MACH64_PAT_REG0, pat_reg0);
930 MACH64_WRITE(MACH64_PAT_REG1, pat_reg1);
931 DRM_INFO("freeing data buffer memory.\n");
932 drm_pci_free(dev, cpu_addr_dmah);
935 DRM_DEBUG("waiting for idle...done\n");
937 DRM_DEBUG("BUS_CNTL = 0x%08x\n", MACH64_READ(MACH64_BUS_CNTL));
938 DRM_DEBUG("SRC_CNTL = 0x%08x\n", MACH64_READ(MACH64_SRC_CNTL));
940 DRM_DEBUG("data bus addr = 0x%08x\n", data_addr);
941 DRM_DEBUG("table bus addr = 0x%08x\n", dev_priv->ring.start_addr);
943 DRM_DEBUG("starting DMA transfer...\n");
944 MACH64_WRITE(MACH64_BM_GUI_TABLE_CMD,
945 dev_priv->ring.start_addr | MACH64_CIRCULAR_BUF_SIZE_16KB);
947 MACH64_WRITE(MACH64_SRC_CNTL,
948 MACH64_SRC_BM_ENABLE | MACH64_SRC_BM_SYNC |
949 MACH64_SRC_BM_OP_SYSTEM_TO_REG);
951 /* Kick off the transfer */
952 DRM_DEBUG("starting DMA transfer... done.\n");
953 MACH64_WRITE(MACH64_DST_HEIGHT_WIDTH, 0);
955 DRM_DEBUG("waiting for idle...\n");
957 if ((i = mach64_do_wait_for_idle(dev_priv))) {
958 /* engine locked up, dump register state and reset */
959 DRM_INFO("mach64_do_wait_for_idle failed (result=%d)\n", i);
960 mach64_dump_engine_info(dev_priv);
961 DRM_INFO("resetting engine ...\n");
962 mach64_do_engine_reset(dev_priv);
963 mach64_do_wait_for_fifo(dev_priv, 3);
964 MACH64_WRITE(MACH64_SRC_CNTL, src_cntl);
965 MACH64_WRITE(MACH64_PAT_REG0, pat_reg0);
966 MACH64_WRITE(MACH64_PAT_REG1, pat_reg1);
967 DRM_INFO("freeing data buffer memory.\n");
968 drm_pci_free(dev, cpu_addr_dmah);
972 DRM_DEBUG("waiting for idle...done\n");
974 /* restore SRC_CNTL */
975 mach64_do_wait_for_fifo(dev_priv, 1);
976 MACH64_WRITE(MACH64_SRC_CNTL, src_cntl);
980 /* Check register values to see if the GUI master operation succeeded */
981 for (i = 0; i < 2; i++) {
983 reg = MACH64_READ((MACH64_PAT_REG0 + i * 4));
984 DRM_DEBUG("(After DMA Transfer) reg %d = 0x%08x\n", i, reg);
985 if (reg != expected[i]) {
990 /* restore pattern registers */
991 mach64_do_wait_for_fifo(dev_priv, 2);
992 MACH64_WRITE(MACH64_PAT_REG0, pat_reg0);
993 MACH64_WRITE(MACH64_PAT_REG1, pat_reg1);
995 DRM_DEBUG("freeing data buffer memory.\n");
996 drm_pci_free(dev, cpu_addr_dmah);
997 DRM_DEBUG("returning ...\n");
1003 * Called during the DMA initialization ioctl to initialize all the necessary
1004 * software and hardware state for DMA operation.
1006 static int mach64_do_dma_init(struct drm_device * dev, drm_mach64_init_t * init)
1008 drm_mach64_private_t *dev_priv;
1014 dev_priv = drm_alloc(sizeof(drm_mach64_private_t), DRM_MEM_DRIVER);
1015 if (dev_priv == NULL)
1018 memset(dev_priv, 0, sizeof(drm_mach64_private_t));
1020 dev_priv->is_pci = init->is_pci;
1022 dev_priv->fb_bpp = init->fb_bpp;
1023 dev_priv->front_offset = init->front_offset;
1024 dev_priv->front_pitch = init->front_pitch;
1025 dev_priv->back_offset = init->back_offset;
1026 dev_priv->back_pitch = init->back_pitch;
1028 dev_priv->depth_bpp = init->depth_bpp;
1029 dev_priv->depth_offset = init->depth_offset;
1030 dev_priv->depth_pitch = init->depth_pitch;
1032 dev_priv->front_offset_pitch = (((dev_priv->front_pitch / 8) << 22) |
1033 (dev_priv->front_offset >> 3));
1034 dev_priv->back_offset_pitch = (((dev_priv->back_pitch / 8) << 22) |
1035 (dev_priv->back_offset >> 3));
1036 dev_priv->depth_offset_pitch = (((dev_priv->depth_pitch / 8) << 22) |
1037 (dev_priv->depth_offset >> 3));
1039 dev_priv->usec_timeout = 1000000;
1041 /* Set up the freelist, placeholder list and pending list */
1042 INIT_LIST_HEAD(&dev_priv->free_list);
1043 INIT_LIST_HEAD(&dev_priv->placeholders);
1044 INIT_LIST_HEAD(&dev_priv->pending);
1046 dev_priv->sarea = drm_getsarea(dev);
1047 if (!dev_priv->sarea) {
1048 DRM_ERROR("can not find sarea!\n");
1049 dev->dev_private = (void *)dev_priv;
1050 mach64_do_cleanup_dma(dev);
1053 dev_priv->fb = drm_core_findmap(dev, init->fb_offset);
1054 if (!dev_priv->fb) {
1055 DRM_ERROR("can not find frame buffer map!\n");
1056 dev->dev_private = (void *)dev_priv;
1057 mach64_do_cleanup_dma(dev);
1060 dev_priv->mmio = drm_core_findmap(dev, init->mmio_offset);
1061 if (!dev_priv->mmio) {
1062 DRM_ERROR("can not find mmio map!\n");
1063 dev->dev_private = (void *)dev_priv;
1064 mach64_do_cleanup_dma(dev);
1068 dev_priv->ring_map = drm_core_findmap(dev, init->ring_offset);
1069 if (!dev_priv->ring_map) {
1070 DRM_ERROR("can not find ring map!\n");
1071 dev->dev_private = (void *)dev_priv;
1072 mach64_do_cleanup_dma(dev);
1076 dev_priv->sarea_priv = (drm_mach64_sarea_t *)
1077 ((u8 *) dev_priv->sarea->virtual + init->sarea_priv_offset);
1079 if (!dev_priv->is_pci) {
1080 drm_core_ioremap(dev_priv->ring_map, dev);
1081 if (!dev_priv->ring_map->virtual) {
1082 DRM_ERROR("can not ioremap virtual address for"
1083 " descriptor ring\n");
1084 dev->dev_private = (void *)dev_priv;
1085 mach64_do_cleanup_dma(dev);
1088 dev->agp_buffer_token = init->buffers_offset;
1089 dev->agp_buffer_map =
1090 drm_core_findmap(dev, init->buffers_offset);
1091 if (!dev->agp_buffer_map) {
1092 DRM_ERROR("can not find dma buffer map!\n");
1093 dev->dev_private = (void *)dev_priv;
1094 mach64_do_cleanup_dma(dev);
1097 /* there might be a nicer way to do this -
1098 dev isn't passed all the way though the mach64 - DA */
1099 dev_priv->dev_buffers = dev->agp_buffer_map;
1101 drm_core_ioremap(dev->agp_buffer_map, dev);
1102 if (!dev->agp_buffer_map->virtual) {
1103 DRM_ERROR("can not ioremap virtual address for"
1105 dev->dev_private = (void *)dev_priv;
1106 mach64_do_cleanup_dma(dev);
1109 dev_priv->agp_textures =
1110 drm_core_findmap(dev, init->agp_textures_offset);
1111 if (!dev_priv->agp_textures) {
1112 DRM_ERROR("can not find agp texture region!\n");
1113 dev->dev_private = (void *)dev_priv;
1114 mach64_do_cleanup_dma(dev);
1119 dev->dev_private = (void *)dev_priv;
1121 dev_priv->driver_mode = init->dma_mode;
1123 /* changing the FIFO size from the default causes problems with DMA */
1124 tmp = MACH64_READ(MACH64_GUI_CNTL);
1125 if ((tmp & MACH64_CMDFIFO_SIZE_MASK) != MACH64_CMDFIFO_SIZE_128) {
1126 DRM_INFO("Setting FIFO size to 128 entries\n");
1127 /* FIFO must be empty to change the FIFO depth */
1128 if ((ret = mach64_do_wait_for_idle(dev_priv))) {
1130 ("wait for idle failed before changing FIFO depth!\n");
1131 mach64_do_cleanup_dma(dev);
1134 MACH64_WRITE(MACH64_GUI_CNTL, ((tmp & ~MACH64_CMDFIFO_SIZE_MASK)
1135 | MACH64_CMDFIFO_SIZE_128));
1136 /* need to read GUI_STAT for proper sync according to docs */
1137 if ((ret = mach64_do_wait_for_idle(dev_priv))) {
1139 ("wait for idle failed when changing FIFO depth!\n");
1140 mach64_do_cleanup_dma(dev);
1145 dev_priv->ring.size = 0x4000; /* 16KB */
1146 dev_priv->ring.start = dev_priv->ring_map->virtual;
1147 dev_priv->ring.start_addr = (u32) dev_priv->ring_map->offset;
1149 memset(dev_priv->ring.start, 0, dev_priv->ring.size);
1150 DRM_INFO("descriptor ring: cpu addr %p, bus addr: 0x%08x\n",
1151 dev_priv->ring.start, dev_priv->ring.start_addr);
1154 if (dev_priv->driver_mode != MACH64_MODE_MMIO) {
1156 /* enable block 1 registers and bus mastering */
1157 MACH64_WRITE(MACH64_BUS_CNTL, ((MACH64_READ(MACH64_BUS_CNTL)
1158 | MACH64_BUS_EXT_REG_EN)
1159 & ~MACH64_BUS_MASTER_DIS));
1161 /* try a DMA GUI-mastering pass and fall back to MMIO if it fails */
1162 DRM_DEBUG("Starting DMA test...\n");
1163 if ((ret = mach64_bm_dma_test(dev))) {
1164 dev_priv->driver_mode = MACH64_MODE_MMIO;
1168 switch (dev_priv->driver_mode) {
1169 case MACH64_MODE_MMIO:
1170 MACH64_WRITE(MACH64_BUS_CNTL, (MACH64_READ(MACH64_BUS_CNTL)
1171 | MACH64_BUS_EXT_REG_EN
1172 | MACH64_BUS_MASTER_DIS));
1173 if (init->dma_mode == MACH64_MODE_MMIO)
1174 DRM_INFO("Forcing pseudo-DMA mode\n");
1177 ("DMA test failed (ret=%d), using pseudo-DMA mode\n",
1180 case MACH64_MODE_DMA_SYNC:
1181 DRM_INFO("DMA test succeeded, using synchronous DMA mode\n");
1183 case MACH64_MODE_DMA_ASYNC:
1185 DRM_INFO("DMA test succeeded, using asynchronous DMA mode\n");
1188 dev_priv->ring_running = 0;
1190 /* setup offsets for physical address of table start and end */
1191 dev_priv->ring.head_addr = dev_priv->ring.start_addr;
1192 dev_priv->ring.head = dev_priv->ring.tail = 0;
1193 dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1194 dev_priv->ring.space = dev_priv->ring.size;
1196 /* setup physical address and size of descriptor table */
1197 mach64_do_wait_for_fifo(dev_priv, 1);
1198 MACH64_WRITE(MACH64_BM_GUI_TABLE_CMD,
1200 head_addr | MACH64_CIRCULAR_BUF_SIZE_16KB));
1202 /* init frame counter */
1203 dev_priv->sarea_priv->frames_queued = 0;
1204 for (i = 0; i < MACH64_MAX_QUEUED_FRAMES; i++) {
1205 dev_priv->frame_ofs[i] = ~0; /* All ones indicates placeholder */
1208 /* Allocate the DMA buffer freelist */
1209 if ((ret = mach64_init_freelist(dev))) {
1210 DRM_ERROR("Freelist allocation failed\n");
1211 mach64_do_cleanup_dma(dev);
1218 /*******************************************************************/
1219 /** MMIO Pseudo-DMA (intended primarily for debugging, not performance)
1222 int mach64_do_dispatch_pseudo_dma(drm_mach64_private_t *dev_priv)
1224 drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
1225 volatile u32 *ring_read;
1226 struct list_head *ptr;
1227 drm_mach64_freelist_t *entry;
1228 struct drm_buf *buf = NULL;
1230 u32 used, reg, target;
1231 int fifo, count, found, ret, no_idle_wait;
1233 fifo = count = reg = no_idle_wait = 0;
1234 target = MACH64_BM_ADDR;
1236 if ((ret = mach64_do_wait_for_idle(dev_priv)) < 0) {
1237 DRM_INFO("idle failed before pseudo-dma dispatch, resetting engine\n");
1238 mach64_dump_engine_info(dev_priv);
1239 mach64_do_engine_reset(dev_priv);
1243 ring_read = (u32 *) ring->start;
1245 while (ring->tail != ring->head) {
1246 u32 buf_addr, new_target, offset;
1247 u32 bytes, remaining, head, eol;
1252 le32_to_cpu(ring_read[head++]) - MACH64_APERTURE_OFFSET;
1253 buf_addr = le32_to_cpu(ring_read[head++]);
1254 eol = le32_to_cpu(ring_read[head]) & MACH64_DMA_EOL;
1255 bytes = le32_to_cpu(ring_read[head++])
1256 & ~(MACH64_DMA_HOLD_OFFSET | MACH64_DMA_EOL);
1258 head &= ring->tail_mask;
1260 /* can't wait for idle between a blit setup descriptor
1261 * and a HOSTDATA descriptor or the engine will lock
1263 if (new_target == MACH64_BM_HOSTDATA
1264 && target == MACH64_BM_ADDR)
1267 target = new_target;
1271 list_for_each(ptr, &dev_priv->pending) {
1272 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1274 offset = buf_addr - GETBUFADDR(buf);
1275 if (offset < MACH64_BUFFER_SIZE) {
1281 if (!found || buf == NULL) {
1283 ("Couldn't find pending buffer: head: %u tail: %u buf_addr: 0x%08x %s\n",
1284 head, ring->tail, buf_addr, (eol ? "eol" : ""));
1285 mach64_dump_ring_info(dev_priv);
1286 mach64_do_engine_reset(dev_priv);
1290 /* Hand feed the buffer to the card via MMIO, waiting for the fifo
1293 DRM_DEBUG("target: (0x%08x) %s\n", target,
1295 MACH64_BM_HOSTDATA ? "BM_HOSTDATA" : "BM_ADDR"));
1296 DRM_DEBUG("offset: %u bytes: %u used: %u\n", offset, bytes,
1299 remaining = (buf->used - offset) >> 2; /* dwords remaining in buffer */
1300 used = bytes >> 2; /* dwords in buffer for this descriptor */
1301 buf_ptr = (u32 *) ((char *)GETBUFPTR(buf) + offset);
1306 if (target == MACH64_BM_HOSTDATA) {
1307 reg = DMAREG(MACH64_HOST_DATA0);
1309 (remaining > 16) ? 16 : remaining;
1312 reg = le32_to_cpu(*buf_ptr++);
1314 count = (reg >> 16) + 1;
1318 reg = MMSELECT(reg);
1320 while (count && used) {
1324 mach64_do_wait_for_fifo
1325 (dev_priv, 16)) < 0) {
1331 mach64_do_wait_for_idle
1339 MACH64_WRITE(reg, le32_to_cpu(*buf_ptr++));
1348 ring->head_addr = ring->start_addr + (ring->head * sizeof(u32));
1349 ring->space += (4 * sizeof(u32));
1352 if ((ret = mach64_do_wait_for_idle(dev_priv)) < 0) {
1355 MACH64_WRITE(MACH64_BM_GUI_TABLE_CMD,
1356 ring->head_addr | MACH64_CIRCULAR_BUF_SIZE_16KB);
1358 DRM_DEBUG("completed\n");
1365 /*******************************************************************/
1366 /** \name DMA cleanup */
1369 int mach64_do_cleanup_dma(struct drm_device * dev)
1373 /* Make sure interrupts are disabled here because the uninstall ioctl
1374 * may not have been called from userspace and after dev_private
1375 * is freed, it's too late.
1378 drm_irq_uninstall(dev);
1380 if (dev->dev_private) {
1381 drm_mach64_private_t *dev_priv = dev->dev_private;
1383 if (!dev_priv->is_pci) {
1384 if (dev_priv->ring_map)
1385 drm_core_ioremapfree(dev_priv->ring_map, dev);
1387 if (dev->agp_buffer_map) {
1388 drm_core_ioremapfree(dev->agp_buffer_map, dev);
1389 dev->agp_buffer_map = NULL;
1393 mach64_destroy_freelist(dev);
1395 drm_free(dev_priv, DRM_MEM_DRIVER);
1396 dev->dev_private = NULL;
1405 /*******************************************************************/
1406 /** \name IOCTL handlers */
1409 int mach64_dma_init(struct drm_device *dev, void *data,
1410 struct drm_file *file_priv)
1412 drm_mach64_init_t *init = data;
1416 LOCK_TEST_WITH_RETURN(dev, file_priv);
1418 switch (init->func) {
1419 case DRM_MACH64_INIT_DMA:
1420 return mach64_do_dma_init(dev, init);
1421 case DRM_MACH64_CLEANUP_DMA:
1422 return mach64_do_cleanup_dma(dev);
1428 int mach64_dma_idle(struct drm_device *dev, void *data,
1429 struct drm_file *file_priv)
1431 drm_mach64_private_t *dev_priv = dev->dev_private;
1435 LOCK_TEST_WITH_RETURN(dev, file_priv);
1437 return mach64_do_dma_idle(dev_priv);
1440 int mach64_dma_flush(struct drm_device *dev, void *data,
1441 struct drm_file *file_priv)
1443 drm_mach64_private_t *dev_priv = dev->dev_private;
1447 LOCK_TEST_WITH_RETURN(dev, file_priv);
1449 return mach64_do_dma_flush(dev_priv);
1452 int mach64_engine_reset(struct drm_device *dev, void *data,
1453 struct drm_file *file_priv)
1455 drm_mach64_private_t *dev_priv = dev->dev_private;
1459 LOCK_TEST_WITH_RETURN(dev, file_priv);
1461 return mach64_do_engine_reset(dev_priv);
1467 /*******************************************************************/
1468 /** \name Freelist management */
1471 int mach64_init_freelist(struct drm_device * dev)
1473 struct drm_device_dma *dma = dev->dma;
1474 drm_mach64_private_t *dev_priv = dev->dev_private;
1475 drm_mach64_freelist_t *entry;
1476 struct list_head *ptr;
1479 DRM_DEBUG("adding %d buffers to freelist\n", dma->buf_count);
1481 for (i = 0; i < dma->buf_count; i++) {
1483 (drm_mach64_freelist_t *)
1484 drm_alloc(sizeof(drm_mach64_freelist_t),
1485 DRM_MEM_BUFLISTS)) == NULL)
1487 memset(entry, 0, sizeof(drm_mach64_freelist_t));
1488 entry->buf = dma->buflist[i];
1490 list_add_tail(ptr, &dev_priv->free_list);
1496 void mach64_destroy_freelist(struct drm_device * dev)
1498 drm_mach64_private_t *dev_priv = dev->dev_private;
1499 drm_mach64_freelist_t *entry;
1500 struct list_head *ptr;
1501 struct list_head *tmp;
1505 list_for_each_safe(ptr, tmp, &dev_priv->pending) {
1507 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1508 drm_free(entry, DRM_MEM_BUFLISTS);
1510 list_for_each_safe(ptr, tmp, &dev_priv->placeholders) {
1512 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1513 drm_free(entry, DRM_MEM_BUFLISTS);
1516 list_for_each_safe(ptr, tmp, &dev_priv->free_list) {
1518 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1519 drm_free(entry, DRM_MEM_BUFLISTS);
1523 /* IMPORTANT: This function should only be called when the engine is idle or locked up,
1524 * as it assumes all buffers in the pending list have been completed by the hardware.
1526 int mach64_do_release_used_buffers(drm_mach64_private_t *dev_priv)
1528 struct list_head *ptr;
1529 struct list_head *tmp;
1530 drm_mach64_freelist_t *entry;
1533 if (list_empty(&dev_priv->pending))
1536 /* Iterate the pending list and move all buffers into the freelist... */
1538 list_for_each_safe(ptr, tmp, &dev_priv->pending) {
1539 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1540 if (entry->discard) {
1541 entry->buf->pending = 0;
1543 list_add_tail(ptr, &dev_priv->free_list);
1548 DRM_DEBUG("released %d buffers from pending list\n", i);
1553 static int mach64_do_reclaim_completed(drm_mach64_private_t *dev_priv)
1555 drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
1556 struct list_head *ptr;
1557 struct list_head *tmp;
1558 drm_mach64_freelist_t *entry;
1559 u32 head, tail, ofs;
1561 mach64_ring_tick(dev_priv, ring);
1566 #if MACH64_EXTRA_CHECKING
1567 if (MACH64_READ(MACH64_GUI_STAT) & MACH64_GUI_ACTIVE) {
1568 DRM_ERROR("Empty ring with non-idle engine!\n");
1569 mach64_dump_ring_info(dev_priv);
1573 /* last pass is complete, so release everything */
1574 mach64_do_release_used_buffers(dev_priv);
1575 DRM_DEBUG("idle engine, freed all buffers.\n");
1576 if (list_empty(&dev_priv->free_list)) {
1577 DRM_ERROR("Freelist empty with idle engine\n");
1582 /* Look for a completed buffer and bail out of the loop
1583 * as soon as we find one -- don't waste time trying
1584 * to free extra bufs here, leave that to do_release_used_buffers
1586 list_for_each_safe(ptr, tmp, &dev_priv->pending) {
1587 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1588 ofs = entry->ring_ofs;
1589 if (entry->discard &&
1590 ((head < tail && (ofs < head || ofs >= tail)) ||
1591 (head > tail && (ofs < head && ofs >= tail)))) {
1592 #if MACH64_EXTRA_CHECKING
1595 for (i = head; i != tail; i = (i + 4) & ring->tail_mask)
1597 u32 o1 = le32_to_cpu(((u32 *) ring->
1599 u32 o2 = GETBUFADDR(entry->buf);
1603 ("Attempting to free used buffer: "
1604 "i=%d buf=0x%08x\n",
1606 mach64_dump_ring_info(dev_priv);
1611 /* found a processed buffer */
1612 entry->buf->pending = 0;
1614 list_add_tail(ptr, &dev_priv->free_list);
1616 ("freed processed buffer (head=%d tail=%d "
1617 "buf ring ofs=%d).\n",
1626 struct drm_buf *mach64_freelist_get(drm_mach64_private_t *dev_priv)
1628 drm_mach64_descriptor_ring_t *ring = &dev_priv->ring;
1629 drm_mach64_freelist_t *entry;
1630 struct list_head *ptr;
1633 if (list_empty(&dev_priv->free_list)) {
1634 if (list_empty(&dev_priv->pending)) {
1636 ("Couldn't get buffer - pending and free lists empty\n");
1638 list_for_each(ptr, &dev_priv->placeholders) {
1641 DRM_INFO("Placeholders: %d\n", t);
1645 for (t = 0; t < dev_priv->usec_timeout; t++) {
1648 ret = mach64_do_reclaim_completed(dev_priv);
1650 goto _freelist_entry_found;
1656 mach64_dump_ring_info(dev_priv);
1658 ("timeout waiting for buffers: ring head_addr: 0x%08x head: %d tail: %d\n",
1659 ring->head_addr, ring->head, ring->tail);
1663 _freelist_entry_found:
1664 ptr = dev_priv->free_list.next;
1666 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1667 entry->buf->used = 0;
1668 list_add_tail(ptr, &dev_priv->placeholders);
1672 int mach64_freelist_put(drm_mach64_private_t *dev_priv, struct drm_buf *copy_buf)
1674 struct list_head *ptr;
1675 drm_mach64_freelist_t *entry;
1677 #if MACH64_EXTRA_CHECKING
1678 list_for_each(ptr, &dev_priv->pending) {
1679 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1680 if (copy_buf == entry->buf) {
1681 DRM_ERROR("Trying to release a pending buf\n");
1686 ptr = dev_priv->placeholders.next;
1687 entry = list_entry(ptr, drm_mach64_freelist_t, list);
1688 copy_buf->pending = 0;
1690 entry->buf = copy_buf;
1693 list_add_tail(ptr, &dev_priv->free_list);
1701 /*******************************************************************/
1702 /** \name DMA buffer request and submission IOCTL handler */
1705 static int mach64_dma_get_buffers(struct drm_device *dev,
1706 struct drm_file *file_priv,
1710 struct drm_buf *buf;
1711 drm_mach64_private_t *dev_priv = dev->dev_private;
1713 for (i = d->granted_count; i < d->request_count; i++) {
1714 buf = mach64_freelist_get(dev_priv);
1715 #if MACH64_EXTRA_CHECKING
1723 buf->file_priv = file_priv;
1725 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
1728 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
1729 sizeof(buf->total)))
1737 int mach64_dma_buffers(struct drm_device *dev, void *data,
1738 struct drm_file *file_priv)
1740 struct drm_device_dma *dma = dev->dma;
1741 struct drm_dma *d = data;
1744 LOCK_TEST_WITH_RETURN(dev, file_priv);
1746 /* Please don't send us buffers.
1748 if (d->send_count != 0) {
1749 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
1750 DRM_CURRENTPID, d->send_count);
1754 /* We'll send you buffers.
1756 if (d->request_count < 0 || d->request_count > dma->buf_count) {
1757 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
1758 DRM_CURRENTPID, d->request_count, dma->buf_count);
1762 d->granted_count = 0;
1764 if (d->request_count) {
1765 ret = mach64_dma_get_buffers(dev, file_priv, d);
1771 void mach64_driver_lastclose(struct drm_device * dev)
1773 mach64_do_cleanup_dma(dev);