sys/dev/drm/r128/r128_cce.c

   1 /* r128_cce.c -- ATI Rage 128 driver -*- linux-c -*-
   2  * Created: Wed Apr  5 19:24:19 2000 by kevin@precisioninsight.com
   3  *
   4  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   5  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
   6  * All Rights Reserved.
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  23  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  24  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  *
  27  * Authors:
  28  *    Gareth Hughes <gareth@valinux.com>
  29  *
  30  * $FreeBSD: src/sys/dev/drm/r128_cce.c,v 1.6.2.1 2003/04/26 07:05:29 anholt Exp $
  31  */
  32
  33 #include "dev/drm/r128.h"
  34 #include "dev/drm/drmP.h"
  35 #include "dev/drm/drm.h"
  36 #include "dev/drm/r128_drm.h"
  37 #include "dev/drm/r128_drv.h"
  38
  39 #define R128_FIFO_DEBUG         0
  40
  41 /* CCE microcode (from ATI) */
  42 static u32 r128_cce_microcode[] = {
  43         0, 276838400, 0, 268449792, 2, 142, 2, 145, 0, 1076765731, 0,
  44         1617039951, 0, 774592877, 0, 1987540286, 0, 2307490946U, 0,
  45         599558925, 0, 589505315, 0, 596487092, 0, 589505315, 1,
  46         11544576, 1, 206848, 1, 311296, 1, 198656, 2, 912273422, 11,
  47         262144, 0, 0, 1, 33559837, 1, 7438, 1, 14809, 1, 6615, 12, 28,
  48         1, 6614, 12, 28, 2, 23, 11, 18874368, 0, 16790922, 1, 409600, 9,
  49         30, 1, 147854772, 16, 420483072, 3, 8192, 0, 10240, 1, 198656,
  50         1, 15630, 1, 51200, 10, 34858, 9, 42, 1, 33559823, 2, 10276, 1,
  51         15717, 1, 15718, 2, 43, 1, 15936948, 1, 570480831, 1, 14715071,
  52         12, 322123831, 1, 33953125, 12, 55, 1, 33559908, 1, 15718, 2,
  53         46, 4, 2099258, 1, 526336, 1, 442623, 4, 4194365, 1, 509952, 1,
  54         459007, 3, 0, 12, 92, 2, 46, 12, 176, 1, 15734, 1, 206848, 1,
  55         18432, 1, 133120, 1, 100670734, 1, 149504, 1, 165888, 1,
  56         15975928, 1, 1048576, 6, 3145806, 1, 15715, 16, 2150645232U, 2,
  57         268449859, 2, 10307, 12, 176, 1, 15734, 1, 15735, 1, 15630, 1,
  58         15631, 1, 5253120, 6, 3145810, 16, 2150645232U, 1, 15864, 2, 82,
  59         1, 343310, 1, 1064207, 2, 3145813, 1, 15728, 1, 7817, 1, 15729,
  60         3, 15730, 12, 92, 2, 98, 1, 16168, 1, 16167, 1, 16002, 1, 16008,
  61         1, 15974, 1, 15975, 1, 15990, 1, 15976, 1, 15977, 1, 15980, 0,
  62         15981, 1, 10240, 1, 5253120, 1, 15720, 1, 198656, 6, 110, 1,
  63         180224, 1, 103824738, 2, 112, 2, 3145839, 0, 536885440, 1,
  64         114880, 14, 125, 12, 206975, 1, 33559995, 12, 198784, 0,
  65         33570236, 1, 15803, 0, 15804, 3, 294912, 1, 294912, 3, 442370,
  66         1, 11544576, 0, 811612160, 1, 12593152, 1, 11536384, 1,
  67         14024704, 7, 310382726, 0, 10240, 1, 14796, 1, 14797, 1, 14793,
  68         1, 14794, 0, 14795, 1, 268679168, 1, 9437184, 1, 268449792, 1,
  69         198656, 1, 9452827, 1, 1075854602, 1, 1075854603, 1, 557056, 1,
  70         114880, 14, 159, 12, 198784, 1, 1109409213, 12, 198783, 1,
  71         1107312059, 12, 198784, 1, 1109409212, 2, 162, 1, 1075854781, 1,
  72         1073757627, 1, 1075854780, 1, 540672, 1, 10485760, 6, 3145894,
  73         16, 274741248, 9, 168, 3, 4194304, 3, 4209949, 0, 0, 0, 256, 14,
  74         174, 1, 114857, 1, 33560007, 12, 176, 0, 10240, 1, 114858, 1,
  75         33560018, 1, 114857, 3, 33560007, 1, 16008, 1, 114874, 1,
  76         33560360, 1, 114875, 1, 33560154, 0, 15963, 0, 256, 0, 4096, 1,
  77         409611, 9, 188, 0, 10240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  78         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  79         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  80         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  81         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  82         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  83         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
  84 };
  85
  86 int r128_do_wait_for_idle( drm_r128_private_t *dev_priv );
  87
  88 int R128_READ_PLL(drm_device_t *dev, int addr)
  89 {
  90         drm_r128_private_t *dev_priv = dev->dev_private;
  91
  92         R128_WRITE8(R128_CLOCK_CNTL_INDEX, addr & 0x1f);
  93         return R128_READ(R128_CLOCK_CNTL_DATA);
  94 }
  95
  96 #if R128_FIFO_DEBUG
  97 static void r128_status( drm_r128_private_t *dev_priv )
  98 {
  99         printk( "GUI_STAT           = 0x%08x\n",
 100                 (unsigned int)R128_READ( R128_GUI_STAT ) );
 101         printk( "PM4_STAT           = 0x%08x\n",
 102                 (unsigned int)R128_READ( R128_PM4_STAT ) );
 103         printk( "PM4_BUFFER_DL_WPTR = 0x%08x\n",
 104                 (unsigned int)R128_READ( R128_PM4_BUFFER_DL_WPTR ) );
 105         printk( "PM4_BUFFER_DL_RPTR = 0x%08x\n",
 106                 (unsigned int)R128_READ( R128_PM4_BUFFER_DL_RPTR ) );
 107         printk( "PM4_MICRO_CNTL     = 0x%08x\n",
 108                 (unsigned int)R128_READ( R128_PM4_MICRO_CNTL ) );
 109         printk( "PM4_BUFFER_CNTL    = 0x%08x\n",
 110                 (unsigned int)R128_READ( R128_PM4_BUFFER_CNTL ) );
 111 }
 112 #endif
 113
 114
 115 /* ================================================================
 116  * Engine, FIFO control
 117  */
 118
 119 static int r128_do_pixcache_flush( drm_r128_private_t *dev_priv )
 120 {
 121         u32 tmp;
 122         int i;
 123
 124         tmp = R128_READ( R128_PC_NGUI_CTLSTAT ) | R128_PC_FLUSH_ALL;
 125         R128_WRITE( R128_PC_NGUI_CTLSTAT, tmp );
 126
 127         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 128                 if ( !(R128_READ( R128_PC_NGUI_CTLSTAT ) & R128_PC_BUSY) ) {
 129                         return 0;
 130                 }
 131                 DRM_UDELAY( 1 );
 132         }
 133
 134 #if R128_FIFO_DEBUG
 135         DRM_ERROR( "failed!\n" );
 136 #endif
 137         return DRM_ERR(EBUSY);
 138 }
 139
 140 static int r128_do_wait_for_fifo( drm_r128_private_t *dev_priv, int entries )
 141 {
 142         int i;
 143
 144         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 145                 int slots = R128_READ( R128_GUI_STAT ) & R128_GUI_FIFOCNT_MASK;
 146                 if ( slots >= entries ) return 0;
 147                 DRM_UDELAY( 1 );
 148         }
 149
 150 #if R128_FIFO_DEBUG
 151         DRM_ERROR( "failed!\n" );
 152 #endif
 153         return DRM_ERR(EBUSY);
 154 }
 155
 156 int r128_do_wait_for_idle( drm_r128_private_t *dev_priv )
 157 {
 158         int i, ret;
 159
 160         ret = r128_do_wait_for_fifo( dev_priv, 64 );
 161         if ( ret ) return ret;
 162
 163         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 164                 if ( !(R128_READ( R128_GUI_STAT ) & R128_GUI_ACTIVE) ) {
 165                         r128_do_pixcache_flush( dev_priv );
 166                         return 0;
 167                 }
 168                 DRM_UDELAY( 1 );
 169         }
 170
 171 #if R128_FIFO_DEBUG
 172         DRM_ERROR( "failed!\n" );
 173 #endif
 174         return DRM_ERR(EBUSY);
 175 }
 176
 177
 178 /* ================================================================
 179  * CCE control, initialization
 180  */
 181
 182 /* Load the microcode for the CCE */
 183 static void r128_cce_load_microcode( drm_r128_private_t *dev_priv )
 184 {
 185         int i;
 186
 187         DRM_DEBUG( "\n" );
 188
 189         r128_do_wait_for_idle( dev_priv );
 190
 191         R128_WRITE( R128_PM4_MICROCODE_ADDR, 0 );
 192         for ( i = 0 ; i < 256 ; i++ ) {
 193                 R128_WRITE( R128_PM4_MICROCODE_DATAH,
 194                             r128_cce_microcode[i * 2] );
 195                 R128_WRITE( R128_PM4_MICROCODE_DATAL,
 196                             r128_cce_microcode[i * 2 + 1] );
 197         }
 198 }
 199
 200 /* Flush any pending commands to the CCE.  This should only be used just
 201  * prior to a wait for idle, as it informs the engine that the command
 202  * stream is ending.
 203  */
 204 static void r128_do_cce_flush( drm_r128_private_t *dev_priv )
 205 {
 206         u32 tmp;
 207
 208         tmp = R128_READ( R128_PM4_BUFFER_DL_WPTR ) | R128_PM4_BUFFER_DL_DONE;
 209         R128_WRITE( R128_PM4_BUFFER_DL_WPTR, tmp );
 210 }
 211
 212 /* Wait for the CCE to go idle.
 213  */
 214 int r128_do_cce_idle( drm_r128_private_t *dev_priv )
 215 {
 216         int i;
 217
 218         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 219                 if ( GET_RING_HEAD( &dev_priv->ring ) == dev_priv->ring.tail ) {
 220                         int pm4stat = R128_READ( R128_PM4_STAT );
 221                         if ( ( (pm4stat & R128_PM4_FIFOCNT_MASK) >=
 222                                dev_priv->cce_fifo_size ) &&
 223                              !(pm4stat & (R128_PM4_BUSY |
 224                                           R128_PM4_GUI_ACTIVE)) ) {
 225                                 return r128_do_pixcache_flush( dev_priv );
 226                         }
 227                 }
 228                 DRM_UDELAY( 1 );
 229         }
 230
 231 #if R128_FIFO_DEBUG
 232         DRM_ERROR( "failed!\n" );
 233         r128_status( dev_priv );
 234 #endif
 235         return DRM_ERR(EBUSY);
 236 }
 237
 238 /* Start the Concurrent Command Engine.
 239  */
 240 static void r128_do_cce_start( drm_r128_private_t *dev_priv )
 241 {
 242         r128_do_wait_for_idle( dev_priv );
 243
 244         R128_WRITE( R128_PM4_BUFFER_CNTL,
 245                     dev_priv->cce_mode | dev_priv->ring.size_l2qw );
 246         R128_READ( R128_PM4_BUFFER_ADDR ); /* as per the sample code */
 247         R128_WRITE( R128_PM4_MICRO_CNTL, R128_PM4_MICRO_FREERUN );
 248
 249         dev_priv->cce_running = 1;
 250 }
 251
 252 /* Reset the Concurrent Command Engine.  This will not flush any pending
 253  * commands, so you must wait for the CCE command stream to complete
 254  * before calling this routine.
 255  */
 256 static void r128_do_cce_reset( drm_r128_private_t *dev_priv )
 257 {
 258         R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
 259         R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
 260         SET_RING_HEAD( &dev_priv->ring, 0 );
 261         dev_priv->ring.tail = 0;
 262 }
 263
 264 /* Stop the Concurrent Command Engine.  This will not flush any pending
 265  * commands, so you must flush the command stream and wait for the CCE
 266  * to go idle before calling this routine.
 267  */
 268 static void r128_do_cce_stop( drm_r128_private_t *dev_priv )
 269 {
 270         R128_WRITE( R128_PM4_MICRO_CNTL, 0 );
 271         R128_WRITE( R128_PM4_BUFFER_CNTL, R128_PM4_NONPM4 );
 272
 273         dev_priv->cce_running = 0;
 274 }
 275
 276 /* Reset the engine.  This will stop the CCE if it is running.
 277  */
 278 static int r128_do_engine_reset( drm_device_t *dev )
 279 {
 280         drm_r128_private_t *dev_priv = dev->dev_private;
 281         u32 clock_cntl_index, mclk_cntl, gen_reset_cntl;
 282
 283         r128_do_pixcache_flush( dev_priv );
 284
 285         clock_cntl_index = R128_READ( R128_CLOCK_CNTL_INDEX );
 286         mclk_cntl = R128_READ_PLL( dev, R128_MCLK_CNTL );
 287
 288         R128_WRITE_PLL( R128_MCLK_CNTL,
 289                         mclk_cntl | R128_FORCE_GCP | R128_FORCE_PIPE3D_CP );
 290
 291         gen_reset_cntl = R128_READ( R128_GEN_RESET_CNTL );
 292
 293         /* Taken from the sample code - do not change */
 294         R128_WRITE( R128_GEN_RESET_CNTL,
 295                     gen_reset_cntl | R128_SOFT_RESET_GUI );
 296         R128_READ( R128_GEN_RESET_CNTL );
 297         R128_WRITE( R128_GEN_RESET_CNTL,
 298                     gen_reset_cntl & ~R128_SOFT_RESET_GUI );
 299         R128_READ( R128_GEN_RESET_CNTL );
 300
 301         R128_WRITE_PLL( R128_MCLK_CNTL, mclk_cntl );
 302         R128_WRITE( R128_CLOCK_CNTL_INDEX, clock_cntl_index );
 303         R128_WRITE( R128_GEN_RESET_CNTL, gen_reset_cntl );
 304
 305         /* Reset the CCE ring */
 306         r128_do_cce_reset( dev_priv );
 307
 308         /* The CCE is no longer running after an engine reset */
 309         dev_priv->cce_running = 0;
 310
 311         /* Reset any pending vertex, indirect buffers */
 312         r128_freelist_reset( dev );
 313
 314         return 0;
 315 }
 316
 317 static void r128_cce_init_ring_buffer( drm_device_t *dev,
 318                                        drm_r128_private_t *dev_priv )
 319 {
 320         u32 ring_start;
 321         u32 tmp;
 322
 323         DRM_DEBUG( "\n" );
 324
 325         /* The manual (p. 2) says this address is in "VM space".  This
 326          * means it's an offset from the start of AGP space.
 327          */
 328 #if __REALLY_HAVE_AGP
 329         if ( !dev_priv->is_pci )
 330                 ring_start = dev_priv->cce_ring->offset - dev->agp->base;
 331         else
 332 #endif
 333                 ring_start = dev_priv->cce_ring->offset - dev->sg->handle;
 334
 335         R128_WRITE( R128_PM4_BUFFER_OFFSET, ring_start | R128_AGP_OFFSET );
 336
 337         R128_WRITE( R128_PM4_BUFFER_DL_WPTR, 0 );
 338         R128_WRITE( R128_PM4_BUFFER_DL_RPTR, 0 );
 339
 340         /* DL_RPTR_ADDR is a physical address in AGP space. */
 341         SET_RING_HEAD( &dev_priv->ring, 0 );
 342
 343         if ( !dev_priv->is_pci ) {
 344                 R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
 345                             dev_priv->ring_rptr->offset );
 346         } else {
 347                 drm_sg_mem_t *entry = dev->sg;
 348                 unsigned long tmp_ofs, page_ofs;
 349
 350                 tmp_ofs = dev_priv->ring_rptr->offset - dev->sg->handle;
 351                 page_ofs = tmp_ofs >> PAGE_SHIFT;
 352
 353                 R128_WRITE( R128_PM4_BUFFER_DL_RPTR_ADDR,
 354                             entry->busaddr[page_ofs]);
 355                 DRM_DEBUG( "ring rptr: offset=0x%08lx handle=0x%08lx\n",
 356                            entry->busaddr[page_ofs],
 357                            entry->handle + tmp_ofs );
 358         }
 359
 360         /* Set watermark control */
 361         R128_WRITE( R128_PM4_BUFFER_WM_CNTL,
 362                     ((R128_WATERMARK_L/4) << R128_WMA_SHIFT)
 363                     | ((R128_WATERMARK_M/4) << R128_WMB_SHIFT)
 364                     | ((R128_WATERMARK_N/4) << R128_WMC_SHIFT)
 365                     | ((R128_WATERMARK_K/64) << R128_WB_WM_SHIFT) );
 366
 367         /* Force read.  Why?  Because it's in the examples... */
 368         R128_READ( R128_PM4_BUFFER_ADDR );
 369
 370         /* Turn on bus mastering */
 371         tmp = R128_READ( R128_BUS_CNTL ) & ~R128_BUS_MASTER_DIS;
 372         R128_WRITE( R128_BUS_CNTL, tmp );
 373 }
 374
 375 static int r128_do_init_cce( drm_device_t *dev, drm_r128_init_t *init )
 376 {
 377         drm_r128_private_t *dev_priv;
 378
 379         DRM_DEBUG( "\n" );
 380
 381         dev_priv = DRM(alloc)( sizeof(drm_r128_private_t), DRM_MEM_DRIVER );
 382         if ( dev_priv == NULL )
 383                 return DRM_ERR(ENOMEM);
 384
 385         memset( dev_priv, 0, sizeof(drm_r128_private_t) );
 386
 387         dev_priv->is_pci = init->is_pci;
 388
 389         if ( dev_priv->is_pci && !dev->sg ) {
 390                 DRM_ERROR( "PCI GART memory not allocated!\n" );
 391                 dev->dev_private = (void *)dev_priv;
 392                 r128_do_cleanup_cce( dev );
 393                 return DRM_ERR(EINVAL);
 394         }
 395
 396         dev_priv->usec_timeout = init->usec_timeout;
 397         if ( dev_priv->usec_timeout < 1 ||
 398              dev_priv->usec_timeout > R128_MAX_USEC_TIMEOUT ) {
 399                 DRM_DEBUG( "TIMEOUT problem!\n" );
 400                 dev->dev_private = (void *)dev_priv;
 401                 r128_do_cleanup_cce( dev );
 402                 return DRM_ERR(EINVAL);
 403         }
 404
 405         dev_priv->cce_mode = init->cce_mode;
 406
 407         /* GH: Simple idle check.
 408          */
 409         atomic_set( &dev_priv->idle_count, 0 );
 410
 411         /* We don't support anything other than bus-mastering ring mode,
 412          * but the ring can be in either AGP or PCI space for the ring
 413          * read pointer.
 414          */
 415         if ( ( init->cce_mode != R128_PM4_192BM ) &&
 416              ( init->cce_mode != R128_PM4_128BM_64INDBM ) &&
 417              ( init->cce_mode != R128_PM4_64BM_128INDBM ) &&
 418              ( init->cce_mode != R128_PM4_64BM_64VCBM_64INDBM ) ) {
 419                 DRM_DEBUG( "Bad cce_mode!\n" );
 420                 dev->dev_private = (void *)dev_priv;
 421                 r128_do_cleanup_cce( dev );
 422                 return DRM_ERR(EINVAL);
 423         }
 424
 425         switch ( init->cce_mode ) {
 426         case R128_PM4_NONPM4:
 427                 dev_priv->cce_fifo_size = 0;
 428                 break;
 429         case R128_PM4_192PIO:
 430         case R128_PM4_192BM:
 431                 dev_priv->cce_fifo_size = 192;
 432                 break;
 433         case R128_PM4_128PIO_64INDBM:
 434         case R128_PM4_128BM_64INDBM:
 435                 dev_priv->cce_fifo_size = 128;
 436                 break;
 437         case R128_PM4_64PIO_128INDBM:
 438         case R128_PM4_64BM_128INDBM:
 439         case R128_PM4_64PIO_64VCBM_64INDBM:
 440         case R128_PM4_64BM_64VCBM_64INDBM:
 441         case R128_PM4_64PIO_64VCPIO_64INDPIO:
 442                 dev_priv->cce_fifo_size = 64;
 443                 break;
 444         }
 445
 446         switch ( init->fb_bpp ) {
 447         case 16:
 448                 dev_priv->color_fmt = R128_DATATYPE_RGB565;
 449                 break;
 450         case 32:
 451         default:
 452                 dev_priv->color_fmt = R128_DATATYPE_ARGB8888;
 453                 break;
 454         }
 455         dev_priv->front_offset  = init->front_offset;
 456         dev_priv->front_pitch   = init->front_pitch;
 457         dev_priv->back_offset   = init->back_offset;
 458         dev_priv->back_pitch    = init->back_pitch;
 459
 460         switch ( init->depth_bpp ) {
 461         case 16:
 462                 dev_priv->depth_fmt = R128_DATATYPE_RGB565;
 463                 break;
 464         case 24:
 465         case 32:
 466         default:
 467                 dev_priv->depth_fmt = R128_DATATYPE_ARGB8888;
 468                 break;
 469         }
 470         dev_priv->depth_offset  = init->depth_offset;
 471         dev_priv->depth_pitch   = init->depth_pitch;
 472         dev_priv->span_offset   = init->span_offset;
 473
 474         dev_priv->front_pitch_offset_c = (((dev_priv->front_pitch/8) << 21) |
 475                                           (dev_priv->front_offset >> 5));
 476         dev_priv->back_pitch_offset_c = (((dev_priv->back_pitch/8) << 21) |
 477                                          (dev_priv->back_offset >> 5));
 478         dev_priv->depth_pitch_offset_c = (((dev_priv->depth_pitch/8) << 21) |
 479                                           (dev_priv->depth_offset >> 5) |
 480                                           R128_DST_TILE);
 481         dev_priv->span_pitch_offset_c = (((dev_priv->depth_pitch/8) << 21) |
 482                                          (dev_priv->span_offset >> 5));
 483
 484         DRM_GETSAREA();
 485
 486         if(!dev_priv->sarea) {
 487                 DRM_ERROR("could not find sarea!\n");
 488                 dev->dev_private = (void *)dev_priv;
 489                 r128_do_cleanup_cce( dev );
 490                 return DRM_ERR(EINVAL);
 491         }
 492
 493         DRM_FIND_MAP( dev_priv->fb, init->fb_offset );
 494         if(!dev_priv->fb) {
 495                 DRM_ERROR("could not find framebuffer!\n");
 496                 dev->dev_private = (void *)dev_priv;
 497                 r128_do_cleanup_cce( dev );
 498                 return DRM_ERR(EINVAL);
 499         }
 500         DRM_FIND_MAP( dev_priv->mmio, init->mmio_offset );
 501         if(!dev_priv->mmio) {
 502                 DRM_ERROR("could not find mmio region!\n");
 503                 dev->dev_private = (void *)dev_priv;
 504                 r128_do_cleanup_cce( dev );
 505                 return DRM_ERR(EINVAL);
 506         }
 507         DRM_FIND_MAP( dev_priv->cce_ring, init->ring_offset );
 508         if(!dev_priv->cce_ring) {
 509                 DRM_ERROR("could not find cce ring region!\n");
 510                 dev->dev_private = (void *)dev_priv;
 511                 r128_do_cleanup_cce( dev );
 512                 return DRM_ERR(EINVAL);
 513         }
 514         DRM_FIND_MAP( dev_priv->ring_rptr, init->ring_rptr_offset );
 515         if(!dev_priv->ring_rptr) {
 516                 DRM_ERROR("could not find ring read pointer!\n");
 517                 dev->dev_private = (void *)dev_priv;
 518                 r128_do_cleanup_cce( dev );
 519                 return DRM_ERR(EINVAL);
 520         }
 521         DRM_FIND_MAP( dev_priv->buffers, init->buffers_offset );
 522         if(!dev_priv->buffers) {
 523                 DRM_ERROR("could not find dma buffer region!\n");
 524                 dev->dev_private = (void *)dev_priv;
 525                 r128_do_cleanup_cce( dev );
 526                 return DRM_ERR(EINVAL);
 527         }
 528
 529         if ( !dev_priv->is_pci ) {
 530                 DRM_FIND_MAP( dev_priv->agp_textures,
 531                               init->agp_textures_offset );
 532                 if(!dev_priv->agp_textures) {
 533                         DRM_ERROR("could not find agp texture region!\n");
 534                         dev->dev_private = (void *)dev_priv;
 535                         r128_do_cleanup_cce( dev );
 536                         return DRM_ERR(EINVAL);
 537                 }
 538         }
 539
 540         dev_priv->sarea_priv =
 541                 (drm_r128_sarea_t *)((u8 *)dev_priv->sarea->handle +
 542                                      init->sarea_priv_offset);
 543
 544         if ( !dev_priv->is_pci ) {
 545                 DRM_IOREMAP( dev_priv->cce_ring );
 546                 DRM_IOREMAP( dev_priv->ring_rptr );
 547                 DRM_IOREMAP( dev_priv->buffers );
 548                 if(!dev_priv->cce_ring->handle ||
 549                    !dev_priv->ring_rptr->handle ||
 550                    !dev_priv->buffers->handle) {
 551                         DRM_ERROR("Could not ioremap agp regions!\n");
 552                         dev->dev_private = (void *)dev_priv;
 553                         r128_do_cleanup_cce( dev );
 554                         return DRM_ERR(ENOMEM);
 555                 }
 556         } else {
 557                 dev_priv->cce_ring->handle =
 558                         (void *)dev_priv->cce_ring->offset;
 559                 dev_priv->ring_rptr->handle =
 560                         (void *)dev_priv->ring_rptr->offset;
 561                 dev_priv->buffers->handle = (void *)dev_priv->buffers->offset;
 562         }
 563
 564 #if __REALLY_HAVE_AGP
 565         if ( !dev_priv->is_pci )
 566                 dev_priv->cce_buffers_offset = dev->agp->base;
 567         else
 568 #endif
 569                 dev_priv->cce_buffers_offset = dev->sg->handle;
 570
 571         dev_priv->ring.head = ((__volatile__ u32 *)
 572                                dev_priv->ring_rptr->handle);
 573
 574         dev_priv->ring.start = (u32 *)dev_priv->cce_ring->handle;
 575         dev_priv->ring.end = ((u32 *)dev_priv->cce_ring->handle
 576                               + init->ring_size / sizeof(u32));
 577         dev_priv->ring.size = init->ring_size;
 578         dev_priv->ring.size_l2qw = DRM(order)( init->ring_size / 8 );
 579
 580         dev_priv->ring.tail_mask =
 581                 (dev_priv->ring.size / sizeof(u32)) - 1;
 582
 583         dev_priv->ring.high_mark = 128;
 584         dev_priv->ring.ring_rptr = dev_priv->ring_rptr;
 585
 586         dev_priv->sarea_priv->last_frame = 0;
 587         R128_WRITE( R128_LAST_FRAME_REG, dev_priv->sarea_priv->last_frame );
 588
 589         dev_priv->sarea_priv->last_dispatch = 0;
 590         R128_WRITE( R128_LAST_DISPATCH_REG,
 591                     dev_priv->sarea_priv->last_dispatch );
 592
 593 #if __REALLY_HAVE_SG
 594         if ( dev_priv->is_pci ) {
 595                 if (!DRM(ati_pcigart_init)( dev, &dev_priv->phys_pci_gart,
 596                                             &dev_priv->bus_pci_gart) ) {
 597                         DRM_ERROR( "failed to init PCI GART!\n" );
 598                         dev->dev_private = (void *)dev_priv;
 599                         r128_do_cleanup_cce( dev );
 600                         return DRM_ERR(ENOMEM);
 601                 }
 602                 R128_WRITE( R128_PCI_GART_PAGE, dev_priv->bus_pci_gart );
 603         }
 604 #endif
 605
 606         r128_cce_init_ring_buffer( dev, dev_priv );
 607         r128_cce_load_microcode( dev_priv );
 608
 609         dev->dev_private = (void *)dev_priv;
 610
 611         r128_do_engine_reset( dev );
 612
 613         return 0;
 614 }
 615
 616 int r128_do_cleanup_cce( drm_device_t *dev )
 617 {
 618         if ( dev->dev_private ) {
 619                 drm_r128_private_t *dev_priv = dev->dev_private;
 620
 621 #if __REALLY_HAVE_SG
 622                 if ( !dev_priv->is_pci ) {
 623 #endif
 624                         if ( dev_priv->cce_ring != NULL )
 625                                 DRM_IOREMAPFREE( dev_priv->cce_ring );
 626                         if ( dev_priv->ring_rptr != NULL )
 627                                 DRM_IOREMAPFREE( dev_priv->ring_rptr );
 628                         if ( dev_priv->buffers != NULL )
 629                                 DRM_IOREMAPFREE( dev_priv->buffers );
 630 #if __REALLY_HAVE_SG
 631                 } else {
 632                         if (!DRM(ati_pcigart_cleanup)( dev,
 633                                                 dev_priv->phys_pci_gart,
 634                                                 dev_priv->bus_pci_gart ))
 635                                 DRM_ERROR( "failed to cleanup PCI GART!\n" );
 636                 }
 637 #endif
 638
 639                 DRM(free)( dev->dev_private, sizeof(drm_r128_private_t),
 640                            DRM_MEM_DRIVER );
 641                 dev->dev_private = NULL;
 642         }
 643
 644         return 0;
 645 }
 646
 647 int r128_cce_init( DRM_IOCTL_ARGS )
 648 {
 649         DRM_DEVICE;
 650         drm_r128_init_t init;
 651
 652         DRM_DEBUG( "\n" );
 653
 654         DRM_COPY_FROM_USER_IOCTL( init, (drm_r128_init_t *)data, sizeof(init) );
 655
 656         switch ( init.func ) {
 657         case R128_INIT_CCE:
 658                 return r128_do_init_cce( dev, &init );
 659         case R128_CLEANUP_CCE:
 660                 return r128_do_cleanup_cce( dev );
 661         }
 662
 663         return DRM_ERR(EINVAL);
 664 }
 665
 666 int r128_cce_start( DRM_IOCTL_ARGS )
 667 {
 668         DRM_DEVICE;
 669         drm_r128_private_t *dev_priv = dev->dev_private;
 670         DRM_DEBUG( "\n" );
 671
 672         LOCK_TEST_WITH_RETURN( dev, filp );
 673
 674         if ( dev_priv->cce_running || dev_priv->cce_mode == R128_PM4_NONPM4 ) {
 675                 DRM_DEBUG( "%s while CCE running\n", __FUNCTION__ );
 676                 return 0;
 677         }
 678
 679         r128_do_cce_start( dev_priv );
 680
 681         return 0;
 682 }
 683
 684 /* Stop the CCE.  The engine must have been idled before calling this
 685  * routine.
 686  */
 687 int r128_cce_stop( DRM_IOCTL_ARGS )
 688 {
 689         DRM_DEVICE;
 690         drm_r128_private_t *dev_priv = dev->dev_private;
 691         drm_r128_cce_stop_t stop;
 692         int ret;
 693         DRM_DEBUG( "\n" );
 694
 695         LOCK_TEST_WITH_RETURN( dev, filp );
 696
 697         DRM_COPY_FROM_USER_IOCTL(stop, (drm_r128_cce_stop_t *)data, sizeof(stop) );
 698
 699         /* Flush any pending CCE commands.  This ensures any outstanding
 700          * commands are exectuted by the engine before we turn it off.
 701          */
 702         if ( stop.flush ) {
 703                 r128_do_cce_flush( dev_priv );
 704         }
 705
 706         /* If we fail to make the engine go idle, we return an error
 707          * code so that the DRM ioctl wrapper can try again.
 708          */
 709         if ( stop.idle ) {
 710                 ret = r128_do_cce_idle( dev_priv );
 711                 if ( ret ) return ret;
 712         }
 713
 714         /* Finally, we can turn off the CCE.  If the engine isn't idle,
 715          * we will get some dropped triangles as they won't be fully
 716          * rendered before the CCE is shut down.
 717          */
 718         r128_do_cce_stop( dev_priv );
 719
 720         /* Reset the engine */
 721         r128_do_engine_reset( dev );
 722
 723         return 0;
 724 }
 725
 726 /* Just reset the CCE ring.  Called as part of an X Server engine reset.
 727  */
 728 int r128_cce_reset( DRM_IOCTL_ARGS )
 729 {
 730         DRM_DEVICE;
 731         drm_r128_private_t *dev_priv = dev->dev_private;
 732         DRM_DEBUG( "\n" );
 733
 734         LOCK_TEST_WITH_RETURN( dev, filp );
 735
 736         if ( !dev_priv ) {
 737                 DRM_DEBUG( "%s called before init done\n", __FUNCTION__ );
 738                 return DRM_ERR(EINVAL);
 739         }
 740
 741         r128_do_cce_reset( dev_priv );
 742
 743         /* The CCE is no longer running after an engine reset */
 744         dev_priv->cce_running = 0;
 745
 746         return 0;
 747 }
 748
 749 int r128_cce_idle( DRM_IOCTL_ARGS )
 750 {
 751         DRM_DEVICE;
 752         drm_r128_private_t *dev_priv = dev->dev_private;
 753         DRM_DEBUG( "\n" );
 754
 755         LOCK_TEST_WITH_RETURN( dev, filp );
 756
 757         if ( dev_priv->cce_running ) {
 758                 r128_do_cce_flush( dev_priv );
 759         }
 760
 761         return r128_do_cce_idle( dev_priv );
 762 }
 763
 764 int r128_engine_reset( DRM_IOCTL_ARGS )
 765 {
 766         DRM_DEVICE;
 767         DRM_DEBUG( "\n" );
 768
 769         LOCK_TEST_WITH_RETURN( dev, filp );
 770
 771         return r128_do_engine_reset( dev );
 772 }
 773
 774
 775 /* ================================================================
 776  * Fullscreen mode
 777  */
 778
 779 static int r128_do_init_pageflip( drm_device_t *dev )
 780 {
 781         drm_r128_private_t *dev_priv = dev->dev_private;
 782         DRM_DEBUG( "\n" );
 783
 784         dev_priv->crtc_offset =      R128_READ( R128_CRTC_OFFSET );
 785         dev_priv->crtc_offset_cntl = R128_READ( R128_CRTC_OFFSET_CNTL );
 786
 787         R128_WRITE( R128_CRTC_OFFSET, dev_priv->front_offset );
 788         R128_WRITE( R128_CRTC_OFFSET_CNTL,
 789                     dev_priv->crtc_offset_cntl | R128_CRTC_OFFSET_FLIP_CNTL );
 790
 791         dev_priv->page_flipping = 1;
 792         dev_priv->current_page = 0;
 793
 794         return 0;
 795 }
 796
 797 int r128_do_cleanup_pageflip( drm_device_t *dev )
 798 {
 799         drm_r128_private_t *dev_priv = dev->dev_private;
 800         DRM_DEBUG( "\n" );
 801
 802         R128_WRITE( R128_CRTC_OFFSET,      dev_priv->crtc_offset );
 803         R128_WRITE( R128_CRTC_OFFSET_CNTL, dev_priv->crtc_offset_cntl );
 804
 805         dev_priv->page_flipping = 0;
 806         dev_priv->current_page = 0;
 807
 808         return 0;
 809 }
 810
 811 int r128_fullscreen( DRM_IOCTL_ARGS )
 812 {
 813         DRM_DEVICE;
 814         drm_r128_fullscreen_t fs;
 815
 816         LOCK_TEST_WITH_RETURN( dev, filp );
 817
 818         DRM_COPY_FROM_USER_IOCTL( fs, (drm_r128_fullscreen_t *)data, sizeof(fs) );
 819
 820         switch ( fs.func ) {
 821         case R128_INIT_FULLSCREEN:
 822                 return r128_do_init_pageflip( dev );
 823         case R128_CLEANUP_FULLSCREEN:
 824                 return r128_do_cleanup_pageflip( dev );
 825         }
 826
 827         return DRM_ERR(EINVAL);
 828 }
 829
 830
 831 /* ================================================================
 832  * Freelist management
 833  */
 834 #define R128_BUFFER_USED        0xffffffff
 835 #define R128_BUFFER_FREE        0
 836
 837 #if 0
 838 static int r128_freelist_init( drm_device_t *dev )
 839 {
 840         drm_device_dma_t *dma = dev->dma;
 841         drm_r128_private_t *dev_priv = dev->dev_private;
 842         drm_buf_t *buf;
 843         drm_r128_buf_priv_t *buf_priv;
 844         drm_r128_freelist_t *entry;
 845         int i;
 846
 847         dev_priv->head = DRM(alloc)( sizeof(drm_r128_freelist_t),
 848                                      DRM_MEM_DRIVER );
 849         if ( dev_priv->head == NULL )
 850                 return DRM_ERR(ENOMEM);
 851
 852         memset( dev_priv->head, 0, sizeof(drm_r128_freelist_t) );
 853         dev_priv->head->age = R128_BUFFER_USED;
 854
 855         for ( i = 0 ; i < dma->buf_count ; i++ ) {
 856                 buf = dma->buflist[i];
 857                 buf_priv = buf->dev_private;
 858
 859                 entry = DRM(alloc)( sizeof(drm_r128_freelist_t),
 860                                     DRM_MEM_DRIVER );
 861                 if ( !entry ) return DRM_ERR(ENOMEM);
 862
 863                 entry->age = R128_BUFFER_FREE;
 864                 entry->buf = buf;
 865                 entry->prev = dev_priv->head;
 866                 entry->next = dev_priv->head->next;
 867                 if ( !entry->next )
 868                         dev_priv->tail = entry;
 869
 870                 buf_priv->discard = 0;
 871                 buf_priv->dispatched = 0;
 872                 buf_priv->list_entry = entry;
 873
 874                 dev_priv->head->next = entry;
 875
 876                 if ( dev_priv->head->next )
 877                         dev_priv->head->next->prev = entry;
 878         }
 879
 880         return 0;
 881
 882 }
 883 #endif
 884
 885 drm_buf_t *r128_freelist_get( drm_device_t *dev )
 886 {
 887         drm_device_dma_t *dma = dev->dma;
 888         drm_r128_private_t *dev_priv = dev->dev_private;
 889         drm_r128_buf_priv_t *buf_priv;
 890         drm_buf_t *buf;
 891         int i, t;
 892
 893         /* FIXME: Optimize -- use freelist code */
 894
 895         for ( i = 0 ; i < dma->buf_count ; i++ ) {
 896                 buf = dma->buflist[i];
 897                 buf_priv = buf->dev_private;
 898                 if ( buf->filp == 0 )
 899                         return buf;
 900         }
 901
 902         for ( t = 0 ; t < dev_priv->usec_timeout ; t++ ) {
 903                 u32 done_age = R128_READ( R128_LAST_DISPATCH_REG );
 904
 905                 for ( i = 0 ; i < dma->buf_count ; i++ ) {
 906                         buf = dma->buflist[i];
 907                         buf_priv = buf->dev_private;
 908                         if ( buf->pending && buf_priv->age <= done_age ) {
 909                                 /* The buffer has been processed, so it
 910                                  * can now be used.
 911                                  */
 912                                 buf->pending = 0;
 913                                 return buf;
 914                         }
 915                 }
 916                 DRM_UDELAY( 1 );
 917         }
 918
 919         DRM_ERROR( "returning NULL!\n" );
 920         return NULL;
 921 }
 922
 923 void r128_freelist_reset( drm_device_t *dev )
 924 {
 925         drm_device_dma_t *dma = dev->dma;
 926         int i;
 927
 928         for ( i = 0 ; i < dma->buf_count ; i++ ) {
 929                 drm_buf_t *buf = dma->buflist[i];
 930                 drm_r128_buf_priv_t *buf_priv = buf->dev_private;
 931                 buf_priv->age = 0;
 932         }
 933 }
 934
 935
 936 /* ================================================================
 937  * CCE command submission
 938  */
 939
 940 int r128_wait_ring( drm_r128_private_t *dev_priv, int n )
 941 {
 942         drm_r128_ring_buffer_t *ring = &dev_priv->ring;
 943         int i;
 944
 945         for ( i = 0 ; i < dev_priv->usec_timeout ; i++ ) {
 946                 r128_update_ring_snapshot( ring );
 947                 if ( ring->space >= n )
 948                         return 0;
 949                 DRM_UDELAY( 1 );
 950         }
 951
 952         /* FIXME: This is being ignored... */
 953         DRM_ERROR( "failed!\n" );
 954         return DRM_ERR(EBUSY);
 955 }
 956
 957 static int r128_cce_get_buffers( DRMFILE filp, drm_device_t *dev, drm_dma_t *d )
 958 {
 959         int i;
 960         drm_buf_t *buf;
 961
 962         for ( i = d->granted_count ; i < d->request_count ; i++ ) {
 963                 buf = r128_freelist_get( dev );
 964                 if ( !buf ) return DRM_ERR(EAGAIN);
 965
 966                 buf->filp = filp;
 967
 968                 if ( DRM_COPY_TO_USER( &d->request_indices[i], &buf->idx,
 969                                    sizeof(buf->idx) ) )
 970                         return DRM_ERR(EFAULT);
 971                 if ( DRM_COPY_TO_USER( &d->request_sizes[i], &buf->total,
 972                                    sizeof(buf->total) ) )
 973                         return DRM_ERR(EFAULT);
 974
 975                 d->granted_count++;
 976         }
 977         return 0;
 978 }
 979
 980 int r128_cce_buffers( DRM_IOCTL_ARGS )
 981 {
 982         DRM_DEVICE;
 983         drm_device_dma_t *dma = dev->dma;
 984         int ret = 0;
 985         drm_dma_t d;
 986
 987         LOCK_TEST_WITH_RETURN( dev, filp );
 988
 989         DRM_COPY_FROM_USER_IOCTL( d, (drm_dma_t *) data, sizeof(d) );
 990
 991         /* Please don't send us buffers.
 992          */
 993         if ( d.send_count != 0 ) {
 994                 DRM_ERROR( "Process %d trying to send %d buffers via drmDMA\n",
 995                            DRM_CURRENTPID, d.send_count );
 996                 return DRM_ERR(EINVAL);
 997         }
 998
 999         /* We'll send you buffers.
1000          */
1001         if ( d.request_count < 0 || d.request_count > dma->buf_count ) {
1002                 DRM_ERROR( "Process %d trying to get %d buffers (of %d max)\n",
1003                            DRM_CURRENTPID, d.request_count, dma->buf_count );
1004                 return DRM_ERR(EINVAL);
1005         }
1006
1007         d.granted_count = 0;
1008
1009         if ( d.request_count ) {
1010                 ret = r128_cce_get_buffers( filp, dev, &d );
1011         }
1012
1013         DRM_COPY_TO_USER_IOCTL((drm_dma_t *) data, d, sizeof(d) );
1014
1015         return ret;
1016 }