sys/dev/drm/radeon/radeon_cp.c

   1 /*-
   2  * Copyright 2000 Precision Insight, Inc., Cedar Park, Texas.
   3  * Copyright 2000 VA Linux Systems, Inc., Fremont, California.
   4  * Copyright 2007 Advanced Micro Devices, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11  * and/or sell copies of the Software, and to permit persons to whom the
  12  * Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24  * DEALINGS IN THE SOFTWARE.
  25  *
  26  * Authors:
  27  *    Kevin E. Martin <martin@valinux.com>
  28  *    Gareth Hughes <gareth@valinux.com>
  29  * __FBSDID("$FreeBSD: src/sys/dev/drm/radeon_cp.c,v 1.36 2009/10/30 18:07:22 rnoland Exp $");
  30  */
  31
  32 #include <drm/drmP.h>
  33 #include "radeon_drm.h"
  34 #include "radeon_drv.h"
  35 #include "r300_reg.h"
  36
  37 #include "radeon_microcode.h"
  38
  39 #define RADEON_FIFO_DEBUG       0
  40
  41 static int radeon_do_cleanup_cp(struct drm_device * dev);
  42 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv);
  43
  44 u32 radeon_read_ring_rptr(drm_radeon_private_t *dev_priv, u32 off)
  45 {
  46         u32 val;
  47
  48         if (dev_priv->flags & RADEON_IS_AGP) {
  49                 val = DRM_READ32(dev_priv->ring_rptr, off);
  50         } else {
  51                 val = *(((volatile u32 *)
  52                          dev_priv->ring_rptr->virtual) +
  53                         (off / sizeof(u32)));
  54                 val = le32_to_cpu(val);
  55         }
  56         return val;
  57 }
  58
  59 u32 radeon_get_ring_head(drm_radeon_private_t *dev_priv)
  60 {
  61         if (dev_priv->writeback_works)
  62                 return radeon_read_ring_rptr(dev_priv, 0);
  63         else {
  64                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
  65                         return RADEON_READ(R600_CP_RB_RPTR);
  66                 else
  67                         return RADEON_READ(RADEON_CP_RB_RPTR);
  68         }
  69 }
  70
  71 void radeon_write_ring_rptr(drm_radeon_private_t *dev_priv, u32 off, u32 val)
  72 {
  73         if (dev_priv->flags & RADEON_IS_AGP)
  74                 DRM_WRITE32(dev_priv->ring_rptr, off, val);
  75         else
  76                 *(((volatile u32 *) dev_priv->ring_rptr->virtual) +
  77                   (off / sizeof(u32))) = cpu_to_le32(val);
  78 }
  79
  80 void radeon_set_ring_head(drm_radeon_private_t *dev_priv, u32 val)
  81 {
  82         radeon_write_ring_rptr(dev_priv, 0, val);
  83 }
  84
  85 u32 radeon_get_scratch(drm_radeon_private_t *dev_priv, int index)
  86 {
  87         if (dev_priv->writeback_works) {
  88                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
  89                         return radeon_read_ring_rptr(dev_priv,
  90                                                      R600_SCRATCHOFF(index));
  91                 else
  92                         return radeon_read_ring_rptr(dev_priv,
  93                                                      RADEON_SCRATCHOFF(index));
  94         } else {
  95                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
  96                         return RADEON_READ(R600_SCRATCH_REG0 + 4*index);
  97                 else
  98                         return RADEON_READ(RADEON_SCRATCH_REG0 + 4*index);
  99         }
 100 }
 101
 102 u32 RADEON_READ_MM(drm_radeon_private_t *dev_priv, int addr)
 103 {
 104         u32 ret;
 105
 106         if (addr < 0x10000)
 107                 ret = DRM_READ32(dev_priv->mmio, addr);
 108         else {
 109                 DRM_WRITE32(dev_priv->mmio, RADEON_MM_INDEX, addr);
 110                 ret = DRM_READ32(dev_priv->mmio, RADEON_MM_DATA);
 111         }
 112
 113         return ret;
 114 }
 115
 116 static u32 R500_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 117 {
 118         u32 ret;
 119         RADEON_WRITE(R520_MC_IND_INDEX, 0x7f0000 | (addr & 0xff));
 120         ret = RADEON_READ(R520_MC_IND_DATA);
 121         RADEON_WRITE(R520_MC_IND_INDEX, 0);
 122         return ret;
 123 }
 124
 125 static u32 RS480_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 126 {
 127         u32 ret;
 128         RADEON_WRITE(RS480_NB_MC_INDEX, addr & 0xff);
 129         ret = RADEON_READ(RS480_NB_MC_DATA);
 130         RADEON_WRITE(RS480_NB_MC_INDEX, 0xff);
 131         return ret;
 132 }
 133
 134 static u32 RS690_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 135 {
 136         u32 ret;
 137         RADEON_WRITE(RS690_MC_INDEX, (addr & RS690_MC_INDEX_MASK));
 138         ret = RADEON_READ(RS690_MC_DATA);
 139         RADEON_WRITE(RS690_MC_INDEX, RS690_MC_INDEX_MASK);
 140         return ret;
 141 }
 142
 143 static u32 RS600_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 144 {
 145         u32 ret;
 146         RADEON_WRITE(RS600_MC_INDEX, ((addr & RS600_MC_ADDR_MASK) |
 147                                       RS600_MC_IND_CITF_ARB0));
 148         ret = RADEON_READ(RS600_MC_DATA);
 149         return ret;
 150 }
 151
 152 static u32 IGP_READ_MCIND(drm_radeon_private_t *dev_priv, int addr)
 153 {
 154         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 155             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 156                 return RS690_READ_MCIND(dev_priv, addr);
 157         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 158                 return RS600_READ_MCIND(dev_priv, addr);
 159         else
 160                 return RS480_READ_MCIND(dev_priv, addr);
 161 }
 162
 163 u32 radeon_read_fb_location(drm_radeon_private_t *dev_priv)
 164 {
 165
 166         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 167                 return RADEON_READ(R700_MC_VM_FB_LOCATION);
 168         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
 169                 return RADEON_READ(R600_MC_VM_FB_LOCATION);
 170         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 171                 return R500_READ_MCIND(dev_priv, RV515_MC_FB_LOCATION);
 172         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 173                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 174                 return RS690_READ_MCIND(dev_priv, RS690_MC_FB_LOCATION);
 175         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 176                 return RS600_READ_MCIND(dev_priv, RS600_MC_FB_LOCATION);
 177         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 178                 return R500_READ_MCIND(dev_priv, R520_MC_FB_LOCATION);
 179         else
 180                 return RADEON_READ(RADEON_MC_FB_LOCATION);
 181 }
 182
 183 static void radeon_write_fb_location(drm_radeon_private_t *dev_priv, u32 fb_loc)
 184 {
 185         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 186                 RADEON_WRITE(R700_MC_VM_FB_LOCATION, fb_loc);
 187         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
 188                 RADEON_WRITE(R600_MC_VM_FB_LOCATION, fb_loc);
 189         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 190                 R500_WRITE_MCIND(RV515_MC_FB_LOCATION, fb_loc);
 191         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 192                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 193                 RS690_WRITE_MCIND(RS690_MC_FB_LOCATION, fb_loc);
 194         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 195                 RS600_WRITE_MCIND(RS600_MC_FB_LOCATION, fb_loc);
 196         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 197                 R500_WRITE_MCIND(R520_MC_FB_LOCATION, fb_loc);
 198         else
 199                 RADEON_WRITE(RADEON_MC_FB_LOCATION, fb_loc);
 200 }
 201
 202 void radeon_write_agp_location(drm_radeon_private_t *dev_priv, u32 agp_loc)
 203 {
 204         /*R6xx/R7xx: AGP_TOP and BOT are actually 18 bits each */
 205         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) {
 206                 RADEON_WRITE(R700_MC_VM_AGP_BOT, agp_loc & 0xffff); /* FIX ME */
 207                 RADEON_WRITE(R700_MC_VM_AGP_TOP, (agp_loc >> 16) & 0xffff);
 208         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
 209                 RADEON_WRITE(R600_MC_VM_AGP_BOT, agp_loc & 0xffff); /* FIX ME */
 210                 RADEON_WRITE(R600_MC_VM_AGP_TOP, (agp_loc >> 16) & 0xffff);
 211         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515)
 212                 R500_WRITE_MCIND(RV515_MC_AGP_LOCATION, agp_loc);
 213         else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 214                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 215                 RS690_WRITE_MCIND(RS690_MC_AGP_LOCATION, agp_loc);
 216         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
 217                 RS600_WRITE_MCIND(RS600_MC_AGP_LOCATION, agp_loc);
 218         else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515)
 219                 R500_WRITE_MCIND(R520_MC_AGP_LOCATION, agp_loc);
 220         else
 221                 RADEON_WRITE(RADEON_MC_AGP_LOCATION, agp_loc);
 222 }
 223
 224 void radeon_write_agp_base(drm_radeon_private_t *dev_priv, u64 agp_base)
 225 {
 226         u32 agp_base_hi = upper_32_bits(agp_base);
 227         u32 agp_base_lo = agp_base & 0xffffffff;
 228         u32 r6xx_agp_base = (agp_base >> 22) & 0x3ffff;
 229
 230         /* R6xx/R7xx must be aligned to a 4MB boundry */
 231         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 232                 RADEON_WRITE(R700_MC_VM_AGP_BASE, r6xx_agp_base);
 233         else if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
 234                 RADEON_WRITE(R600_MC_VM_AGP_BASE, r6xx_agp_base);
 235         else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV515) {
 236                 R500_WRITE_MCIND(RV515_MC_AGP_BASE, agp_base_lo);
 237                 R500_WRITE_MCIND(RV515_MC_AGP_BASE_2, agp_base_hi);
 238         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 239                  ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 240                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE, agp_base_lo);
 241                 RS690_WRITE_MCIND(RS690_MC_AGP_BASE_2, agp_base_hi);
 242         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
 243                 RS600_WRITE_MCIND(RS600_AGP_BASE, agp_base_lo);
 244                 RS600_WRITE_MCIND(RS600_AGP_BASE_2, agp_base_hi);
 245         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_RV515) {
 246                 R500_WRITE_MCIND(R520_MC_AGP_BASE, agp_base_lo);
 247                 R500_WRITE_MCIND(R520_MC_AGP_BASE_2, agp_base_hi);
 248         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 249                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 250                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 251                 RADEON_WRITE(RS480_AGP_BASE_2, agp_base_hi);
 252         } else {
 253                 RADEON_WRITE(RADEON_AGP_BASE, agp_base_lo);
 254                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R200)
 255                         RADEON_WRITE(RADEON_AGP_BASE_2, agp_base_hi);
 256         }
 257 }
 258
 259 void radeon_enable_bm(struct drm_radeon_private *dev_priv)
 260 {
 261         u32 tmp;
 262         /* Turn on bus mastering */
 263         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 264             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740)) {
 265                 /* rs600/rs690/rs740 */
 266                 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RS600_BUS_MASTER_DIS;
 267                 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 268         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV350) ||
 269                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R420) ||
 270                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS400) ||
 271                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS480)) {
 272                 /* r1xx, r2xx, r300, r(v)350, r420/r481, rs400/rs480 */
 273                 tmp = RADEON_READ(RADEON_BUS_CNTL) & ~RADEON_BUS_MASTER_DIS;
 274                 RADEON_WRITE(RADEON_BUS_CNTL, tmp);
 275         } /* PCIE cards appears to not need this */
 276 }
 277
 278 static int RADEON_READ_PLL(struct drm_device * dev, int addr)
 279 {
 280         drm_radeon_private_t *dev_priv = dev->dev_private;
 281
 282         RADEON_WRITE8(RADEON_CLOCK_CNTL_INDEX, addr & 0x1f);
 283         return RADEON_READ(RADEON_CLOCK_CNTL_DATA);
 284 }
 285
 286 static u32 RADEON_READ_PCIE(drm_radeon_private_t *dev_priv, int addr)
 287 {
 288         RADEON_WRITE8(RADEON_PCIE_INDEX, addr & 0xff);
 289         return RADEON_READ(RADEON_PCIE_DATA);
 290 }
 291
 292 #if RADEON_FIFO_DEBUG
 293 static void radeon_status(drm_radeon_private_t * dev_priv)
 294 {
 295         printk("%s:\n", __func__);
 296         printk("RBBM_STATUS = 0x%08x\n",
 297                (unsigned int)RADEON_READ(RADEON_RBBM_STATUS));
 298         printk("CP_RB_RTPR = 0x%08x\n",
 299                (unsigned int)RADEON_READ(RADEON_CP_RB_RPTR));
 300         printk("CP_RB_WTPR = 0x%08x\n",
 301                (unsigned int)RADEON_READ(RADEON_CP_RB_WPTR));
 302         printk("AIC_CNTL = 0x%08x\n",
 303                (unsigned int)RADEON_READ(RADEON_AIC_CNTL));
 304         printk("AIC_STAT = 0x%08x\n",
 305                (unsigned int)RADEON_READ(RADEON_AIC_STAT));
 306         printk("AIC_PT_BASE = 0x%08x\n",
 307                (unsigned int)RADEON_READ(RADEON_AIC_PT_BASE));
 308         printk("TLB_ADDR = 0x%08x\n",
 309                (unsigned int)RADEON_READ(RADEON_AIC_TLB_ADDR));
 310         printk("TLB_DATA = 0x%08x\n",
 311                (unsigned int)RADEON_READ(RADEON_AIC_TLB_DATA));
 312 }
 313 #endif
 314
 315 /* ================================================================
 316  * Engine, FIFO control
 317  */
 318
 319 static int radeon_do_pixcache_flush(drm_radeon_private_t * dev_priv)
 320 {
 321         u32 tmp;
 322         int i;
 323
 324         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 325
 326         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV280) {
 327                 tmp = RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT);
 328                 tmp |= RADEON_RB3D_DC_FLUSH_ALL;
 329                 RADEON_WRITE(RADEON_RB3D_DSTCACHE_CTLSTAT, tmp);
 330
 331                 for (i = 0; i < dev_priv->usec_timeout; i++) {
 332                         if (!(RADEON_READ(RADEON_RB3D_DSTCACHE_CTLSTAT)
 333                               & RADEON_RB3D_DC_BUSY)) {
 334                                 return 0;
 335                         }
 336                         DRM_UDELAY(1);
 337                 }
 338         } else {
 339                 /* don't flush or purge cache here or lockup */
 340                 return 0;
 341         }
 342
 343 #if RADEON_FIFO_DEBUG
 344         DRM_ERROR("failed!\n");
 345         radeon_status(dev_priv);
 346 #endif
 347         return -EBUSY;
 348 }
 349
 350 static int radeon_do_wait_for_fifo(drm_radeon_private_t * dev_priv, int entries)
 351 {
 352         int i;
 353
 354         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 355
 356         for (i = 0; i < dev_priv->usec_timeout; i++) {
 357                 int slots = (RADEON_READ(RADEON_RBBM_STATUS)
 358                              & RADEON_RBBM_FIFOCNT_MASK);
 359                 if (slots >= entries)
 360                         return 0;
 361                 DRM_UDELAY(1);
 362         }
 363         DRM_DEBUG("wait for fifo failed status : 0x%08X 0x%08X\n",
 364                  RADEON_READ(RADEON_RBBM_STATUS),
 365                  RADEON_READ(R300_VAP_CNTL_STATUS));
 366
 367 #if RADEON_FIFO_DEBUG
 368         DRM_ERROR("failed!\n");
 369         radeon_status(dev_priv);
 370 #endif
 371         return -EBUSY;
 372 }
 373
 374 static int radeon_do_wait_for_idle(drm_radeon_private_t * dev_priv)
 375 {
 376         int i, ret;
 377
 378         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 379
 380         ret = radeon_do_wait_for_fifo(dev_priv, 64);
 381         if (ret)
 382                 return ret;
 383
 384         for (i = 0; i < dev_priv->usec_timeout; i++) {
 385                 if (!(RADEON_READ(RADEON_RBBM_STATUS)
 386                       & RADEON_RBBM_ACTIVE)) {
 387                         radeon_do_pixcache_flush(dev_priv);
 388                         return 0;
 389                 }
 390                 DRM_UDELAY(1);
 391         }
 392         DRM_DEBUG("wait idle failed status : 0x%08X 0x%08X\n",
 393                  RADEON_READ(RADEON_RBBM_STATUS),
 394                  RADEON_READ(R300_VAP_CNTL_STATUS));
 395
 396 #if RADEON_FIFO_DEBUG
 397         DRM_ERROR("failed!\n");
 398         radeon_status(dev_priv);
 399 #endif
 400         return -EBUSY;
 401 }
 402
 403 static void radeon_init_pipes(drm_radeon_private_t *dev_priv)
 404 {
 405         uint32_t gb_tile_config, gb_pipe_sel = 0;
 406
 407         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530) {
 408                 uint32_t z_pipe_sel = RADEON_READ(RV530_GB_PIPE_SELECT2);
 409                 if ((z_pipe_sel & 3) == 3)
 410                         dev_priv->num_z_pipes = 2;
 411                 else
 412                         dev_priv->num_z_pipes = 1;
 413         } else
 414                 dev_priv->num_z_pipes = 1;
 415
 416         /* RS4xx/RS6xx/R4xx/R5xx */
 417         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R420) {
 418                 gb_pipe_sel = RADEON_READ(R400_GB_PIPE_SELECT);
 419                 dev_priv->num_gb_pipes = ((gb_pipe_sel >> 12) & 0x3) + 1;
 420         } else {
 421                 /* R3xx */
 422                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R300) ||
 423                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R350)) {
 424                         dev_priv->num_gb_pipes = 2;
 425                 } else {
 426                         /* R3Vxx */
 427                         dev_priv->num_gb_pipes = 1;
 428                 }
 429         }
 430         DRM_INFO("Num pipes: %d\n", dev_priv->num_gb_pipes);
 431
 432         gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16 /*| R300_SUBPIXEL_1_16*/);
 433
 434         switch (dev_priv->num_gb_pipes) {
 435         case 2: gb_tile_config |= R300_PIPE_COUNT_R300; break;
 436         case 3: gb_tile_config |= R300_PIPE_COUNT_R420_3P; break;
 437         case 4: gb_tile_config |= R300_PIPE_COUNT_R420; break;
 438         default:
 439         case 1: gb_tile_config |= R300_PIPE_COUNT_RV350; break;
 440         }
 441
 442         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 443                 RADEON_WRITE_PLL(R500_DYN_SCLK_PWMEM_PIPE, (1 | ((gb_pipe_sel >> 8) & 0xf) << 4));
 444                 RADEON_WRITE(R300_SU_REG_DEST, ((1 << dev_priv->num_gb_pipes) - 1));
 445         }
 446         RADEON_WRITE(R300_GB_TILE_CONFIG, gb_tile_config);
 447         radeon_do_wait_for_idle(dev_priv);
 448         RADEON_WRITE(R300_DST_PIPE_CONFIG, RADEON_READ(R300_DST_PIPE_CONFIG) | R300_PIPE_AUTO_CONFIG);
 449         RADEON_WRITE(R300_RB2D_DSTCACHE_MODE, (RADEON_READ(R300_RB2D_DSTCACHE_MODE) |
 450                                                R300_DC_AUTOFLUSH_ENABLE |
 451                                                R300_DC_DC_DISABLE_IGNORE_PE));
 452
 453
 454 }
 455
 456 /* ================================================================
 457  * CP control, initialization
 458  */
 459
 460 /* Load the microcode for the CP */
 461 static void radeon_cp_load_microcode(drm_radeon_private_t * dev_priv)
 462 {
 463         const u32 (*cp)[2];
 464         int i;
 465
 466         DRM_DEBUG("\n");
 467
 468         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
 469         case CHIP_R100:
 470         case CHIP_RV100:
 471         case CHIP_RV200:
 472         case CHIP_RS100:
 473         case CHIP_RS200:
 474                 DRM_INFO("Loading R100 Microcode\n");
 475                 cp = R100_cp_microcode;
 476                 break;
 477         case CHIP_R200:
 478         case CHIP_RV250:
 479         case CHIP_RV280:
 480         case CHIP_RS300:
 481                 DRM_INFO("Loading R200 Microcode\n");
 482                 cp = R200_cp_microcode;
 483                 break;
 484         case CHIP_R300:
 485         case CHIP_R350:
 486         case CHIP_RV350:
 487         case CHIP_RV380:
 488         case CHIP_RS400:
 489         case CHIP_RS480:
 490                 DRM_INFO("Loading R300 Microcode\n");
 491                 cp = R300_cp_microcode;
 492                 break;
 493         case CHIP_R420:
 494         case CHIP_R423:
 495         case CHIP_RV410:
 496                 DRM_INFO("Loading R400 Microcode\n");
 497                 cp = R420_cp_microcode;
 498                 break;
 499         case CHIP_RS690:
 500         case CHIP_RS740:
 501                 DRM_INFO("Loading RS690/RS740 Microcode\n");
 502                 cp = RS690_cp_microcode;
 503                 break;
 504         case CHIP_RS600:
 505                 DRM_INFO("Loading RS600 Microcode\n");
 506                 cp = RS600_cp_microcode;
 507                 break;
 508         case CHIP_RV515:
 509         case CHIP_R520:
 510         case CHIP_RV530:
 511         case CHIP_R580:
 512         case CHIP_RV560:
 513         case CHIP_RV570:
 514                 DRM_INFO("Loading R500 Microcode\n");
 515                 cp = R520_cp_microcode;
 516                 break;
 517         default:
 518                 return;
 519         }
 520
 521         radeon_do_wait_for_idle(dev_priv);
 522
 523         RADEON_WRITE(RADEON_CP_ME_RAM_ADDR, 0);
 524
 525         for (i = 0; i != 256; i++) {
 526                 RADEON_WRITE(RADEON_CP_ME_RAM_DATAH, cp[i][1]);
 527                 RADEON_WRITE(RADEON_CP_ME_RAM_DATAL, cp[i][0]);
 528         }
 529 }
 530
 531 /* Flush any pending commands to the CP.  This should only be used just
 532  * prior to a wait for idle, as it informs the engine that the command
 533  * stream is ending.
 534  */
 535 static void radeon_do_cp_flush(drm_radeon_private_t * dev_priv)
 536 {
 537         DRM_DEBUG("\n");
 538 #if 0
 539         u32 tmp;
 540
 541         tmp = RADEON_READ(RADEON_CP_RB_WPTR) | (1 << 31);
 542         RADEON_WRITE(RADEON_CP_RB_WPTR, tmp);
 543 #endif
 544 }
 545
 546 /* Wait for the CP to go idle.
 547  */
 548 int radeon_do_cp_idle(drm_radeon_private_t * dev_priv)
 549 {
 550         RING_LOCALS;
 551         DRM_DEBUG("\n");
 552
 553         BEGIN_RING(6);
 554
 555         RADEON_PURGE_CACHE();
 556         RADEON_PURGE_ZCACHE();
 557         RADEON_WAIT_UNTIL_IDLE();
 558
 559         ADVANCE_RING();
 560         COMMIT_RING();
 561
 562         return radeon_do_wait_for_idle(dev_priv);
 563 }
 564
 565 /* Start the Command Processor.
 566  */
 567 static void radeon_do_cp_start(drm_radeon_private_t * dev_priv)
 568 {
 569         RING_LOCALS;
 570         DRM_DEBUG("\n");
 571
 572         radeon_do_wait_for_idle(dev_priv);
 573
 574         RADEON_WRITE(RADEON_CP_CSQ_CNTL, dev_priv->cp_mode);
 575
 576         dev_priv->cp_running = 1;
 577
 578         BEGIN_RING(8);
 579         /* isync can only be written through cp on r5xx write it here */
 580         OUT_RING(CP_PACKET0(RADEON_ISYNC_CNTL, 0));
 581         OUT_RING(RADEON_ISYNC_ANY2D_IDLE3D |
 582                  RADEON_ISYNC_ANY3D_IDLE2D |
 583                  RADEON_ISYNC_WAIT_IDLEGUI |
 584                  RADEON_ISYNC_CPSCRATCH_IDLEGUI);
 585         RADEON_PURGE_CACHE();
 586         RADEON_PURGE_ZCACHE();
 587         RADEON_WAIT_UNTIL_IDLE();
 588         ADVANCE_RING();
 589         COMMIT_RING();
 590
 591         dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 592 }
 593
 594 /* Reset the Command Processor.  This will not flush any pending
 595  * commands, so you must wait for the CP command stream to complete
 596  * before calling this routine.
 597  */
 598 static void radeon_do_cp_reset(drm_radeon_private_t * dev_priv)
 599 {
 600         u32 cur_read_ptr;
 601         DRM_DEBUG("\n");
 602
 603         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 604         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 605         SET_RING_HEAD(dev_priv, cur_read_ptr);
 606         dev_priv->ring.tail = cur_read_ptr;
 607 }
 608
 609 /* Stop the Command Processor.  This will not flush any pending
 610  * commands, so you must flush the command stream and wait for the CP
 611  * to go idle before calling this routine.
 612  */
 613 static void radeon_do_cp_stop(drm_radeon_private_t * dev_priv)
 614 {
 615         DRM_DEBUG("\n");
 616
 617         RADEON_WRITE(RADEON_CP_CSQ_CNTL, RADEON_CSQ_PRIDIS_INDDIS);
 618
 619         dev_priv->cp_running = 0;
 620 }
 621
 622 /* Reset the engine.  This will stop the CP if it is running.
 623  */
 624 static int radeon_do_engine_reset(struct drm_device * dev)
 625 {
 626         drm_radeon_private_t *dev_priv = dev->dev_private;
 627         u32 clock_cntl_index = 0, mclk_cntl = 0, rbbm_soft_reset;
 628         DRM_DEBUG("\n");
 629
 630         radeon_do_pixcache_flush(dev_priv);
 631
 632         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 633                 /* may need something similar for newer chips */
 634                 clock_cntl_index = RADEON_READ(RADEON_CLOCK_CNTL_INDEX);
 635                 mclk_cntl = RADEON_READ_PLL(dev, RADEON_MCLK_CNTL);
 636
 637                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, (mclk_cntl |
 638                                                     RADEON_FORCEON_MCLKA |
 639                                                     RADEON_FORCEON_MCLKB |
 640                                                     RADEON_FORCEON_YCLKA |
 641                                                     RADEON_FORCEON_YCLKB |
 642                                                     RADEON_FORCEON_MC |
 643                                                     RADEON_FORCEON_AIC));
 644         }
 645
 646         rbbm_soft_reset = RADEON_READ(RADEON_RBBM_SOFT_RESET);
 647
 648         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset |
 649                                               RADEON_SOFT_RESET_CP |
 650                                               RADEON_SOFT_RESET_HI |
 651                                               RADEON_SOFT_RESET_SE |
 652                                               RADEON_SOFT_RESET_RE |
 653                                               RADEON_SOFT_RESET_PP |
 654                                               RADEON_SOFT_RESET_E2 |
 655                                               RADEON_SOFT_RESET_RB));
 656         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 657         RADEON_WRITE(RADEON_RBBM_SOFT_RESET, (rbbm_soft_reset &
 658                                               ~(RADEON_SOFT_RESET_CP |
 659                                                 RADEON_SOFT_RESET_HI |
 660                                                 RADEON_SOFT_RESET_SE |
 661                                                 RADEON_SOFT_RESET_RE |
 662                                                 RADEON_SOFT_RESET_PP |
 663                                                 RADEON_SOFT_RESET_E2 |
 664                                                 RADEON_SOFT_RESET_RB)));
 665         RADEON_READ(RADEON_RBBM_SOFT_RESET);
 666
 667         if ((dev_priv->flags & RADEON_FAMILY_MASK) <= CHIP_RV410) {
 668                 RADEON_WRITE_PLL(RADEON_MCLK_CNTL, mclk_cntl);
 669                 RADEON_WRITE(RADEON_CLOCK_CNTL_INDEX, clock_cntl_index);
 670                 RADEON_WRITE(RADEON_RBBM_SOFT_RESET, rbbm_soft_reset);
 671         }
 672
 673         /* setup the raster pipes */
 674         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R300)
 675             radeon_init_pipes(dev_priv);
 676
 677         /* Reset the CP ring */
 678         radeon_do_cp_reset(dev_priv);
 679
 680         /* The CP is no longer running after an engine reset */
 681         dev_priv->cp_running = 0;
 682
 683         /* Reset any pending vertex, indirect buffers */
 684         radeon_freelist_reset(dev);
 685
 686         return 0;
 687 }
 688
 689 static void radeon_cp_init_ring_buffer(struct drm_device * dev,
 690                                        drm_radeon_private_t *dev_priv,
 691                                        struct drm_file *file_priv)
 692 {
 693         u32 ring_start, cur_read_ptr;
 694
 695         /* Initialize the memory controller. With new memory map, the fb location
 696          * is not changed, it should have been properly initialized already. Part
 697          * of the problem is that the code below is bogus, assuming the GART is
 698          * always appended to the fb which is not necessarily the case
 699          */
 700         if (!dev_priv->new_memmap)
 701                 radeon_write_fb_location(dev_priv,
 702                              ((dev_priv->gart_vm_start - 1) & 0xffff0000)
 703                              | (dev_priv->fb_location >> 16));
 704
 705 #if __OS_HAS_AGP
 706         if (dev_priv->flags & RADEON_IS_AGP) {
 707                 radeon_write_agp_base(dev_priv, dev->agp->base);
 708
 709                 radeon_write_agp_location(dev_priv,
 710                              (((dev_priv->gart_vm_start - 1 +
 711                                 dev_priv->gart_size) & 0xffff0000) |
 712                               (dev_priv->gart_vm_start >> 16)));
 713
 714                 ring_start = (dev_priv->cp_ring->offset
 715                               - dev->agp->base
 716                               + dev_priv->gart_vm_start);
 717         } else
 718 #endif
 719                 ring_start = (dev_priv->cp_ring->offset - dev->sg->vaddr +
 720                     dev_priv->gart_vm_start);
 721
 722         RADEON_WRITE(RADEON_CP_RB_BASE, ring_start);
 723
 724         /* Set the write pointer delay */
 725         RADEON_WRITE(RADEON_CP_RB_WPTR_DELAY, 0);
 726
 727         /* Initialize the ring buffer's read and write pointers */
 728         cur_read_ptr = RADEON_READ(RADEON_CP_RB_RPTR);
 729         RADEON_WRITE(RADEON_CP_RB_WPTR, cur_read_ptr);
 730         SET_RING_HEAD(dev_priv, cur_read_ptr);
 731         dev_priv->ring.tail = cur_read_ptr;
 732
 733 #if __OS_HAS_AGP
 734         if (dev_priv->flags & RADEON_IS_AGP) {
 735                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 736                              dev_priv->ring_rptr->offset
 737                              - dev->agp->base + dev_priv->gart_vm_start);
 738         } else
 739 #endif
 740         {
 741                 RADEON_WRITE(RADEON_CP_RB_RPTR_ADDR,
 742                     dev_priv->ring_rptr->offset - dev->sg->vaddr +
 743                     dev_priv->gart_vm_start);
 744         }
 745
 746         /* Set ring buffer size */
 747 #ifdef __BIG_ENDIAN
 748         RADEON_WRITE(RADEON_CP_RB_CNTL,
 749                      RADEON_BUF_SWAP_32BIT |
 750                      (dev_priv->ring.fetch_size_l2ow << 18) |
 751                      (dev_priv->ring.rptr_update_l2qw << 8) |
 752                      dev_priv->ring.size_l2qw);
 753 #else
 754         RADEON_WRITE(RADEON_CP_RB_CNTL,
 755                      (dev_priv->ring.fetch_size_l2ow << 18) |
 756                      (dev_priv->ring.rptr_update_l2qw << 8) |
 757                      dev_priv->ring.size_l2qw);
 758 #endif
 759
 760
 761         /* Initialize the scratch register pointer.  This will cause
 762          * the scratch register values to be written out to memory
 763          * whenever they are updated.
 764          *
 765          * We simply put this behind the ring read pointer, this works
 766          * with PCI GART as well as (whatever kind of) AGP GART
 767          */
 768         RADEON_WRITE(RADEON_SCRATCH_ADDR, RADEON_READ(RADEON_CP_RB_RPTR_ADDR)
 769                      + RADEON_SCRATCH_REG_OFFSET);
 770
 771         RADEON_WRITE(RADEON_SCRATCH_UMSK, 0x7);
 772
 773         radeon_enable_bm(dev_priv);
 774
 775         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(0), 0);
 776         RADEON_WRITE(RADEON_LAST_FRAME_REG, 0);
 777
 778         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
 779         RADEON_WRITE(RADEON_LAST_DISPATCH_REG, 0);
 780
 781         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(2), 0);
 782         RADEON_WRITE(RADEON_LAST_CLEAR_REG, 0);
 783
 784         /* reset sarea copies of these */
 785         if (dev_priv->sarea_priv) {
 786                 dev_priv->sarea_priv->last_frame = 0;
 787                 dev_priv->sarea_priv->last_dispatch = 0;
 788                 dev_priv->sarea_priv->last_clear = 0;
 789         }
 790
 791         radeon_do_wait_for_idle(dev_priv);
 792
 793         /* Sync everything up */
 794         RADEON_WRITE(RADEON_ISYNC_CNTL,
 795                      (RADEON_ISYNC_ANY2D_IDLE3D |
 796                       RADEON_ISYNC_ANY3D_IDLE2D |
 797                       RADEON_ISYNC_WAIT_IDLEGUI |
 798                       RADEON_ISYNC_CPSCRATCH_IDLEGUI));
 799
 800 }
 801
 802 static void radeon_test_writeback(drm_radeon_private_t * dev_priv)
 803 {
 804         u32 tmp;
 805
 806         /* Start with assuming that writeback doesn't work */
 807         dev_priv->writeback_works = 0;
 808
 809         /* Writeback doesn't seem to work everywhere, test it here and possibly
 810          * enable it if it appears to work
 811          */
 812         radeon_write_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1), 0);
 813
 814         RADEON_WRITE(RADEON_SCRATCH_REG1, 0xdeadbeef);
 815
 816         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 817                 u32 val;
 818
 819                 val = radeon_read_ring_rptr(dev_priv, RADEON_SCRATCHOFF(1));
 820                 if (val == 0xdeadbeef)
 821                         break;
 822                 DRM_UDELAY(1);
 823         }
 824
 825         if (tmp < dev_priv->usec_timeout) {
 826                 dev_priv->writeback_works = 1;
 827                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 828         } else {
 829                 dev_priv->writeback_works = 0;
 830                 DRM_INFO("writeback test failed\n");
 831         }
 832         if (radeon_no_wb == 1) {
 833                 dev_priv->writeback_works = 0;
 834                 DRM_INFO("writeback forced off\n");
 835         }
 836
 837         if (!dev_priv->writeback_works) {
 838                 /* Disable writeback to avoid unnecessary bus master transfer */
 839                 RADEON_WRITE(RADEON_CP_RB_CNTL, RADEON_READ(RADEON_CP_RB_CNTL) |
 840                              RADEON_RB_NO_UPDATE);
 841                 RADEON_WRITE(RADEON_SCRATCH_UMSK, 0);
 842         }
 843 }
 844
 845 /* Enable or disable IGP GART on the chip */
 846 static void radeon_set_igpgart(drm_radeon_private_t * dev_priv, int on)
 847 {
 848         u32 temp;
 849
 850         if (on) {
 851                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 852                           dev_priv->gart_vm_start,
 853                           (long)dev_priv->gart_info.bus_addr,
 854                           dev_priv->gart_size);
 855
 856                 temp = IGP_READ_MCIND(dev_priv, RS480_MC_MISC_CNTL);
 857                 if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
 858                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740))
 859                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, (RS480_GART_INDEX_REG_EN |
 860                                                              RS690_BLOCK_GFX_D3_EN));
 861                 else
 862                         IGP_WRITE_MCIND(RS480_MC_MISC_CNTL, RS480_GART_INDEX_REG_EN);
 863
 864                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 865                                                                RS480_VA_SIZE_32MB));
 866
 867                 temp = IGP_READ_MCIND(dev_priv, RS480_GART_FEATURE_ID);
 868                 IGP_WRITE_MCIND(RS480_GART_FEATURE_ID, (RS480_HANG_EN |
 869                                                         RS480_TLB_ENABLE |
 870                                                         RS480_GTW_LAC_EN |
 871                                                         RS480_1LEVEL_GART));
 872
 873                 temp = dev_priv->gart_info.bus_addr & 0xfffff000;
 874                 temp |= (upper_32_bits(dev_priv->gart_info.bus_addr) & 0xff) << 4;
 875                 IGP_WRITE_MCIND(RS480_GART_BASE, temp);
 876
 877                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_MODE_CNTL);
 878                 IGP_WRITE_MCIND(RS480_AGP_MODE_CNTL, ((1 << RS480_REQ_TYPE_SNOOP_SHIFT) |
 879                                                       RS480_REQ_TYPE_SNOOP_DIS));
 880
 881                 radeon_write_agp_base(dev_priv, dev_priv->gart_vm_start);
 882
 883                 dev_priv->gart_size = 32*1024*1024;
 884                 temp = (((dev_priv->gart_vm_start - 1 + dev_priv->gart_size) &
 885                          0xffff0000) | (dev_priv->gart_vm_start >> 16));
 886
 887                 radeon_write_agp_location(dev_priv, temp);
 888
 889                 temp = IGP_READ_MCIND(dev_priv, RS480_AGP_ADDRESS_SPACE_SIZE);
 890                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, (RS480_GART_EN |
 891                                                                RS480_VA_SIZE_32MB));
 892
 893                 do {
 894                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 895                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 896                                 break;
 897                         DRM_UDELAY(1);
 898                 } while (1);
 899
 900                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL,
 901                                 RS480_GART_CACHE_INVALIDATE);
 902
 903                 do {
 904                         temp = IGP_READ_MCIND(dev_priv, RS480_GART_CACHE_CNTRL);
 905                         if ((temp & RS480_GART_CACHE_INVALIDATE) == 0)
 906                                 break;
 907                         DRM_UDELAY(1);
 908                 } while (1);
 909
 910                 IGP_WRITE_MCIND(RS480_GART_CACHE_CNTRL, 0);
 911         } else {
 912                 IGP_WRITE_MCIND(RS480_AGP_ADDRESS_SPACE_SIZE, 0);
 913         }
 914 }
 915
 916 /* Enable or disable IGP GART on the chip */
 917 static void rs600_set_igpgart(drm_radeon_private_t *dev_priv, int on)
 918 {
 919         u32 temp;
 920         int i;
 921
 922         if (on) {
 923                 DRM_DEBUG("programming igp gart %08X %08lX %08X\n",
 924                          dev_priv->gart_vm_start,
 925                          (long)dev_priv->gart_info.bus_addr,
 926                          dev_priv->gart_size);
 927
 928                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, (RS600_EFFECTIVE_L2_CACHE_SIZE(6) |
 929                                                     RS600_EFFECTIVE_L2_QUEUE_SIZE(6)));
 930
 931                 for (i = 0; i < 19; i++)
 932                         IGP_WRITE_MCIND(RS600_MC_PT0_CLIENT0_CNTL + i,
 933                                         (RS600_ENABLE_TRANSLATION_MODE_OVERRIDE |
 934                                          RS600_SYSTEM_ACCESS_MODE_IN_SYS |
 935                                          RS600_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASSTHROUGH |
 936                                          RS600_EFFECTIVE_L1_CACHE_SIZE(3) |
 937                                          RS600_ENABLE_FRAGMENT_PROCESSING |
 938                                          RS600_EFFECTIVE_L1_QUEUE_SIZE(3)));
 939
 940                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_CNTL, (RS600_ENABLE_PAGE_TABLE |
 941                                                              RS600_PAGE_TABLE_TYPE_FLAT));
 942
 943                 /* disable all other contexts */
 944                 for (i = 1; i < 8; i++)
 945                         IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_CNTL + i, 0);
 946
 947                 /* setup the page table aperture */
 948                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_BASE_ADDR,
 949                                 dev_priv->gart_info.bus_addr);
 950                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_START_ADDR,
 951                                 dev_priv->gart_vm_start);
 952                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_FLAT_END_ADDR,
 953                                 (dev_priv->gart_vm_start + dev_priv->gart_size - 1));
 954                 IGP_WRITE_MCIND(RS600_MC_PT0_CONTEXT0_DEFAULT_READ_ADDR, 0);
 955
 956                 /* setup the system aperture */
 957                 IGP_WRITE_MCIND(RS600_MC_PT0_SYSTEM_APERTURE_LOW_ADDR,
 958                                 dev_priv->gart_vm_start);
 959                 IGP_WRITE_MCIND(RS600_MC_PT0_SYSTEM_APERTURE_HIGH_ADDR,
 960                                 (dev_priv->gart_vm_start + dev_priv->gart_size - 1));
 961
 962                 /* enable page tables */
 963                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 964                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, (temp | RS600_ENABLE_PT));
 965
 966                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_CNTL1);
 967                 IGP_WRITE_MCIND(RS600_MC_CNTL1, (temp | RS600_ENABLE_PAGE_TABLES));
 968
 969                 /* invalidate the cache */
 970                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 971
 972                 temp &= ~(RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE);
 973                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
 974                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 975
 976                 temp |= RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE;
 977                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
 978                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 979
 980                 temp &= ~(RS600_INVALIDATE_ALL_L1_TLBS | RS600_INVALIDATE_L2_CACHE);
 981                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, temp);
 982                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_PT0_CNTL);
 983
 984         } else {
 985                 IGP_WRITE_MCIND(RS600_MC_PT0_CNTL, 0);
 986                 temp = IGP_READ_MCIND(dev_priv, RS600_MC_CNTL1);
 987                 temp &= ~RS600_ENABLE_PAGE_TABLES;
 988                 IGP_WRITE_MCIND(RS600_MC_CNTL1, temp);
 989         }
 990 }
 991
 992 static void radeon_set_pciegart(drm_radeon_private_t * dev_priv, int on)
 993 {
 994         u32 tmp = RADEON_READ_PCIE(dev_priv, RADEON_PCIE_TX_GART_CNTL);
 995         if (on) {
 996
 997                 DRM_DEBUG("programming pcie %08X %08lX %08X\n",
 998                           dev_priv->gart_vm_start,
 999                           (long)dev_priv->gart_info.bus_addr,
1000                           dev_priv->gart_size);
1001                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO,
1002                                   dev_priv->gart_vm_start);
1003                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_BASE,
1004                                   dev_priv->gart_info.bus_addr);
1005                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_START_LO,
1006                                   dev_priv->gart_vm_start);
1007                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_END_LO,
1008                                   dev_priv->gart_vm_start +
1009                                   dev_priv->gart_size - 1);
1010
1011                 radeon_write_agp_location(dev_priv, 0xffffffc0); /* ?? */
1012
1013                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
1014                                   RADEON_PCIE_TX_GART_EN);
1015         } else {
1016                 RADEON_WRITE_PCIE(RADEON_PCIE_TX_GART_CNTL,
1017                                   tmp & ~RADEON_PCIE_TX_GART_EN);
1018         }
1019 }
1020
1021 /* Enable or disable PCI GART on the chip */
1022 static void radeon_set_pcigart(drm_radeon_private_t * dev_priv, int on)
1023 {
1024         u32 tmp;
1025
1026         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS690) ||
1027             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS740) ||
1028             (dev_priv->flags & RADEON_IS_IGPGART)) {
1029                 radeon_set_igpgart(dev_priv, on);
1030                 return;
1031         }
1032
1033         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600) {
1034                 rs600_set_igpgart(dev_priv, on);
1035                 return;
1036         }
1037
1038         if (dev_priv->flags & RADEON_IS_PCIE) {
1039                 radeon_set_pciegart(dev_priv, on);
1040                 return;
1041         }
1042
1043         tmp = RADEON_READ(RADEON_AIC_CNTL);
1044
1045         if (on) {
1046                 RADEON_WRITE(RADEON_AIC_CNTL,
1047                              tmp | RADEON_PCIGART_TRANSLATE_EN);
1048
1049                 /* set PCI GART page-table base address
1050                  */
1051                 RADEON_WRITE(RADEON_AIC_PT_BASE, dev_priv->gart_info.bus_addr);
1052
1053                 /* set address range for PCI address translate
1054                  */
1055                 RADEON_WRITE(RADEON_AIC_LO_ADDR, dev_priv->gart_vm_start);
1056                 RADEON_WRITE(RADEON_AIC_HI_ADDR, dev_priv->gart_vm_start
1057                              + dev_priv->gart_size - 1);
1058
1059                 /* Turn off AGP aperture -- is this required for PCI GART?
1060                  */
1061                 radeon_write_agp_location(dev_priv, 0xffffffc0);
1062                 RADEON_WRITE(RADEON_AGP_COMMAND, 0);    /* clear AGP_COMMAND */
1063         } else {
1064                 RADEON_WRITE(RADEON_AIC_CNTL,
1065                              tmp & ~RADEON_PCIGART_TRANSLATE_EN);
1066         }
1067 }
1068
1069 static int radeon_setup_pcigart_surface(drm_radeon_private_t *dev_priv)
1070 {
1071         struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
1072         struct radeon_virt_surface *vp;
1073         int i;
1074
1075         for (i = 0; i < RADEON_MAX_SURFACES * 2; i++) {
1076                 if (!dev_priv->virt_surfaces[i].file_priv ||
1077                     dev_priv->virt_surfaces[i].file_priv == PCIGART_FILE_PRIV)
1078                         break;
1079         }
1080         if (i >= 2 * RADEON_MAX_SURFACES)
1081                 return -ENOMEM;
1082         vp = &dev_priv->virt_surfaces[i];
1083
1084         for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1085                 struct radeon_surface *sp = &dev_priv->surfaces[i];
1086                 if (sp->refcount)
1087                         continue;
1088
1089                 vp->surface_index = i;
1090                 vp->lower = gart_info->bus_addr;
1091                 vp->upper = vp->lower + gart_info->table_size;
1092                 vp->flags = 0;
1093                 vp->file_priv = PCIGART_FILE_PRIV;
1094
1095                 sp->refcount = 1;
1096                 sp->lower = vp->lower;
1097                 sp->upper = vp->upper;
1098                 sp->flags = 0;
1099
1100                 RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, sp->flags);
1101                 RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND + 16 * i, sp->lower);
1102                 RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND + 16 * i, sp->upper);
1103                 return 0;
1104         }
1105
1106         return -ENOMEM;
1107 }
1108
1109 static int radeon_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
1110                              struct drm_file *file_priv)
1111 {
1112         drm_radeon_private_t *dev_priv = dev->dev_private;
1113
1114         DRM_DEBUG("\n");
1115
1116         /* if we require new memory map but we don't have it fail */
1117         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
1118                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
1119                 radeon_do_cleanup_cp(dev);
1120                 return -EINVAL;
1121         }
1122
1123         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
1124                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
1125                 dev_priv->flags &= ~RADEON_IS_AGP;
1126         } else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
1127                    && !init->is_pci) {
1128                 DRM_DEBUG("Restoring AGP flag\n");
1129                 dev_priv->flags |= RADEON_IS_AGP;
1130         }
1131
1132         if ((!(dev_priv->flags & RADEON_IS_AGP)) && !dev->sg) {
1133                 DRM_ERROR("PCI GART memory not allocated!\n");
1134                 radeon_do_cleanup_cp(dev);
1135                 return -EINVAL;
1136         }
1137
1138         dev_priv->usec_timeout = init->usec_timeout;
1139         if (dev_priv->usec_timeout < 1 ||
1140             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1141                 DRM_DEBUG("TIMEOUT problem!\n");
1142                 radeon_do_cleanup_cp(dev);
1143                 return -EINVAL;
1144         }
1145
1146         /* Enable vblank on CRTC1 for older X servers
1147          */
1148         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
1149
1150         switch(init->func) {
1151         case RADEON_INIT_R200_CP:
1152                 dev_priv->microcode_version = UCODE_R200;
1153                 break;
1154         case RADEON_INIT_R300_CP:
1155                 dev_priv->microcode_version = UCODE_R300;
1156                 break;
1157         default:
1158                 dev_priv->microcode_version = UCODE_R100;
1159         }
1160
1161         dev_priv->do_boxes = 0;
1162         dev_priv->cp_mode = init->cp_mode;
1163
1164         /* We don't support anything other than bus-mastering ring mode,
1165          * but the ring can be in either AGP or PCI space for the ring
1166          * read pointer.
1167          */
1168         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
1169             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
1170                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
1171                 radeon_do_cleanup_cp(dev);
1172                 return -EINVAL;
1173         }
1174
1175         switch (init->fb_bpp) {
1176         case 16:
1177                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
1178                 break;
1179         case 32:
1180         default:
1181                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
1182                 break;
1183         }
1184         dev_priv->front_offset = init->front_offset;
1185         dev_priv->front_pitch = init->front_pitch;
1186         dev_priv->back_offset = init->back_offset;
1187         dev_priv->back_pitch = init->back_pitch;
1188
1189         switch (init->depth_bpp) {
1190         case 16:
1191                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_16BIT_INT_Z;
1192                 break;
1193         case 32:
1194         default:
1195                 dev_priv->depth_fmt = RADEON_DEPTH_FORMAT_24BIT_INT_Z;
1196                 break;
1197         }
1198         dev_priv->depth_offset = init->depth_offset;
1199         dev_priv->depth_pitch = init->depth_pitch;
1200
1201         /* Hardware state for depth clears.  Remove this if/when we no
1202          * longer clear the depth buffer with a 3D rectangle.  Hard-code
1203          * all values to prevent unwanted 3D state from slipping through
1204          * and screwing with the clear operation.
1205          */
1206         dev_priv->depth_clear.rb3d_cntl = (RADEON_PLANE_MASK_ENABLE |
1207                                            (dev_priv->color_fmt << 10) |
1208                                            (dev_priv->microcode_version ==
1209                                             UCODE_R100 ? RADEON_ZBLOCK16 : 0));
1210
1211         dev_priv->depth_clear.rb3d_zstencilcntl =
1212             (dev_priv->depth_fmt |
1213              RADEON_Z_TEST_ALWAYS |
1214              RADEON_STENCIL_TEST_ALWAYS |
1215              RADEON_STENCIL_S_FAIL_REPLACE |
1216              RADEON_STENCIL_ZPASS_REPLACE |
1217              RADEON_STENCIL_ZFAIL_REPLACE | RADEON_Z_WRITE_ENABLE);
1218
1219         dev_priv->depth_clear.se_cntl = (RADEON_FFACE_CULL_CW |
1220                                          RADEON_BFACE_SOLID |
1221                                          RADEON_FFACE_SOLID |
1222                                          RADEON_FLAT_SHADE_VTX_LAST |
1223                                          RADEON_DIFFUSE_SHADE_FLAT |
1224                                          RADEON_ALPHA_SHADE_FLAT |
1225                                          RADEON_SPECULAR_SHADE_FLAT |
1226                                          RADEON_FOG_SHADE_FLAT |
1227                                          RADEON_VTX_PIX_CENTER_OGL |
1228                                          RADEON_ROUND_MODE_TRUNC |
1229                                          RADEON_ROUND_PREC_8TH_PIX);
1230
1231
1232         dev_priv->ring_offset = init->ring_offset;
1233         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
1234         dev_priv->buffers_offset = init->buffers_offset;
1235         dev_priv->gart_textures_offset = init->gart_textures_offset;
1236
1237         dev_priv->sarea = drm_getsarea(dev);
1238         if (!dev_priv->sarea) {
1239                 DRM_ERROR("could not find sarea!\n");
1240                 radeon_do_cleanup_cp(dev);
1241                 return -EINVAL;
1242         }
1243
1244         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
1245         if (!dev_priv->cp_ring) {
1246                 DRM_ERROR("could not find cp ring region!\n");
1247                 radeon_do_cleanup_cp(dev);
1248                 return -EINVAL;
1249         }
1250         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
1251         if (!dev_priv->ring_rptr) {
1252                 DRM_ERROR("could not find ring read pointer!\n");
1253                 radeon_do_cleanup_cp(dev);
1254                 return -EINVAL;
1255         }
1256         dev->agp_buffer_token = init->buffers_offset;
1257         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
1258         if (!dev->agp_buffer_map) {
1259                 DRM_ERROR("could not find dma buffer region!\n");
1260                 radeon_do_cleanup_cp(dev);
1261                 return -EINVAL;
1262         }
1263
1264         if (init->gart_textures_offset) {
1265                 dev_priv->gart_textures =
1266                     drm_core_findmap(dev, init->gart_textures_offset);
1267                 if (!dev_priv->gart_textures) {
1268                         DRM_ERROR("could not find GART texture region!\n");
1269                         radeon_do_cleanup_cp(dev);
1270                         return -EINVAL;
1271                 }
1272         }
1273
1274         dev_priv->sarea_priv =
1275             (drm_radeon_sarea_t *) ((u8 *) dev_priv->sarea->virtual +
1276                                     init->sarea_priv_offset);
1277
1278 #if __OS_HAS_AGP
1279         if (dev_priv->flags & RADEON_IS_AGP) {
1280                 drm_core_ioremap_wc(dev_priv->cp_ring, dev);
1281                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
1282                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
1283                 if (!dev_priv->cp_ring->virtual ||
1284                     !dev_priv->ring_rptr->virtual ||
1285                     !dev->agp_buffer_map->virtual) {
1286                         DRM_ERROR("could not find ioremap agp regions!\n");
1287                         radeon_do_cleanup_cp(dev);
1288                         return -EINVAL;
1289                 }
1290         } else
1291 #endif
1292         {
1293                 dev_priv->cp_ring->virtual =
1294                         (void *)(unsigned long)dev_priv->cp_ring->offset;
1295                 dev_priv->ring_rptr->virtual =
1296                         (void *)(unsigned long)dev_priv->ring_rptr->offset;
1297                 dev->agp_buffer_map->virtual =
1298                         (void *)(unsigned long)dev->agp_buffer_map->offset;
1299
1300                 DRM_DEBUG("dev_priv->cp_ring->virtual %p\n",
1301                           dev_priv->cp_ring->virtual);
1302                 DRM_DEBUG("dev_priv->ring_rptr->virtual %p\n",
1303                           dev_priv->ring_rptr->virtual);
1304                 DRM_DEBUG("dev->agp_buffer_map->virtual %p\n",
1305                           dev->agp_buffer_map->virtual);
1306         }
1307
1308         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 16;
1309         dev_priv->fb_size =
1310                 ((radeon_read_fb_location(dev_priv) & 0xffff0000u) + 0x10000)
1311                 - dev_priv->fb_location;
1312
1313         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
1314                                         ((dev_priv->front_offset
1315                                           + dev_priv->fb_location) >> 10));
1316
1317         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
1318                                        ((dev_priv->back_offset
1319                                          + dev_priv->fb_location) >> 10));
1320
1321         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
1322                                         ((dev_priv->depth_offset
1323                                           + dev_priv->fb_location) >> 10));
1324
1325         dev_priv->gart_size = init->gart_size;
1326
1327         /* New let's set the memory map ... */
1328         if (dev_priv->new_memmap) {
1329                 u32 base = 0;
1330
1331                 DRM_INFO("Setting GART location based on new memory map\n");
1332
1333                 /* If using AGP, try to locate the AGP aperture at the same
1334                  * location in the card and on the bus, though we have to
1335                  * align it down.
1336                  */
1337 #if __OS_HAS_AGP
1338                 if (dev_priv->flags & RADEON_IS_AGP) {
1339                         base = dev->agp->base;
1340                         /* Check if valid */
1341                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
1342                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
1343                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
1344                                          dev->agp->base);
1345                                 base = 0;
1346                         }
1347                 }
1348 #endif
1349                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
1350                 if (base == 0) {
1351                         base = dev_priv->fb_location + dev_priv->fb_size;
1352                         if (base < dev_priv->fb_location ||
1353                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
1354                                 base = dev_priv->fb_location
1355                                         - dev_priv->gart_size;
1356                 }
1357                 dev_priv->gart_vm_start = base & 0xffc00000u;
1358                 if (dev_priv->gart_vm_start != base)
1359                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
1360                                  base, dev_priv->gart_vm_start);
1361         } else {
1362                 DRM_INFO("Setting GART location based on old memory map\n");
1363                 dev_priv->gart_vm_start = dev_priv->fb_location +
1364                         RADEON_READ(RADEON_CONFIG_APER_SIZE);
1365         }
1366
1367 #if __OS_HAS_AGP
1368         if (dev_priv->flags & RADEON_IS_AGP)
1369                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
1370                                                  - dev->agp->base
1371                                                  + dev_priv->gart_vm_start);
1372         else
1373 #endif
1374                 dev_priv->gart_buffers_offset = dev->agp_buffer_map->offset -
1375                     dev->sg->vaddr + dev_priv->gart_vm_start;
1376
1377         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
1378         DRM_DEBUG("dev_priv->gart_vm_start 0x%x\n", dev_priv->gart_vm_start);
1379         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%lx\n",
1380                   dev_priv->gart_buffers_offset);
1381
1382         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->virtual;
1383         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->virtual
1384                               + init->ring_size / sizeof(u32));
1385         dev_priv->ring.size = init->ring_size;
1386         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
1387
1388         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
1389         dev_priv->ring.rptr_update_l2qw = drm_order( /* init->rptr_update */ 4096 / 8);
1390
1391         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
1392         dev_priv->ring.fetch_size_l2ow = drm_order( /* init->fetch_size */ 32 / 16);
1393         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
1394
1395         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
1396
1397 #if __OS_HAS_AGP
1398         if (dev_priv->flags & RADEON_IS_AGP) {
1399                 /* Turn off PCI GART */
1400                 radeon_set_pcigart(dev_priv, 0);
1401         } else
1402 #endif
1403         {
1404                 u32 sctrl;
1405                 int ret;
1406
1407                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
1408                 /* if we have an offset set from userspace */
1409                 if (dev_priv->pcigart_offset_set) {
1410                         dev_priv->gart_info.bus_addr =
1411                             dev_priv->pcigart_offset + dev_priv->fb_location;
1412                         dev_priv->gart_info.mapping.offset =
1413                             dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
1414                         dev_priv->gart_info.mapping.size =
1415                             dev_priv->gart_info.table_size;
1416
1417                         drm_core_ioremap_wc(&dev_priv->gart_info.mapping, dev);
1418                         dev_priv->gart_info.addr =
1419                             dev_priv->gart_info.mapping.virtual;
1420
1421                         if (dev_priv->flags & RADEON_IS_PCIE)
1422                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCIE;
1423                         else
1424                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1425                         dev_priv->gart_info.gart_table_location =
1426                             DRM_ATI_GART_FB;
1427
1428                         DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
1429                                   dev_priv->gart_info.addr,
1430                                   dev_priv->pcigart_offset);
1431                 } else {
1432                         if (dev_priv->flags & RADEON_IS_IGPGART)
1433                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_IGP;
1434                         else
1435                                 dev_priv->gart_info.gart_reg_if = DRM_ATI_GART_PCI;
1436                         dev_priv->gart_info.gart_table_location =
1437                             DRM_ATI_GART_MAIN;
1438                         dev_priv->gart_info.addr = NULL;
1439                         dev_priv->gart_info.bus_addr = 0;
1440                         if (dev_priv->flags & RADEON_IS_PCIE) {
1441                                 DRM_ERROR
1442                                     ("Cannot use PCI Express without GART in FB memory\n");
1443                                 radeon_do_cleanup_cp(dev);
1444                                 return -EINVAL;
1445                         }
1446                 }
1447
1448                 sctrl = RADEON_READ(RADEON_SURFACE_CNTL);
1449                 RADEON_WRITE(RADEON_SURFACE_CNTL, 0);
1450                 if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
1451                         ret = r600_page_table_init(dev);
1452                 else
1453                         ret = drm_ati_pcigart_init(dev, &dev_priv->gart_info);
1454                 RADEON_WRITE(RADEON_SURFACE_CNTL, sctrl);
1455
1456                 if (!ret) {
1457                         DRM_ERROR("failed to init PCI GART!\n");
1458                         radeon_do_cleanup_cp(dev);
1459                         return -ENOMEM;
1460                 }
1461
1462                 ret = radeon_setup_pcigart_surface(dev_priv);
1463                 if (ret) {
1464                         DRM_ERROR("failed to setup GART surface!\n");
1465                         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
1466                                 r600_page_table_cleanup(dev, &dev_priv->gart_info);
1467                         else
1468                                 drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info);
1469                         radeon_do_cleanup_cp(dev);
1470                         return ret;
1471                 }
1472
1473                 /* Turn on PCI GART */
1474                 radeon_set_pcigart(dev_priv, 1);
1475         }
1476
1477         radeon_cp_load_microcode(dev_priv);
1478         radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
1479
1480         dev_priv->last_buf = 0;
1481
1482         radeon_do_engine_reset(dev);
1483         radeon_test_writeback(dev_priv);
1484
1485         return 0;
1486 }
1487
1488 static int radeon_do_cleanup_cp(struct drm_device * dev)
1489 {
1490         drm_radeon_private_t *dev_priv = dev->dev_private;
1491         DRM_DEBUG("\n");
1492
1493         /* Make sure interrupts are disabled here because the uninstall ioctl
1494          * may not have been called from userspace and after dev_private
1495          * is freed, it's too late.
1496          */
1497         if (dev->irq_enabled)
1498                 drm_irq_uninstall(dev);
1499
1500 #if __OS_HAS_AGP
1501         if (dev_priv->flags & RADEON_IS_AGP) {
1502                 if (dev_priv->cp_ring != NULL) {
1503                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1504                         dev_priv->cp_ring = NULL;
1505                 }
1506                 if (dev_priv->ring_rptr != NULL) {
1507                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1508                         dev_priv->ring_rptr = NULL;
1509                 }
1510                 if (dev->agp_buffer_map != NULL) {
1511                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1512                         dev->agp_buffer_map = NULL;
1513                 }
1514         } else
1515 #endif
1516         {
1517
1518                 if (dev_priv->gart_info.bus_addr) {
1519                         /* Turn off PCI GART */
1520                         radeon_set_pcigart(dev_priv, 0);
1521                         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS600)
1522                                 r600_page_table_cleanup(dev, &dev_priv->gart_info);
1523                         else {
1524                                 if (!drm_ati_pcigart_cleanup(dev, &dev_priv->gart_info))
1525                                         DRM_ERROR("failed to cleanup PCI GART!\n");
1526                         }
1527                 }
1528
1529                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB)
1530                 {
1531                         drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1532                         dev_priv->gart_info.addr = 0;
1533                 }
1534         }
1535         /* only clear to the start of flags */
1536         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1537
1538         return 0;
1539 }
1540
1541 /* This code will reinit the Radeon CP hardware after a resume from disc.
1542  * AFAIK, it would be very difficult to pickle the state at suspend time, so
1543  * here we make sure that all Radeon hardware initialisation is re-done without
1544  * affecting running applications.
1545  *
1546  * Charl P. Botha <http://cpbotha.net>
1547  */
1548 static int radeon_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
1549 {
1550         drm_radeon_private_t *dev_priv = dev->dev_private;
1551
1552         if (!dev_priv) {
1553                 DRM_ERROR("Called with no initialization\n");
1554                 return -EINVAL;
1555         }
1556
1557         DRM_DEBUG("Starting radeon_do_resume_cp()\n");
1558
1559 #if __OS_HAS_AGP
1560         if (dev_priv->flags & RADEON_IS_AGP) {
1561                 /* Turn off PCI GART */
1562                 radeon_set_pcigart(dev_priv, 0);
1563         } else
1564 #endif
1565         {
1566                 /* Turn on PCI GART */
1567                 radeon_set_pcigart(dev_priv, 1);
1568         }
1569
1570         radeon_cp_load_microcode(dev_priv);
1571         radeon_cp_init_ring_buffer(dev, dev_priv, file_priv);
1572
1573         radeon_do_engine_reset(dev);
1574         radeon_irq_set_state(dev, RADEON_SW_INT_ENABLE, 1);
1575
1576         DRM_DEBUG("radeon_do_resume_cp() complete\n");
1577
1578         return 0;
1579 }
1580
1581 int radeon_cp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
1582 {
1583         drm_radeon_private_t *dev_priv = dev->dev_private;
1584         drm_radeon_init_t *init = data;
1585
1586         LOCK_TEST_WITH_RETURN(dev, file_priv);
1587
1588         if (init->func == RADEON_INIT_R300_CP)
1589                 r300_init_reg_flags(dev);
1590
1591         switch (init->func) {
1592         case RADEON_INIT_CP:
1593         case RADEON_INIT_R200_CP:
1594         case RADEON_INIT_R300_CP:
1595                 return radeon_do_init_cp(dev, init, file_priv);
1596         case RADEON_INIT_R600_CP:
1597                 return r600_do_init_cp(dev, init, file_priv);
1598         case RADEON_CLEANUP_CP:
1599                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1600                         return r600_do_cleanup_cp(dev);
1601                 else
1602                         return radeon_do_cleanup_cp(dev);
1603         }
1604
1605         return -EINVAL;
1606 }
1607
1608 int radeon_cp_start(struct drm_device *dev, void *data, struct drm_file *file_priv)
1609 {
1610         drm_radeon_private_t *dev_priv = dev->dev_private;
1611         DRM_DEBUG("\n");
1612
1613         LOCK_TEST_WITH_RETURN(dev, file_priv);
1614
1615         if (dev_priv->cp_running) {
1616                 DRM_DEBUG("while CP running\n");
1617                 return 0;
1618         }
1619         if (dev_priv->cp_mode == RADEON_CSQ_PRIDIS_INDDIS) {
1620                 DRM_DEBUG("called with bogus CP mode (%d)\n",
1621                           dev_priv->cp_mode);
1622                 return 0;
1623         }
1624
1625         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1626                 r600_do_cp_start(dev_priv);
1627         else
1628                 radeon_do_cp_start(dev_priv);
1629
1630         return 0;
1631 }
1632
1633 /* Stop the CP.  The engine must have been idled before calling this
1634  * routine.
1635  */
1636 int radeon_cp_stop(struct drm_device *dev, void *data, struct drm_file *file_priv)
1637 {
1638         drm_radeon_private_t *dev_priv = dev->dev_private;
1639         drm_radeon_cp_stop_t *stop = data;
1640         int ret;
1641         DRM_DEBUG("\n");
1642
1643         LOCK_TEST_WITH_RETURN(dev, file_priv);
1644
1645         if (!dev_priv->cp_running)
1646                 return 0;
1647
1648         /* Flush any pending CP commands.  This ensures any outstanding
1649          * commands are exectuted by the engine before we turn it off.
1650          */
1651         if (stop->flush) {
1652                 radeon_do_cp_flush(dev_priv);
1653         }
1654
1655         /* If we fail to make the engine go idle, we return an error
1656          * code so that the DRM ioctl wrapper can try again.
1657          */
1658         if (stop->idle) {
1659                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1660                         ret = r600_do_cp_idle(dev_priv);
1661                 else
1662                         ret = radeon_do_cp_idle(dev_priv);
1663                 if (ret)
1664                         return ret;
1665         }
1666
1667         /* Finally, we can turn off the CP.  If the engine isn't idle,
1668          * we will get some dropped triangles as they won't be fully
1669          * rendered before the CP is shut down.
1670          */
1671         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1672                 r600_do_cp_stop(dev_priv);
1673         else
1674                 radeon_do_cp_stop(dev_priv);
1675
1676         /* Reset the engine */
1677         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1678                 r600_do_engine_reset(dev);
1679         else
1680                 radeon_do_engine_reset(dev);
1681
1682         return 0;
1683 }
1684
1685 void radeon_do_release(struct drm_device * dev)
1686 {
1687         drm_radeon_private_t *dev_priv = dev->dev_private;
1688         int i, ret;
1689
1690         if (dev_priv) {
1691                 if (dev_priv->cp_running) {
1692                         /* Stop the cp */
1693                         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
1694                                 while ((ret = r600_do_cp_idle(dev_priv)) != 0) {
1695                                         DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1696                                         tsleep_interlock(&dev->lock.lock_queue,
1697                                                          PCATCH);
1698                                         DRM_UNLOCK(dev);
1699                                         ret = tsleep(&dev->lock.lock_queue,
1700                                                     PCATCH | PINTERLOCKED,
1701                                                     "rdnrel", 0);
1702                                         DRM_LOCK(dev);
1703 /* DragonFly equivalent of
1704  *                                      mtx_sleep(&ret, &dev->dev_lock, 0,
1705  *                                          "rdnrel", 1);
1706  */
1707                                 }
1708                         } else {
1709                                 while ((ret = radeon_do_cp_idle(dev_priv)) != 0) {
1710                                         DRM_DEBUG("radeon_do_cp_idle %d\n", ret);
1711                                         tsleep_interlock(&dev->lock.lock_queue,
1712                                                          PCATCH);
1713                                         DRM_UNLOCK(dev);
1714                                         ret = tsleep(&dev->lock.lock_queue,
1715                                                     PCATCH | PINTERLOCKED,
1716                                                     "rdnrel", 0);
1717                                         DRM_LOCK(dev);
1718                                 }
1719                         }
1720                         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
1721                                 r600_do_cp_stop(dev_priv);
1722                                 r600_do_engine_reset(dev);
1723                         } else {
1724                                 radeon_do_cp_stop(dev_priv);
1725                                 radeon_do_engine_reset(dev);
1726                         }
1727                 }
1728
1729                 if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_R600) {
1730                         /* Disable *all* interrupts */
1731                         if (dev_priv->mmio)     /* remove this after permanent addmaps */
1732                                 RADEON_WRITE(RADEON_GEN_INT_CNTL, 0);
1733
1734                         if (dev_priv->mmio) {   /* remove all surfaces */
1735                                 for (i = 0; i < RADEON_MAX_SURFACES; i++) {
1736                                         RADEON_WRITE(RADEON_SURFACE0_INFO + 16 * i, 0);
1737                                         RADEON_WRITE(RADEON_SURFACE0_LOWER_BOUND +
1738                                                      16 * i, 0);
1739                                         RADEON_WRITE(RADEON_SURFACE0_UPPER_BOUND +
1740                                                      16 * i, 0);
1741                                 }
1742                         }
1743                 }
1744
1745                 /* Free memory heap structures */
1746                 radeon_mem_takedown(&(dev_priv->gart_heap));
1747                 radeon_mem_takedown(&(dev_priv->fb_heap));
1748
1749                 /* deallocate kernel resources */
1750                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1751                         r600_do_cleanup_cp(dev);
1752                 else
1753                         radeon_do_cleanup_cp(dev);
1754         }
1755 }
1756
1757 /* Just reset the CP ring.  Called as part of an X Server engine reset.
1758  */
1759 int radeon_cp_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1760 {
1761         drm_radeon_private_t *dev_priv = dev->dev_private;
1762         DRM_DEBUG("\n");
1763
1764         LOCK_TEST_WITH_RETURN(dev, file_priv);
1765
1766         if (!dev_priv) {
1767                 DRM_DEBUG("called before init done\n");
1768                 return -EINVAL;
1769         }
1770
1771         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1772                 r600_do_cp_reset(dev_priv);
1773         else
1774                 radeon_do_cp_reset(dev_priv);
1775
1776         /* The CP is no longer running after an engine reset */
1777         dev_priv->cp_running = 0;
1778
1779         return 0;
1780 }
1781
1782 int radeon_cp_idle(struct drm_device *dev, void *data, struct drm_file *file_priv)
1783 {
1784         drm_radeon_private_t *dev_priv = dev->dev_private;
1785         DRM_DEBUG("\n");
1786
1787         LOCK_TEST_WITH_RETURN(dev, file_priv);
1788
1789         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1790                 return r600_do_cp_idle(dev_priv);
1791         else
1792                 return radeon_do_cp_idle(dev_priv);
1793 }
1794
1795 /* Added by Charl P. Botha to call radeon_do_resume_cp().
1796  */
1797 int radeon_cp_resume(struct drm_device *dev, void *data, struct drm_file *file_priv)
1798 {
1799         drm_radeon_private_t *dev_priv = dev->dev_private;
1800         DRM_DEBUG("\n");
1801
1802         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1803                 return r600_do_resume_cp(dev, file_priv);
1804         else
1805                 return radeon_do_resume_cp(dev, file_priv);
1806 }
1807
1808 int radeon_engine_reset(struct drm_device *dev, void *data, struct drm_file *file_priv)
1809 {
1810         drm_radeon_private_t *dev_priv = dev->dev_private;
1811         DRM_DEBUG("\n");
1812
1813         LOCK_TEST_WITH_RETURN(dev, file_priv);
1814
1815         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600)
1816                 return r600_do_engine_reset(dev);
1817         else
1818                 return radeon_do_engine_reset(dev);
1819 }
1820
1821 /* ================================================================
1822  * Fullscreen mode
1823  */
1824
1825 /* KW: Deprecated to say the least:
1826  */
1827 int radeon_fullscreen(struct drm_device *dev, void *data, struct drm_file *file_priv)
1828 {
1829         return 0;
1830 }
1831
1832 /* ================================================================
1833  * Freelist management
1834  */
1835
1836 /* Original comment: FIXME: ROTATE_BUFS is a hack to cycle through
1837  *   bufs until freelist code is used.  Note this hides a problem with
1838  *   the scratch register * (used to keep track of last buffer
1839  *   completed) being written to before * the last buffer has actually
1840  *   completed rendering.
1841  *
1842  * KW:  It's also a good way to find free buffers quickly.
1843  *
1844  * KW: Ideally this loop wouldn't exist, and freelist_get wouldn't
1845  * sleep.  However, bugs in older versions of radeon_accel.c mean that
1846  * we essentially have to do this, else old clients will break.
1847  *
1848  * However, it does leave open a potential deadlock where all the
1849  * buffers are held by other clients, which can't release them because
1850  * they can't get the lock.
1851  */
1852
1853 struct drm_buf *radeon_freelist_get(struct drm_device * dev)
1854 {
1855         struct drm_device_dma *dma = dev->dma;
1856         drm_radeon_private_t *dev_priv = dev->dev_private;
1857         drm_radeon_buf_priv_t *buf_priv;
1858         struct drm_buf *buf;
1859         int i, t;
1860         int start;
1861
1862         if (++dev_priv->last_buf >= dma->buf_count)
1863                 dev_priv->last_buf = 0;
1864
1865         start = dev_priv->last_buf;
1866
1867         for (t = 0; t < dev_priv->usec_timeout; t++) {
1868                 u32 done_age = GET_SCRATCH(dev_priv, 1);
1869                 DRM_DEBUG("done_age = %d\n", done_age);
1870                 for (i = 0; i < dma->buf_count; i++) {
1871                         buf = dma->buflist[start];
1872                         buf_priv = buf->dev_private;
1873                         if (buf->file_priv == NULL || (buf->pending &&
1874                                                        buf_priv->age <=
1875                                                        done_age)) {
1876                                 dev_priv->stats.requested_bufs++;
1877                                 buf->pending = 0;
1878                                 return buf;
1879                         }
1880                         if (++start >= dma->buf_count)
1881                                 start = 0;
1882                 }
1883
1884                 if (t) {
1885                         DRM_UDELAY(1);
1886                         dev_priv->stats.freelist_loops++;
1887                 }
1888         }
1889
1890         DRM_DEBUG("returning NULL!\n");
1891         return NULL;
1892 }
1893
1894 void radeon_freelist_reset(struct drm_device * dev)
1895 {
1896         struct drm_device_dma *dma = dev->dma;
1897         drm_radeon_private_t *dev_priv = dev->dev_private;
1898         int i;
1899
1900         dev_priv->last_buf = 0;
1901         for (i = 0; i < dma->buf_count; i++) {
1902                 struct drm_buf *buf = dma->buflist[i];
1903                 drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
1904                 buf_priv->age = 0;
1905         }
1906 }
1907
1908 /* ================================================================
1909  * CP command submission
1910  */
1911
1912 int radeon_wait_ring(drm_radeon_private_t * dev_priv, int n)
1913 {
1914         drm_radeon_ring_buffer_t *ring = &dev_priv->ring;
1915         int i;
1916         u32 last_head = GET_RING_HEAD(dev_priv);
1917
1918         for (i = 0; i < dev_priv->usec_timeout; i++) {
1919                 u32 head = GET_RING_HEAD(dev_priv);
1920
1921                 ring->space = (head - ring->tail) * sizeof(u32);
1922                 if (ring->space <= 0)
1923                         ring->space += ring->size;
1924                 if (ring->space > n)
1925                         return 0;
1926
1927                 dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
1928
1929                 if (head != last_head)
1930                         i = 0;
1931                 last_head = head;
1932
1933                 DRM_UDELAY(1);
1934         }
1935
1936         /* FIXME: This return value is ignored in the BEGIN_RING macro! */
1937 #if RADEON_FIFO_DEBUG
1938         radeon_status(dev_priv);
1939         DRM_ERROR("failed!\n");
1940 #endif
1941         return -EBUSY;
1942 }
1943
1944 static int radeon_cp_get_buffers(struct drm_device *dev,
1945                                  struct drm_file *file_priv,
1946                                  struct drm_dma * d)
1947 {
1948         int i;
1949         struct drm_buf *buf;
1950
1951         for (i = d->granted_count; i < d->request_count; i++) {
1952                 buf = radeon_freelist_get(dev);
1953                 if (!buf)
1954                         return -EBUSY;  /* NOTE: broken client */
1955
1956                 buf->file_priv = file_priv;
1957
1958                 if (DRM_COPY_TO_USER(&d->request_indices[i], &buf->idx,
1959                                      sizeof(buf->idx)))
1960                         return -EFAULT;
1961                 if (DRM_COPY_TO_USER(&d->request_sizes[i], &buf->total,
1962                                      sizeof(buf->total)))
1963                         return -EFAULT;
1964
1965                 d->granted_count++;
1966         }
1967         return 0;
1968 }
1969
1970 int radeon_cp_buffers(struct drm_device *dev, void *data, struct drm_file *file_priv)
1971 {
1972         struct drm_device_dma *dma = dev->dma;
1973         int ret = 0;
1974         struct drm_dma *d = data;
1975
1976         LOCK_TEST_WITH_RETURN(dev, file_priv);
1977
1978         /* Please don't send us buffers.
1979          */
1980         if (d->send_count != 0) {
1981                 DRM_ERROR("Process %d trying to send %d buffers via drmDMA\n",
1982                           DRM_CURRENTPID, d->send_count);
1983                 return -EINVAL;
1984         }
1985
1986         /* We'll send you buffers.
1987          */
1988         if (d->request_count < 0 || d->request_count > dma->buf_count) {
1989                 DRM_ERROR("Process %d trying to get %d buffers (of %d max)\n",
1990                           DRM_CURRENTPID, d->request_count, dma->buf_count);
1991                 return -EINVAL;
1992         }
1993
1994         d->granted_count = 0;
1995
1996         if (d->request_count) {
1997                 ret = radeon_cp_get_buffers(dev, file_priv, d);
1998         }
1999
2000         return ret;
2001 }
2002
2003 int radeon_driver_load(struct drm_device *dev, unsigned long flags)
2004 {
2005         drm_radeon_private_t *dev_priv;
2006         int ret = 0;
2007
2008         dev_priv = drm_alloc(sizeof(drm_radeon_private_t), DRM_MEM_DRIVER);
2009         if (dev_priv == NULL)
2010                 return -ENOMEM;
2011
2012         memset(dev_priv, 0, sizeof(drm_radeon_private_t));
2013         dev->dev_private = (void *)dev_priv;
2014         dev_priv->flags = flags;
2015
2016         switch (flags & RADEON_FAMILY_MASK) {
2017         case CHIP_R100:
2018         case CHIP_RV200:
2019         case CHIP_R200:
2020         case CHIP_R300:
2021         case CHIP_R350:
2022         case CHIP_R420:
2023         case CHIP_R423:
2024         case CHIP_RV410:
2025         case CHIP_RV515:
2026         case CHIP_R520:
2027         case CHIP_RV570:
2028         case CHIP_R580:
2029                 dev_priv->flags |= RADEON_HAS_HIERZ;
2030                 break;
2031         default:
2032                 /* all other chips have no hierarchical z buffer */
2033                 break;
2034         }
2035
2036         if (drm_device_is_agp(dev))
2037                 dev_priv->flags |= RADEON_IS_AGP;
2038         else if (drm_device_is_pcie(dev))
2039                 dev_priv->flags |= RADEON_IS_PCIE;
2040         else
2041                 dev_priv->flags |= RADEON_IS_PCI;
2042
2043         lockinit(&dev_priv->cs.cs_mutex, "cs_mtx", 0, LK_EXCLUSIVE);
2044
2045         ret = drm_addmap(dev, drm_get_resource_start(dev, 2),
2046                          drm_get_resource_len(dev, 2), _DRM_REGISTERS,
2047                          _DRM_READ_ONLY | _DRM_DRIVER, &dev_priv->mmio);
2048         if (ret != 0)
2049                 goto error;
2050
2051         ret = drm_vblank_init(dev, 2);
2052         if (ret != 0)
2053                 goto error;
2054
2055         dev->max_vblank_count = 0x001fffff;
2056
2057         DRM_DEBUG("%s card detected\n",
2058                   ((dev_priv->flags & RADEON_IS_AGP) ? "AGP" :
2059                     (((dev_priv->flags & RADEON_IS_PCIE) ? "PCIE" : "PCI"))));
2060
2061         return ret;
2062
2063 error:
2064         radeon_driver_unload(dev);
2065         return ret;
2066 }
2067
2068 /* Create mappings for registers and framebuffer so userland doesn't necessarily
2069  * have to find them.
2070  */
2071 int radeon_driver_firstopen(struct drm_device *dev)
2072 {
2073         int ret;
2074         drm_local_map_t *map;
2075         drm_radeon_private_t *dev_priv = dev->dev_private;
2076
2077         dev_priv->gart_info.table_size = RADEON_PCIGART_TABLE_SIZE;
2078
2079         dev_priv->fb_aper_offset = drm_get_resource_start(dev, 0);
2080         ret = drm_addmap(dev, dev_priv->fb_aper_offset,
2081                          drm_get_resource_len(dev, 0), _DRM_FRAME_BUFFER,
2082                          _DRM_WRITE_COMBINING, &map);
2083         if (ret != 0)
2084                 return ret;
2085
2086         return 0;
2087 }
2088
2089 int radeon_driver_unload(struct drm_device *dev)
2090 {
2091         drm_radeon_private_t *dev_priv = dev->dev_private;
2092
2093         DRM_DEBUG("\n");
2094
2095         drm_rmmap(dev, dev_priv->mmio);
2096
2097         lockuninit(&dev_priv->cs.cs_mutex);
2098
2099         drm_free(dev_priv, DRM_MEM_DRIVER);
2100
2101         dev->dev_private = NULL;
2102         return 0;
2103 }
2104
2105 void radeon_commit_ring(drm_radeon_private_t *dev_priv)
2106 {
2107         int i;
2108         u32 *ring;
2109         int tail_aligned;
2110
2111         /* check if the ring is padded out to 16-dword alignment */
2112
2113         tail_aligned = dev_priv->ring.tail & (RADEON_RING_ALIGN - 1);
2114         if (tail_aligned) {
2115                 int num_p2 = RADEON_RING_ALIGN - tail_aligned;
2116
2117                 ring = dev_priv->ring.start;
2118                 /* pad with some CP_PACKET2 */
2119                 for (i = 0; i < num_p2; i++)
2120                         ring[dev_priv->ring.tail + i] = CP_PACKET2();
2121
2122                 dev_priv->ring.tail += i;
2123
2124                 dev_priv->ring.space -= num_p2 * sizeof(u32);
2125         }
2126
2127         dev_priv->ring.tail &= dev_priv->ring.tail_mask;
2128
2129         DRM_MEMORYBARRIER();
2130         GET_RING_HEAD( dev_priv );
2131
2132         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_R600) {
2133                 RADEON_WRITE(R600_CP_RB_WPTR, dev_priv->ring.tail);
2134                 /* read from PCI bus to ensure correct posting */
2135                 RADEON_READ(R600_CP_RB_RPTR);
2136         } else {
2137                 RADEON_WRITE(RADEON_CP_RB_WPTR, dev_priv->ring.tail);
2138                 /* read from PCI bus to ensure correct posting */
2139                 RADEON_READ(RADEON_CP_RB_RPTR);
2140         }
2141 }