sys/dev/drm/radeon/r600_cp.c

   1 /*
   2  * Copyright 2008-2009 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *     Dave Airlie <airlied@redhat.com>
  26  *     Alex Deucher <alexander.deucher@amd.com>
  27  *
  28  * $FreeBSD: head/sys/dev/drm2/radeon/r600_cp.c 254885 2013-08-25 19:37:15Z dumbbell $
  29  */
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/linker.h>
  34 #include <sys/firmware.h>
  35
  36 #include <drm/drmP.h>
  37 #include <uapi_drm/radeon_drm.h>
  38 #include "radeon_drv.h"
  39 #include "r600_cp.h"
  40
  41 #define PFP_UCODE_SIZE 576
  42 #define PM4_UCODE_SIZE 1792
  43 #define R700_PFP_UCODE_SIZE 848
  44 #define R700_PM4_UCODE_SIZE 1360
  45
  46 # define ATI_PCIGART_PAGE_SIZE          4096    /**< PCI GART page size */
  47 # define ATI_PCIGART_PAGE_MASK          (~(ATI_PCIGART_PAGE_SIZE-1))
  48
  49 #define R600_PTE_VALID     (1 << 0)
  50 #define R600_PTE_SYSTEM    (1 << 1)
  51 #define R600_PTE_SNOOPED   (1 << 2)
  52 #define R600_PTE_READABLE  (1 << 5)
  53 #define R600_PTE_WRITEABLE (1 << 6)
  54
  55 /* MAX values used for gfx init */
  56 #define R6XX_MAX_SH_GPRS           256
  57 #define R6XX_MAX_TEMP_GPRS         16
  58 #define R6XX_MAX_SH_THREADS        256
  59 #define R6XX_MAX_SH_STACK_ENTRIES  4096
  60 #define R6XX_MAX_BACKENDS          8
  61 #define R6XX_MAX_BACKENDS_MASK     0xff
  62 #define R6XX_MAX_SIMDS             8
  63 #define R6XX_MAX_SIMDS_MASK        0xff
  64 #define R6XX_MAX_PIPES             8
  65 #define R6XX_MAX_PIPES_MASK        0xff
  66
  67 #define R7XX_MAX_SH_GPRS           256
  68 #define R7XX_MAX_TEMP_GPRS         16
  69 #define R7XX_MAX_SH_THREADS        256
  70 #define R7XX_MAX_SH_STACK_ENTRIES  4096
  71 #define R7XX_MAX_BACKENDS          8
  72 #define R7XX_MAX_BACKENDS_MASK     0xff
  73 #define R7XX_MAX_SIMDS             16
  74 #define R7XX_MAX_SIMDS_MASK        0xffff
  75 #define R7XX_MAX_PIPES             8
  76 #define R7XX_MAX_PIPES_MASK        0xff
  77
  78 static int r600_do_wait_for_fifo(drm_radeon_private_t *dev_priv, int entries)
  79 {
  80         int i;
  81
  82         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
  83
  84         for (i = 0; i < dev_priv->usec_timeout; i++) {
  85                 int slots;
  86                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
  87                         slots = (RADEON_READ(R600_GRBM_STATUS)
  88                                  & R700_CMDFIFO_AVAIL_MASK);
  89                 else
  90                         slots = (RADEON_READ(R600_GRBM_STATUS)
  91                                  & R600_CMDFIFO_AVAIL_MASK);
  92                 if (slots >= entries)
  93                         return 0;
  94                 DRM_UDELAY(1);
  95         }
  96         DRM_INFO("wait for fifo failed status : 0x%08X 0x%08X\n",
  97                  RADEON_READ(R600_GRBM_STATUS),
  98                  RADEON_READ(R600_GRBM_STATUS2));
  99
 100         return -EBUSY;
 101 }
 102
 103 static int r600_do_wait_for_idle(drm_radeon_private_t *dev_priv)
 104 {
 105         int i, ret;
 106
 107         dev_priv->stats.boxes |= RADEON_BOX_WAIT_IDLE;
 108
 109         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 110                 ret = r600_do_wait_for_fifo(dev_priv, 8);
 111         else
 112                 ret = r600_do_wait_for_fifo(dev_priv, 16);
 113         if (ret)
 114                 return ret;
 115         for (i = 0; i < dev_priv->usec_timeout; i++) {
 116                 if (!(RADEON_READ(R600_GRBM_STATUS) & R600_GUI_ACTIVE))
 117                         return 0;
 118                 DRM_UDELAY(1);
 119         }
 120         DRM_INFO("wait idle failed status : 0x%08X 0x%08X\n",
 121                  RADEON_READ(R600_GRBM_STATUS),
 122                  RADEON_READ(R600_GRBM_STATUS2));
 123
 124         return -EBUSY;
 125 }
 126
 127 void r600_page_table_cleanup(struct drm_device *dev, struct drm_ati_pcigart_info *gart_info)
 128 {
 129         struct drm_sg_mem *entry = dev->sg;
 130 #ifdef __linux__
 131         int max_pages;
 132         int pages;
 133         int i;
 134 #endif
 135
 136         if (!entry)
 137                 return;
 138
 139         if (gart_info->bus_addr) {
 140 #ifdef __linux__
 141                 max_pages = (gart_info->table_size / sizeof(u64));
 142                 pages = (entry->pages <= max_pages)
 143                   ? entry->pages : max_pages;
 144
 145                 for (i = 0; i < pages; i++) {
 146                         if (!entry->busaddr[i])
 147                                 break;
 148                         pci_unmap_page(dev->pdev, entry->busaddr[i],
 149                                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 150                 }
 151 #endif
 152                 if (gart_info->gart_table_location == DRM_ATI_GART_MAIN)
 153                         gart_info->bus_addr = 0;
 154         }
 155 }
 156
 157 /* R600 has page table setup */
 158 int r600_page_table_init(struct drm_device *dev)
 159 {
 160         drm_radeon_private_t *dev_priv = dev->dev_private;
 161         struct drm_ati_pcigart_info *gart_info = &dev_priv->gart_info;
 162         struct drm_local_map *map = &gart_info->mapping;
 163         struct drm_sg_mem *entry = dev->sg;
 164         int ret = 0;
 165         int i, j;
 166         int pages;
 167         u64 page_base;
 168         dma_addr_t entry_addr;
 169         int max_ati_pages, max_real_pages, gart_idx;
 170
 171         /* okay page table is available - lets rock */
 172         max_ati_pages = (gart_info->table_size / sizeof(u64));
 173         max_real_pages = max_ati_pages / (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE);
 174
 175         pages = (entry->pages <= max_real_pages) ?
 176                 entry->pages : max_real_pages;
 177
 178         memset_io((void __iomem *)map->handle, 0, max_ati_pages * sizeof(u64));
 179
 180         gart_idx = 0;
 181         for (i = 0; i < pages; i++) {
 182 #ifdef __linux__
 183                 entry->busaddr[i] = pci_map_page(dev->pdev,
 184                                                  entry->pagelist[i], 0,
 185                                                  PAGE_SIZE,
 186                                                  PCI_DMA_BIDIRECTIONAL);
 187                 if (pci_dma_mapping_error(dev->pdev, entry->busaddr[i])) {
 188                         DRM_ERROR("unable to map PCIGART pages!\n");
 189                         r600_page_table_cleanup(dev, gart_info);
 190                         goto done;
 191                 }
 192 #endif
 193                 entry_addr = entry->busaddr[i];
 194                 for (j = 0; j < (PAGE_SIZE / ATI_PCIGART_PAGE_SIZE); j++) {
 195                         page_base = (u64) entry_addr & ATI_PCIGART_PAGE_MASK;
 196                         page_base |= R600_PTE_VALID | R600_PTE_SYSTEM | R600_PTE_SNOOPED;
 197                         page_base |= R600_PTE_READABLE | R600_PTE_WRITEABLE;
 198
 199                         DRM_WRITE64(map, gart_idx * sizeof(u64), page_base);
 200
 201                         gart_idx++;
 202
 203                         if ((i % 128) == 0)
 204                                 DRM_DEBUG("page entry %d: 0x%016llx\n",
 205                                     i, (unsigned long long)page_base);
 206                         entry_addr += ATI_PCIGART_PAGE_SIZE;
 207                 }
 208         }
 209         ret = 1;
 210 #ifdef __linux__
 211 done:
 212 #endif
 213         return ret;
 214 }
 215
 216 static void r600_vm_flush_gart_range(struct drm_device *dev)
 217 {
 218         drm_radeon_private_t *dev_priv = dev->dev_private;
 219         u32 resp, countdown = 1000;
 220         RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 221         RADEON_WRITE(R600_VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 222         RADEON_WRITE(R600_VM_CONTEXT0_REQUEST_RESPONSE, 2);
 223
 224         do {
 225                 resp = RADEON_READ(R600_VM_CONTEXT0_REQUEST_RESPONSE);
 226                 countdown--;
 227                 DRM_UDELAY(1);
 228         } while (((resp & 0xf0) == 0) && countdown);
 229 }
 230
 231 static void r600_vm_init(struct drm_device *dev)
 232 {
 233         drm_radeon_private_t *dev_priv = dev->dev_private;
 234         /* initialise the VM to use the page table we constructed up there */
 235         u32 vm_c0, i;
 236         u32 mc_rd_a;
 237         u32 vm_l2_cntl, vm_l2_cntl3;
 238         /* okay set up the PCIE aperture type thingo */
 239         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 240         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 241         RADEON_WRITE(R600_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 242
 243         /* setup MC RD a */
 244         mc_rd_a = R600_MCD_L1_TLB | R600_MCD_L1_FRAG_PROC | R600_MCD_SYSTEM_ACCESS_MODE_IN_SYS |
 245                 R600_MCD_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | R600_MCD_EFFECTIVE_L1_TLB_SIZE(5) |
 246                 R600_MCD_EFFECTIVE_L1_QUEUE_SIZE(5) | R600_MCD_WAIT_L2_QUERY;
 247
 248         RADEON_WRITE(R600_MCD_RD_A_CNTL, mc_rd_a);
 249         RADEON_WRITE(R600_MCD_RD_B_CNTL, mc_rd_a);
 250
 251         RADEON_WRITE(R600_MCD_WR_A_CNTL, mc_rd_a);
 252         RADEON_WRITE(R600_MCD_WR_B_CNTL, mc_rd_a);
 253
 254         RADEON_WRITE(R600_MCD_RD_GFX_CNTL, mc_rd_a);
 255         RADEON_WRITE(R600_MCD_WR_GFX_CNTL, mc_rd_a);
 256
 257         RADEON_WRITE(R600_MCD_RD_SYS_CNTL, mc_rd_a);
 258         RADEON_WRITE(R600_MCD_WR_SYS_CNTL, mc_rd_a);
 259
 260         RADEON_WRITE(R600_MCD_RD_HDP_CNTL, mc_rd_a | R600_MCD_L1_STRICT_ORDERING);
 261         RADEON_WRITE(R600_MCD_WR_HDP_CNTL, mc_rd_a /*| R600_MCD_L1_STRICT_ORDERING*/);
 262
 263         RADEON_WRITE(R600_MCD_RD_PDMA_CNTL, mc_rd_a);
 264         RADEON_WRITE(R600_MCD_WR_PDMA_CNTL, mc_rd_a);
 265
 266         RADEON_WRITE(R600_MCD_RD_SEM_CNTL, mc_rd_a | R600_MCD_SEMAPHORE_MODE);
 267         RADEON_WRITE(R600_MCD_WR_SEM_CNTL, mc_rd_a);
 268
 269         vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
 270         vm_l2_cntl |= R600_VM_L2_CNTL_QUEUE_SIZE(7);
 271         RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
 272
 273         RADEON_WRITE(R600_VM_L2_CNTL2, 0);
 274         vm_l2_cntl3 = (R600_VM_L2_CNTL3_BANK_SELECT_0(0) |
 275                        R600_VM_L2_CNTL3_BANK_SELECT_1(1) |
 276                        R600_VM_L2_CNTL3_CACHE_UPDATE_MODE(2));
 277         RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
 278
 279         vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
 280
 281         RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
 282
 283         vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
 284
 285         /* disable all other contexts */
 286         for (i = 1; i < 8; i++)
 287                 RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
 288
 289         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
 290         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
 291         RADEON_WRITE(R600_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 292
 293         r600_vm_flush_gart_range(dev);
 294 }
 295
 296 static int r600_cp_init_microcode(drm_radeon_private_t *dev_priv)
 297 {
 298         const char *chip_name;
 299         size_t pfp_req_size, me_req_size;
 300         char fw_name[30];
 301         int err;
 302
 303         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
 304         case CHIP_R600:  chip_name = "R600";  break;
 305         case CHIP_RV610: chip_name = "RV610"; break;
 306         case CHIP_RV630: chip_name = "RV630"; break;
 307         case CHIP_RV620: chip_name = "RV620"; break;
 308         case CHIP_RV635: chip_name = "RV635"; break;
 309         case CHIP_RV670: chip_name = "RV670"; break;
 310         case CHIP_RS780:
 311         case CHIP_RS880: chip_name = "RS780"; break;
 312         case CHIP_RV770: chip_name = "RV770"; break;
 313         case CHIP_RV730:
 314         case CHIP_RV740: chip_name = "RV730"; break;
 315         case CHIP_RV710: chip_name = "RV710"; break;
 316         default:         panic("%s: Unsupported family %d", __func__, dev_priv->flags & RADEON_FAMILY_MASK);
 317         }
 318
 319         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) {
 320                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
 321                 me_req_size = R700_PM4_UCODE_SIZE * 4;
 322         } else {
 323                 pfp_req_size = PFP_UCODE_SIZE * 4;
 324                 me_req_size = PM4_UCODE_SIZE * 12;
 325         }
 326
 327         DRM_INFO("Loading %s CP Microcode\n", chip_name);
 328         err = 0;
 329
 330         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_pfp", chip_name);
 331         dev_priv->pfp_fw = firmware_get(fw_name);
 332         if (dev_priv->pfp_fw == NULL) {
 333                 err = -ENOENT;
 334                 goto out;
 335         }
 336         if (dev_priv->pfp_fw->datasize != pfp_req_size) {
 337                 DRM_ERROR(
 338                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
 339                        dev_priv->pfp_fw->datasize, fw_name);
 340                 err = -EINVAL;
 341                 goto out;
 342         }
 343
 344         ksnprintf(fw_name, sizeof(fw_name), "radeonkmsfw_%s_me", chip_name);
 345         dev_priv->me_fw = firmware_get(fw_name);
 346         if (dev_priv->me_fw == NULL) {
 347                 err = -ENOENT;
 348                 goto out;
 349         }
 350         if (dev_priv->me_fw->datasize != me_req_size) {
 351                 DRM_ERROR(
 352                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
 353                        dev_priv->me_fw->datasize, fw_name);
 354                 err = -EINVAL;
 355         }
 356 out:
 357         if (err) {
 358                 if (err != -EINVAL)
 359                         DRM_ERROR(
 360                                "r600_cp: Failed to load firmware \"%s\"\n",
 361                                fw_name);
 362                 if (dev_priv->pfp_fw != NULL) {
 363                         firmware_put(dev_priv->pfp_fw, FIRMWARE_UNLOAD);
 364                         dev_priv->pfp_fw = NULL;
 365                 }
 366                 if (dev_priv->me_fw != NULL) {
 367                         firmware_put(dev_priv->me_fw, FIRMWARE_UNLOAD);
 368                         dev_priv->me_fw = NULL;
 369                 }
 370         }
 371         return err;
 372 }
 373
 374 static void r600_cp_load_microcode(drm_radeon_private_t *dev_priv)
 375 {
 376         const __be32 *fw_data;
 377         int i;
 378
 379         if (!dev_priv->me_fw || !dev_priv->pfp_fw)
 380                 return;
 381
 382         r600_do_cp_stop(dev_priv);
 383
 384         RADEON_WRITE(R600_CP_RB_CNTL,
 385 #ifdef __BIG_ENDIAN
 386                      R600_BUF_SWAP_32BIT |
 387 #endif
 388                      R600_RB_NO_UPDATE |
 389                      R600_RB_BLKSZ(15) |
 390                      R600_RB_BUFSZ(3));
 391
 392         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
 393         RADEON_READ(R600_GRBM_SOFT_RESET);
 394         DRM_MDELAY(15);
 395         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 396
 397         fw_data = (const __be32 *)dev_priv->me_fw->data;
 398         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 399         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
 400                 RADEON_WRITE(R600_CP_ME_RAM_DATA,
 401                              be32_to_cpup(fw_data++));
 402
 403         fw_data = (const __be32 *)dev_priv->pfp_fw->data;
 404         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 405         for (i = 0; i < PFP_UCODE_SIZE; i++)
 406                 RADEON_WRITE(R600_CP_PFP_UCODE_DATA,
 407                              be32_to_cpup(fw_data++));
 408
 409         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 410         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 411         RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
 412
 413 }
 414
 415 static void r700_vm_init(struct drm_device *dev)
 416 {
 417         drm_radeon_private_t *dev_priv = dev->dev_private;
 418         /* initialise the VM to use the page table we constructed up there */
 419         u32 vm_c0, i;
 420         u32 mc_vm_md_l1;
 421         u32 vm_l2_cntl, vm_l2_cntl3;
 422         /* okay set up the PCIE aperture type thingo */
 423         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_LOW_ADDR, dev_priv->gart_vm_start >> 12);
 424         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_HIGH_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 425         RADEON_WRITE(R700_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
 426
 427         mc_vm_md_l1 = R700_ENABLE_L1_TLB |
 428             R700_ENABLE_L1_FRAGMENT_PROCESSING |
 429             R700_SYSTEM_ACCESS_MODE_IN_SYS |
 430             R700_SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
 431             R700_EFFECTIVE_L1_TLB_SIZE(5) |
 432             R700_EFFECTIVE_L1_QUEUE_SIZE(5);
 433
 434         RADEON_WRITE(R700_MC_VM_MD_L1_TLB0_CNTL, mc_vm_md_l1);
 435         RADEON_WRITE(R700_MC_VM_MD_L1_TLB1_CNTL, mc_vm_md_l1);
 436         RADEON_WRITE(R700_MC_VM_MD_L1_TLB2_CNTL, mc_vm_md_l1);
 437         RADEON_WRITE(R700_MC_VM_MB_L1_TLB0_CNTL, mc_vm_md_l1);
 438         RADEON_WRITE(R700_MC_VM_MB_L1_TLB1_CNTL, mc_vm_md_l1);
 439         RADEON_WRITE(R700_MC_VM_MB_L1_TLB2_CNTL, mc_vm_md_l1);
 440         RADEON_WRITE(R700_MC_VM_MB_L1_TLB3_CNTL, mc_vm_md_l1);
 441
 442         vm_l2_cntl = R600_VM_L2_CACHE_EN | R600_VM_L2_FRAG_PROC | R600_VM_ENABLE_PTE_CACHE_LRU_W;
 443         vm_l2_cntl |= R700_VM_L2_CNTL_QUEUE_SIZE(7);
 444         RADEON_WRITE(R600_VM_L2_CNTL, vm_l2_cntl);
 445
 446         RADEON_WRITE(R600_VM_L2_CNTL2, 0);
 447         vm_l2_cntl3 = R700_VM_L2_CNTL3_BANK_SELECT(0) | R700_VM_L2_CNTL3_CACHE_UPDATE_MODE(2);
 448         RADEON_WRITE(R600_VM_L2_CNTL3, vm_l2_cntl3);
 449
 450         vm_c0 = R600_VM_ENABLE_CONTEXT | R600_VM_PAGE_TABLE_DEPTH_FLAT;
 451
 452         RADEON_WRITE(R600_VM_CONTEXT0_CNTL, vm_c0);
 453
 454         vm_c0 &= ~R600_VM_ENABLE_CONTEXT;
 455
 456         /* disable all other contexts */
 457         for (i = 1; i < 8; i++)
 458                 RADEON_WRITE(R600_VM_CONTEXT0_CNTL + (i * 4), vm_c0);
 459
 460         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, dev_priv->gart_info.bus_addr >> 12);
 461         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_START_ADDR, dev_priv->gart_vm_start >> 12);
 462         RADEON_WRITE(R700_VM_CONTEXT0_PAGE_TABLE_END_ADDR, (dev_priv->gart_vm_start + dev_priv->gart_size - 1) >> 12);
 463
 464         r600_vm_flush_gart_range(dev);
 465 }
 466
 467 static void r700_cp_load_microcode(drm_radeon_private_t *dev_priv)
 468 {
 469         const __be32 *fw_data;
 470         int i;
 471
 472         if (!dev_priv->me_fw || !dev_priv->pfp_fw)
 473                 return;
 474
 475         r600_do_cp_stop(dev_priv);
 476
 477         RADEON_WRITE(R600_CP_RB_CNTL,
 478 #ifdef __BIG_ENDIAN
 479                      R600_BUF_SWAP_32BIT |
 480 #endif
 481                      R600_RB_NO_UPDATE |
 482                      R600_RB_BLKSZ(15) |
 483                      R600_RB_BUFSZ(3));
 484
 485         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
 486         RADEON_READ(R600_GRBM_SOFT_RESET);
 487         DRM_MDELAY(15);
 488         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 489
 490         fw_data = (const __be32 *)dev_priv->pfp_fw->data;
 491         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 492         for (i = 0; i < R700_PFP_UCODE_SIZE; i++)
 493                 RADEON_WRITE(R600_CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
 494         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 495
 496         fw_data = (const __be32 *)dev_priv->me_fw->data;
 497         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 498         for (i = 0; i < R700_PM4_UCODE_SIZE; i++)
 499                 RADEON_WRITE(R600_CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
 500         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 501
 502         RADEON_WRITE(R600_CP_PFP_UCODE_ADDR, 0);
 503         RADEON_WRITE(R600_CP_ME_RAM_WADDR, 0);
 504         RADEON_WRITE(R600_CP_ME_RAM_RADDR, 0);
 505
 506 }
 507
 508 static void r600_test_writeback(drm_radeon_private_t *dev_priv)
 509 {
 510         u32 tmp;
 511
 512         /* Start with assuming that writeback doesn't work */
 513         dev_priv->writeback_works = 0;
 514
 515         /* Writeback doesn't seem to work everywhere, test it here and possibly
 516          * enable it if it appears to work
 517          */
 518         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
 519
 520         RADEON_WRITE(R600_SCRATCH_REG1, 0xdeadbeef);
 521
 522         for (tmp = 0; tmp < dev_priv->usec_timeout; tmp++) {
 523                 u32 val;
 524
 525                 val = radeon_read_ring_rptr(dev_priv, R600_SCRATCHOFF(1));
 526                 if (val == 0xdeadbeef)
 527                         break;
 528                 DRM_UDELAY(1);
 529         }
 530
 531         if (tmp < dev_priv->usec_timeout) {
 532                 dev_priv->writeback_works = 1;
 533                 DRM_INFO("writeback test succeeded in %d usecs\n", tmp);
 534         } else {
 535                 dev_priv->writeback_works = 0;
 536                 DRM_INFO("writeback test failed\n");
 537         }
 538         if (radeon_no_wb == 1) {
 539                 dev_priv->writeback_works = 0;
 540                 DRM_INFO("writeback forced off\n");
 541         }
 542
 543         if (!dev_priv->writeback_works) {
 544                 /* Disable writeback to avoid unnecessary bus master transfer */
 545                 RADEON_WRITE(R600_CP_RB_CNTL,
 546 #ifdef __BIG_ENDIAN
 547                              R600_BUF_SWAP_32BIT |
 548 #endif
 549                              RADEON_READ(R600_CP_RB_CNTL) |
 550                              R600_RB_NO_UPDATE);
 551                 RADEON_WRITE(R600_SCRATCH_UMSK, 0);
 552         }
 553 }
 554
 555 int r600_do_engine_reset(struct drm_device *dev)
 556 {
 557         drm_radeon_private_t *dev_priv = dev->dev_private;
 558         u32 cp_ptr, cp_me_cntl, cp_rb_cntl;
 559
 560         DRM_INFO("Resetting GPU\n");
 561
 562         cp_ptr = RADEON_READ(R600_CP_RB_WPTR);
 563         cp_me_cntl = RADEON_READ(R600_CP_ME_CNTL);
 564         RADEON_WRITE(R600_CP_ME_CNTL, R600_CP_ME_HALT);
 565
 566         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0x7fff);
 567         RADEON_READ(R600_GRBM_SOFT_RESET);
 568         DRM_UDELAY(50);
 569         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
 570         RADEON_READ(R600_GRBM_SOFT_RESET);
 571
 572         RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
 573         cp_rb_cntl = RADEON_READ(R600_CP_RB_CNTL);
 574         RADEON_WRITE(R600_CP_RB_CNTL,
 575 #ifdef __BIG_ENDIAN
 576                      R600_BUF_SWAP_32BIT |
 577 #endif
 578                      R600_RB_RPTR_WR_ENA);
 579
 580         RADEON_WRITE(R600_CP_RB_RPTR_WR, cp_ptr);
 581         RADEON_WRITE(R600_CP_RB_WPTR, cp_ptr);
 582         RADEON_WRITE(R600_CP_RB_CNTL, cp_rb_cntl);
 583         RADEON_WRITE(R600_CP_ME_CNTL, cp_me_cntl);
 584
 585         /* Reset the CP ring */
 586         r600_do_cp_reset(dev_priv);
 587
 588         /* The CP is no longer running after an engine reset */
 589         dev_priv->cp_running = 0;
 590
 591         /* Reset any pending vertex, indirect buffers */
 592         radeon_freelist_reset(dev);
 593
 594         return 0;
 595
 596 }
 597
 598 static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
 599                                              u32 num_backends,
 600                                              u32 backend_disable_mask)
 601 {
 602         u32 backend_map = 0;
 603         u32 enabled_backends_mask;
 604         u32 enabled_backends_count;
 605         u32 cur_pipe;
 606         u32 swizzle_pipe[R6XX_MAX_PIPES];
 607         u32 cur_backend;
 608         u32 i;
 609
 610         if (num_tile_pipes > R6XX_MAX_PIPES)
 611                 num_tile_pipes = R6XX_MAX_PIPES;
 612         if (num_tile_pipes < 1)
 613                 num_tile_pipes = 1;
 614         if (num_backends > R6XX_MAX_BACKENDS)
 615                 num_backends = R6XX_MAX_BACKENDS;
 616         if (num_backends < 1)
 617                 num_backends = 1;
 618
 619         enabled_backends_mask = 0;
 620         enabled_backends_count = 0;
 621         for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
 622                 if (((backend_disable_mask >> i) & 1) == 0) {
 623                         enabled_backends_mask |= (1 << i);
 624                         ++enabled_backends_count;
 625                 }
 626                 if (enabled_backends_count == num_backends)
 627                         break;
 628         }
 629
 630         if (enabled_backends_count == 0) {
 631                 enabled_backends_mask = 1;
 632                 enabled_backends_count = 1;
 633         }
 634
 635         if (enabled_backends_count != num_backends)
 636                 num_backends = enabled_backends_count;
 637
 638         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R6XX_MAX_PIPES);
 639         switch (num_tile_pipes) {
 640         case 1:
 641                 swizzle_pipe[0] = 0;
 642                 break;
 643         case 2:
 644                 swizzle_pipe[0] = 0;
 645                 swizzle_pipe[1] = 1;
 646                 break;
 647         case 3:
 648                 swizzle_pipe[0] = 0;
 649                 swizzle_pipe[1] = 1;
 650                 swizzle_pipe[2] = 2;
 651                 break;
 652         case 4:
 653                 swizzle_pipe[0] = 0;
 654                 swizzle_pipe[1] = 1;
 655                 swizzle_pipe[2] = 2;
 656                 swizzle_pipe[3] = 3;
 657                 break;
 658         case 5:
 659                 swizzle_pipe[0] = 0;
 660                 swizzle_pipe[1] = 1;
 661                 swizzle_pipe[2] = 2;
 662                 swizzle_pipe[3] = 3;
 663                 swizzle_pipe[4] = 4;
 664                 break;
 665         case 6:
 666                 swizzle_pipe[0] = 0;
 667                 swizzle_pipe[1] = 2;
 668                 swizzle_pipe[2] = 4;
 669                 swizzle_pipe[3] = 5;
 670                 swizzle_pipe[4] = 1;
 671                 swizzle_pipe[5] = 3;
 672                 break;
 673         case 7:
 674                 swizzle_pipe[0] = 0;
 675                 swizzle_pipe[1] = 2;
 676                 swizzle_pipe[2] = 4;
 677                 swizzle_pipe[3] = 6;
 678                 swizzle_pipe[4] = 1;
 679                 swizzle_pipe[5] = 3;
 680                 swizzle_pipe[6] = 5;
 681                 break;
 682         case 8:
 683                 swizzle_pipe[0] = 0;
 684                 swizzle_pipe[1] = 2;
 685                 swizzle_pipe[2] = 4;
 686                 swizzle_pipe[3] = 6;
 687                 swizzle_pipe[4] = 1;
 688                 swizzle_pipe[5] = 3;
 689                 swizzle_pipe[6] = 5;
 690                 swizzle_pipe[7] = 7;
 691                 break;
 692         }
 693
 694         cur_backend = 0;
 695         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
 696                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
 697                         cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 698
 699                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
 700
 701                 cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
 702         }
 703
 704         return backend_map;
 705 }
 706
 707 static int r600_count_pipe_bits(uint32_t val)
 708 {
 709         return hweight32(val);
 710 }
 711
 712 static void r600_gfx_init(struct drm_device *dev,
 713                           drm_radeon_private_t *dev_priv)
 714 {
 715         int i, j, num_qd_pipes;
 716         u32 sx_debug_1;
 717         u32 tc_cntl;
 718         u32 arb_pop;
 719         u32 num_gs_verts_per_thread;
 720         u32 vgt_gs_per_es;
 721         u32 gs_prim_buffer_depth = 0;
 722         u32 sq_ms_fifo_sizes;
 723         u32 sq_config;
 724         u32 sq_gpr_resource_mgmt_1 = 0;
 725         u32 sq_gpr_resource_mgmt_2 = 0;
 726         u32 sq_thread_resource_mgmt = 0;
 727         u32 sq_stack_resource_mgmt_1 = 0;
 728         u32 sq_stack_resource_mgmt_2 = 0;
 729         u32 hdp_host_path_cntl;
 730         u32 backend_map;
 731         u32 gb_tiling_config = 0;
 732         u32 cc_rb_backend_disable;
 733         u32 cc_gc_shader_pipe_config;
 734         u32 ramcfg;
 735
 736         /* setup chip specs */
 737         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
 738         case CHIP_R600:
 739                 dev_priv->r600_max_pipes = 4;
 740                 dev_priv->r600_max_tile_pipes = 8;
 741                 dev_priv->r600_max_simds = 4;
 742                 dev_priv->r600_max_backends = 4;
 743                 dev_priv->r600_max_gprs = 256;
 744                 dev_priv->r600_max_threads = 192;
 745                 dev_priv->r600_max_stack_entries = 256;
 746                 dev_priv->r600_max_hw_contexts = 8;
 747                 dev_priv->r600_max_gs_threads = 16;
 748                 dev_priv->r600_sx_max_export_size = 128;
 749                 dev_priv->r600_sx_max_export_pos_size = 16;
 750                 dev_priv->r600_sx_max_export_smx_size = 128;
 751                 dev_priv->r600_sq_num_cf_insts = 2;
 752                 break;
 753         case CHIP_RV630:
 754         case CHIP_RV635:
 755                 dev_priv->r600_max_pipes = 2;
 756                 dev_priv->r600_max_tile_pipes = 2;
 757                 dev_priv->r600_max_simds = 3;
 758                 dev_priv->r600_max_backends = 1;
 759                 dev_priv->r600_max_gprs = 128;
 760                 dev_priv->r600_max_threads = 192;
 761                 dev_priv->r600_max_stack_entries = 128;
 762                 dev_priv->r600_max_hw_contexts = 8;
 763                 dev_priv->r600_max_gs_threads = 4;
 764                 dev_priv->r600_sx_max_export_size = 128;
 765                 dev_priv->r600_sx_max_export_pos_size = 16;
 766                 dev_priv->r600_sx_max_export_smx_size = 128;
 767                 dev_priv->r600_sq_num_cf_insts = 2;
 768                 break;
 769         case CHIP_RV610:
 770         case CHIP_RS780:
 771         case CHIP_RS880:
 772         case CHIP_RV620:
 773                 dev_priv->r600_max_pipes = 1;
 774                 dev_priv->r600_max_tile_pipes = 1;
 775                 dev_priv->r600_max_simds = 2;
 776                 dev_priv->r600_max_backends = 1;
 777                 dev_priv->r600_max_gprs = 128;
 778                 dev_priv->r600_max_threads = 192;
 779                 dev_priv->r600_max_stack_entries = 128;
 780                 dev_priv->r600_max_hw_contexts = 4;
 781                 dev_priv->r600_max_gs_threads = 4;
 782                 dev_priv->r600_sx_max_export_size = 128;
 783                 dev_priv->r600_sx_max_export_pos_size = 16;
 784                 dev_priv->r600_sx_max_export_smx_size = 128;
 785                 dev_priv->r600_sq_num_cf_insts = 1;
 786                 break;
 787         case CHIP_RV670:
 788                 dev_priv->r600_max_pipes = 4;
 789                 dev_priv->r600_max_tile_pipes = 4;
 790                 dev_priv->r600_max_simds = 4;
 791                 dev_priv->r600_max_backends = 4;
 792                 dev_priv->r600_max_gprs = 192;
 793                 dev_priv->r600_max_threads = 192;
 794                 dev_priv->r600_max_stack_entries = 256;
 795                 dev_priv->r600_max_hw_contexts = 8;
 796                 dev_priv->r600_max_gs_threads = 16;
 797                 dev_priv->r600_sx_max_export_size = 128;
 798                 dev_priv->r600_sx_max_export_pos_size = 16;
 799                 dev_priv->r600_sx_max_export_smx_size = 128;
 800                 dev_priv->r600_sq_num_cf_insts = 2;
 801                 break;
 802         default:
 803                 break;
 804         }
 805
 806         /* Initialize HDP */
 807         j = 0;
 808         for (i = 0; i < 32; i++) {
 809                 RADEON_WRITE((0x2c14 + j), 0x00000000);
 810                 RADEON_WRITE((0x2c18 + j), 0x00000000);
 811                 RADEON_WRITE((0x2c1c + j), 0x00000000);
 812                 RADEON_WRITE((0x2c20 + j), 0x00000000);
 813                 RADEON_WRITE((0x2c24 + j), 0x00000000);
 814                 j += 0x18;
 815         }
 816
 817         RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
 818
 819         /* setup tiling, simd, pipe config */
 820         ramcfg = RADEON_READ(R600_RAMCFG);
 821
 822         switch (dev_priv->r600_max_tile_pipes) {
 823         case 1:
 824                 gb_tiling_config |= R600_PIPE_TILING(0);
 825                 break;
 826         case 2:
 827                 gb_tiling_config |= R600_PIPE_TILING(1);
 828                 break;
 829         case 4:
 830                 gb_tiling_config |= R600_PIPE_TILING(2);
 831                 break;
 832         case 8:
 833                 gb_tiling_config |= R600_PIPE_TILING(3);
 834                 break;
 835         default:
 836                 break;
 837         }
 838
 839         gb_tiling_config |= R600_BANK_TILING((ramcfg >> R600_NOOFBANK_SHIFT) & R600_NOOFBANK_MASK);
 840
 841         gb_tiling_config |= R600_GROUP_SIZE(0);
 842
 843         if (((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK) > 3) {
 844                 gb_tiling_config |= R600_ROW_TILING(3);
 845                 gb_tiling_config |= R600_SAMPLE_SPLIT(3);
 846         } else {
 847                 gb_tiling_config |=
 848                         R600_ROW_TILING(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
 849                 gb_tiling_config |=
 850                         R600_SAMPLE_SPLIT(((ramcfg >> R600_NOOFROWS_SHIFT) & R600_NOOFROWS_MASK));
 851         }
 852
 853         gb_tiling_config |= R600_BANK_SWAPS(1);
 854
 855         cc_rb_backend_disable = RADEON_READ(R600_CC_RB_BACKEND_DISABLE) & 0x00ff0000;
 856         cc_rb_backend_disable |=
 857                 R600_BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R6XX_MAX_BACKENDS_MASK);
 858
 859         cc_gc_shader_pipe_config = RADEON_READ(R600_CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
 860         cc_gc_shader_pipe_config |=
 861                 R600_INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R6XX_MAX_PIPES_MASK);
 862         cc_gc_shader_pipe_config |=
 863                 R600_INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R6XX_MAX_SIMDS_MASK);
 864
 865         backend_map = r600_get_tile_pipe_to_backend_map(dev_priv->r600_max_tile_pipes,
 866                                                         (R6XX_MAX_BACKENDS -
 867                                                          r600_count_pipe_bits((cc_rb_backend_disable &
 868                                                                                R6XX_MAX_BACKENDS_MASK) >> 16)),
 869                                                         (cc_rb_backend_disable >> 16));
 870         gb_tiling_config |= R600_BACKEND_MAP(backend_map);
 871
 872         RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
 873         RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
 874         RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
 875         if (gb_tiling_config & 0xc0) {
 876                 dev_priv->r600_group_size = 512;
 877         } else {
 878                 dev_priv->r600_group_size = 256;
 879         }
 880         dev_priv->r600_npipes = 1 << ((gb_tiling_config >> 1) & 0x7);
 881         if (gb_tiling_config & 0x30) {
 882                 dev_priv->r600_nbanks = 8;
 883         } else {
 884                 dev_priv->r600_nbanks = 4;
 885         }
 886
 887         RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
 888         RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
 889         RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
 890
 891         num_qd_pipes =
 892                 R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK) >> 8);
 893         RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
 894         RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
 895
 896         /* set HW defaults for 3D engine */
 897         RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
 898                                                 R600_ROQ_IB2_START(0x2b)));
 899
 900         RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, (R600_MEQ_END(0x40) |
 901                                               R600_ROQ_END(0x40)));
 902
 903         RADEON_WRITE(R600_TA_CNTL_AUX, (R600_DISABLE_CUBE_ANISO |
 904                                         R600_SYNC_GRADIENT |
 905                                         R600_SYNC_WALKER |
 906                                         R600_SYNC_ALIGNER));
 907
 908         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670)
 909                 RADEON_WRITE(R600_ARB_GDEC_RD_CNTL, 0x00000021);
 910
 911         sx_debug_1 = RADEON_READ(R600_SX_DEBUG_1);
 912         sx_debug_1 |= R600_SMX_EVENT_RELEASE;
 913         if (((dev_priv->flags & RADEON_FAMILY_MASK) > CHIP_R600))
 914                 sx_debug_1 |= R600_ENABLE_NEW_SMX_ADDRESS;
 915         RADEON_WRITE(R600_SX_DEBUG_1, sx_debug_1);
 916
 917         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
 918             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
 919             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 920             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 921             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
 922             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880))
 923                 RADEON_WRITE(R600_DB_DEBUG, R600_PREZ_MUST_WAIT_FOR_POSTZ_DONE);
 924         else
 925                 RADEON_WRITE(R600_DB_DEBUG, 0);
 926
 927         RADEON_WRITE(R600_DB_WATERMARKS, (R600_DEPTH_FREE(4) |
 928                                           R600_DEPTH_FLUSH(16) |
 929                                           R600_DEPTH_PENDING_FREE(4) |
 930                                           R600_DEPTH_CACHELINE_FREE(16)));
 931         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
 932         RADEON_WRITE(R600_VGT_NUM_INSTANCES, 0);
 933
 934         RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
 935         RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(0));
 936
 937         sq_ms_fifo_sizes = RADEON_READ(R600_SQ_MS_FIFO_SIZES);
 938         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 939             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 940             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
 941             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880)) {
 942                 sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(0xa) |
 943                                     R600_FETCH_FIFO_HIWATER(0xa) |
 944                                     R600_DONE_FIFO_HIWATER(0xe0) |
 945                                     R600_ALU_UPDATE_FIFO_HIWATER(0x8));
 946         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) ||
 947                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630)) {
 948                 sq_ms_fifo_sizes &= ~R600_DONE_FIFO_HIWATER(0xff);
 949                 sq_ms_fifo_sizes |= R600_DONE_FIFO_HIWATER(0x4);
 950         }
 951         RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
 952
 953         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
 954          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
 955          */
 956         sq_config = RADEON_READ(R600_SQ_CONFIG);
 957         sq_config &= ~(R600_PS_PRIO(3) |
 958                        R600_VS_PRIO(3) |
 959                        R600_GS_PRIO(3) |
 960                        R600_ES_PRIO(3));
 961         sq_config |= (R600_DX9_CONSTS |
 962                       R600_VC_ENABLE |
 963                       R600_PS_PRIO(0) |
 964                       R600_VS_PRIO(1) |
 965                       R600_GS_PRIO(2) |
 966                       R600_ES_PRIO(3));
 967
 968         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600) {
 969                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(124) |
 970                                           R600_NUM_VS_GPRS(124) |
 971                                           R600_NUM_CLAUSE_TEMP_GPRS(4));
 972                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(0) |
 973                                           R600_NUM_ES_GPRS(0));
 974                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(136) |
 975                                            R600_NUM_VS_THREADS(48) |
 976                                            R600_NUM_GS_THREADS(4) |
 977                                            R600_NUM_ES_THREADS(4));
 978                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(128) |
 979                                             R600_NUM_VS_STACK_ENTRIES(128));
 980                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(0) |
 981                                             R600_NUM_ES_STACK_ENTRIES(0));
 982         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
 983                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
 984                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
 985                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880)) {
 986                 /* no vertex cache */
 987                 sq_config &= ~R600_VC_ENABLE;
 988
 989                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
 990                                           R600_NUM_VS_GPRS(44) |
 991                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
 992                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
 993                                           R600_NUM_ES_GPRS(17));
 994                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
 995                                            R600_NUM_VS_THREADS(78) |
 996                                            R600_NUM_GS_THREADS(4) |
 997                                            R600_NUM_ES_THREADS(31));
 998                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
 999                                             R600_NUM_VS_STACK_ENTRIES(40));
1000                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
1001                                             R600_NUM_ES_STACK_ENTRIES(16));
1002         } else if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV630) ||
1003                    ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV635)) {
1004                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
1005                                           R600_NUM_VS_GPRS(44) |
1006                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
1007                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(18) |
1008                                           R600_NUM_ES_GPRS(18));
1009                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
1010                                            R600_NUM_VS_THREADS(78) |
1011                                            R600_NUM_GS_THREADS(4) |
1012                                            R600_NUM_ES_THREADS(31));
1013                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(40) |
1014                                             R600_NUM_VS_STACK_ENTRIES(40));
1015                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(32) |
1016                                             R600_NUM_ES_STACK_ENTRIES(16));
1017         } else if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV670) {
1018                 sq_gpr_resource_mgmt_1 = (R600_NUM_PS_GPRS(44) |
1019                                           R600_NUM_VS_GPRS(44) |
1020                                           R600_NUM_CLAUSE_TEMP_GPRS(2));
1021                 sq_gpr_resource_mgmt_2 = (R600_NUM_GS_GPRS(17) |
1022                                           R600_NUM_ES_GPRS(17));
1023                 sq_thread_resource_mgmt = (R600_NUM_PS_THREADS(79) |
1024                                            R600_NUM_VS_THREADS(78) |
1025                                            R600_NUM_GS_THREADS(4) |
1026                                            R600_NUM_ES_THREADS(31));
1027                 sq_stack_resource_mgmt_1 = (R600_NUM_PS_STACK_ENTRIES(64) |
1028                                             R600_NUM_VS_STACK_ENTRIES(64));
1029                 sq_stack_resource_mgmt_2 = (R600_NUM_GS_STACK_ENTRIES(64) |
1030                                             R600_NUM_ES_STACK_ENTRIES(64));
1031         }
1032
1033         RADEON_WRITE(R600_SQ_CONFIG, sq_config);
1034         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1035         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1036         RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1037         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1038         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1039
1040         if (((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV610) ||
1041             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV620) ||
1042             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS780) ||
1043             ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RS880))
1044                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_TC_ONLY));
1045         else
1046                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, R600_CACHE_INVALIDATION(R600_VC_AND_TC));
1047
1048         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_2S, (R600_S0_X(0xc) |
1049                                                     R600_S0_Y(0x4) |
1050                                                     R600_S1_X(0x4) |
1051                                                     R600_S1_Y(0xc)));
1052         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_4S, (R600_S0_X(0xe) |
1053                                                     R600_S0_Y(0xe) |
1054                                                     R600_S1_X(0x2) |
1055                                                     R600_S1_Y(0x2) |
1056                                                     R600_S2_X(0xa) |
1057                                                     R600_S2_Y(0x6) |
1058                                                     R600_S3_X(0x6) |
1059                                                     R600_S3_Y(0xa)));
1060         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD0, (R600_S0_X(0xe) |
1061                                                         R600_S0_Y(0xb) |
1062                                                         R600_S1_X(0x4) |
1063                                                         R600_S1_Y(0xc) |
1064                                                         R600_S2_X(0x1) |
1065                                                         R600_S2_Y(0x6) |
1066                                                         R600_S3_X(0xa) |
1067                                                         R600_S3_Y(0xe)));
1068         RADEON_WRITE(R600_PA_SC_AA_SAMPLE_LOCS_8S_WD1, (R600_S4_X(0x6) |
1069                                                         R600_S4_Y(0x1) |
1070                                                         R600_S5_X(0x0) |
1071                                                         R600_S5_Y(0x0) |
1072                                                         R600_S6_X(0xb) |
1073                                                         R600_S6_Y(0x4) |
1074                                                         R600_S7_X(0x7) |
1075                                                         R600_S7_Y(0x8)));
1076
1077
1078         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1079         case CHIP_R600:
1080         case CHIP_RV630:
1081         case CHIP_RV635:
1082                 gs_prim_buffer_depth = 0;
1083                 break;
1084         case CHIP_RV610:
1085         case CHIP_RS780:
1086         case CHIP_RS880:
1087         case CHIP_RV620:
1088                 gs_prim_buffer_depth = 32;
1089                 break;
1090         case CHIP_RV670:
1091                 gs_prim_buffer_depth = 128;
1092                 break;
1093         default:
1094                 break;
1095         }
1096
1097         num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
1098         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1099         /* Max value for this is 256 */
1100         if (vgt_gs_per_es > 256)
1101                 vgt_gs_per_es = 256;
1102
1103         RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
1104         RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
1105         RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
1106         RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
1107
1108         /* more default values. 2D/3D driver should adjust as needed */
1109         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
1110         RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
1111         RADEON_WRITE(R600_SX_MISC, 0);
1112         RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
1113         RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
1114         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
1115         RADEON_WRITE(R600_SPI_INPUT_Z, 0);
1116         RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
1117         RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
1118
1119         /* clear render buffer base addresses */
1120         RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
1121         RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
1122         RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
1123         RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
1124         RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
1125         RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
1126         RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
1127         RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
1128
1129         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1130         case CHIP_RV610:
1131         case CHIP_RS780:
1132         case CHIP_RS880:
1133         case CHIP_RV620:
1134                 tc_cntl = R600_TC_L2_SIZE(8);
1135                 break;
1136         case CHIP_RV630:
1137         case CHIP_RV635:
1138                 tc_cntl = R600_TC_L2_SIZE(4);
1139                 break;
1140         case CHIP_R600:
1141                 tc_cntl = R600_TC_L2_SIZE(0) | R600_L2_DISABLE_LATE_HIT;
1142                 break;
1143         default:
1144                 tc_cntl = R600_TC_L2_SIZE(0);
1145                 break;
1146         }
1147
1148         RADEON_WRITE(R600_TC_CNTL, tc_cntl);
1149
1150         hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
1151         RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1152
1153         arb_pop = RADEON_READ(R600_ARB_POP);
1154         arb_pop |= R600_ENABLE_TC128;
1155         RADEON_WRITE(R600_ARB_POP, arb_pop);
1156
1157         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1158         RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
1159                                           R600_NUM_CLIP_SEQ(3)));
1160         RADEON_WRITE(R600_PA_SC_ENHANCE, R600_FORCE_EOV_MAX_CLK_CNT(4095));
1161
1162 }
1163
1164 static u32 r700_get_tile_pipe_to_backend_map(drm_radeon_private_t *dev_priv,
1165                                              u32 num_tile_pipes,
1166                                              u32 num_backends,
1167                                              u32 backend_disable_mask)
1168 {
1169         u32 backend_map = 0;
1170         u32 enabled_backends_mask;
1171         u32 enabled_backends_count;
1172         u32 cur_pipe;
1173         u32 swizzle_pipe[R7XX_MAX_PIPES];
1174         u32 cur_backend;
1175         u32 i;
1176         bool force_no_swizzle;
1177
1178         if (num_tile_pipes > R7XX_MAX_PIPES)
1179                 num_tile_pipes = R7XX_MAX_PIPES;
1180         if (num_tile_pipes < 1)
1181                 num_tile_pipes = 1;
1182         if (num_backends > R7XX_MAX_BACKENDS)
1183                 num_backends = R7XX_MAX_BACKENDS;
1184         if (num_backends < 1)
1185                 num_backends = 1;
1186
1187         enabled_backends_mask = 0;
1188         enabled_backends_count = 0;
1189         for (i = 0; i < R7XX_MAX_BACKENDS; ++i) {
1190                 if (((backend_disable_mask >> i) & 1) == 0) {
1191                         enabled_backends_mask |= (1 << i);
1192                         ++enabled_backends_count;
1193                 }
1194                 if (enabled_backends_count == num_backends)
1195                         break;
1196         }
1197
1198         if (enabled_backends_count == 0) {
1199                 enabled_backends_mask = 1;
1200                 enabled_backends_count = 1;
1201         }
1202
1203         if (enabled_backends_count != num_backends)
1204                 num_backends = enabled_backends_count;
1205
1206         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1207         case CHIP_RV770:
1208         case CHIP_RV730:
1209                 force_no_swizzle = false;
1210                 break;
1211         case CHIP_RV710:
1212         case CHIP_RV740:
1213         default:
1214                 force_no_swizzle = true;
1215                 break;
1216         }
1217
1218         memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * R7XX_MAX_PIPES);
1219         switch (num_tile_pipes) {
1220         case 1:
1221                 swizzle_pipe[0] = 0;
1222                 break;
1223         case 2:
1224                 swizzle_pipe[0] = 0;
1225                 swizzle_pipe[1] = 1;
1226                 break;
1227         case 3:
1228                 if (force_no_swizzle) {
1229                         swizzle_pipe[0] = 0;
1230                         swizzle_pipe[1] = 1;
1231                         swizzle_pipe[2] = 2;
1232                 } else {
1233                         swizzle_pipe[0] = 0;
1234                         swizzle_pipe[1] = 2;
1235                         swizzle_pipe[2] = 1;
1236                 }
1237                 break;
1238         case 4:
1239                 if (force_no_swizzle) {
1240                         swizzle_pipe[0] = 0;
1241                         swizzle_pipe[1] = 1;
1242                         swizzle_pipe[2] = 2;
1243                         swizzle_pipe[3] = 3;
1244                 } else {
1245                         swizzle_pipe[0] = 0;
1246                         swizzle_pipe[1] = 2;
1247                         swizzle_pipe[2] = 3;
1248                         swizzle_pipe[3] = 1;
1249                 }
1250                 break;
1251         case 5:
1252                 if (force_no_swizzle) {
1253                         swizzle_pipe[0] = 0;
1254                         swizzle_pipe[1] = 1;
1255                         swizzle_pipe[2] = 2;
1256                         swizzle_pipe[3] = 3;
1257                         swizzle_pipe[4] = 4;
1258                 } else {
1259                         swizzle_pipe[0] = 0;
1260                         swizzle_pipe[1] = 2;
1261                         swizzle_pipe[2] = 4;
1262                         swizzle_pipe[3] = 1;
1263                         swizzle_pipe[4] = 3;
1264                 }
1265                 break;
1266         case 6:
1267                 if (force_no_swizzle) {
1268                         swizzle_pipe[0] = 0;
1269                         swizzle_pipe[1] = 1;
1270                         swizzle_pipe[2] = 2;
1271                         swizzle_pipe[3] = 3;
1272                         swizzle_pipe[4] = 4;
1273                         swizzle_pipe[5] = 5;
1274                 } else {
1275                         swizzle_pipe[0] = 0;
1276                         swizzle_pipe[1] = 2;
1277                         swizzle_pipe[2] = 4;
1278                         swizzle_pipe[3] = 5;
1279                         swizzle_pipe[4] = 3;
1280                         swizzle_pipe[5] = 1;
1281                 }
1282                 break;
1283         case 7:
1284                 if (force_no_swizzle) {
1285                         swizzle_pipe[0] = 0;
1286                         swizzle_pipe[1] = 1;
1287                         swizzle_pipe[2] = 2;
1288                         swizzle_pipe[3] = 3;
1289                         swizzle_pipe[4] = 4;
1290                         swizzle_pipe[5] = 5;
1291                         swizzle_pipe[6] = 6;
1292                 } else {
1293                         swizzle_pipe[0] = 0;
1294                         swizzle_pipe[1] = 2;
1295                         swizzle_pipe[2] = 4;
1296                         swizzle_pipe[3] = 6;
1297                         swizzle_pipe[4] = 3;
1298                         swizzle_pipe[5] = 1;
1299                         swizzle_pipe[6] = 5;
1300                 }
1301                 break;
1302         case 8:
1303                 if (force_no_swizzle) {
1304                         swizzle_pipe[0] = 0;
1305                         swizzle_pipe[1] = 1;
1306                         swizzle_pipe[2] = 2;
1307                         swizzle_pipe[3] = 3;
1308                         swizzle_pipe[4] = 4;
1309                         swizzle_pipe[5] = 5;
1310                         swizzle_pipe[6] = 6;
1311                         swizzle_pipe[7] = 7;
1312                 } else {
1313                         swizzle_pipe[0] = 0;
1314                         swizzle_pipe[1] = 2;
1315                         swizzle_pipe[2] = 4;
1316                         swizzle_pipe[3] = 6;
1317                         swizzle_pipe[4] = 3;
1318                         swizzle_pipe[5] = 1;
1319                         swizzle_pipe[6] = 7;
1320                         swizzle_pipe[7] = 5;
1321                 }
1322                 break;
1323         }
1324
1325         cur_backend = 0;
1326         for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1327                 while (((1 << cur_backend) & enabled_backends_mask) == 0)
1328                         cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
1329
1330                 backend_map |= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
1331
1332                 cur_backend = (cur_backend + 1) % R7XX_MAX_BACKENDS;
1333         }
1334
1335         return backend_map;
1336 }
1337
1338 static void r700_gfx_init(struct drm_device *dev,
1339                           drm_radeon_private_t *dev_priv)
1340 {
1341         int i, j, num_qd_pipes;
1342         u32 ta_aux_cntl;
1343         u32 sx_debug_1;
1344         u32 smx_dc_ctl0;
1345         u32 db_debug3;
1346         u32 num_gs_verts_per_thread;
1347         u32 vgt_gs_per_es;
1348         u32 gs_prim_buffer_depth = 0;
1349         u32 sq_ms_fifo_sizes;
1350         u32 sq_config;
1351         u32 sq_thread_resource_mgmt;
1352         u32 hdp_host_path_cntl;
1353         u32 sq_dyn_gpr_size_simd_ab_0;
1354         u32 backend_map;
1355         u32 gb_tiling_config = 0;
1356         u32 cc_rb_backend_disable;
1357         u32 cc_gc_shader_pipe_config;
1358         u32 mc_arb_ramcfg;
1359         u32 db_debug4;
1360
1361         /* setup chip specs */
1362         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1363         case CHIP_RV770:
1364                 dev_priv->r600_max_pipes = 4;
1365                 dev_priv->r600_max_tile_pipes = 8;
1366                 dev_priv->r600_max_simds = 10;
1367                 dev_priv->r600_max_backends = 4;
1368                 dev_priv->r600_max_gprs = 256;
1369                 dev_priv->r600_max_threads = 248;
1370                 dev_priv->r600_max_stack_entries = 512;
1371                 dev_priv->r600_max_hw_contexts = 8;
1372                 dev_priv->r600_max_gs_threads = 16 * 2;
1373                 dev_priv->r600_sx_max_export_size = 128;
1374                 dev_priv->r600_sx_max_export_pos_size = 16;
1375                 dev_priv->r600_sx_max_export_smx_size = 112;
1376                 dev_priv->r600_sq_num_cf_insts = 2;
1377
1378                 dev_priv->r700_sx_num_of_sets = 7;
1379                 dev_priv->r700_sc_prim_fifo_size = 0xF9;
1380                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1381                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1382                 break;
1383         case CHIP_RV730:
1384                 dev_priv->r600_max_pipes = 2;
1385                 dev_priv->r600_max_tile_pipes = 4;
1386                 dev_priv->r600_max_simds = 8;
1387                 dev_priv->r600_max_backends = 2;
1388                 dev_priv->r600_max_gprs = 128;
1389                 dev_priv->r600_max_threads = 248;
1390                 dev_priv->r600_max_stack_entries = 256;
1391                 dev_priv->r600_max_hw_contexts = 8;
1392                 dev_priv->r600_max_gs_threads = 16 * 2;
1393                 dev_priv->r600_sx_max_export_size = 256;
1394                 dev_priv->r600_sx_max_export_pos_size = 32;
1395                 dev_priv->r600_sx_max_export_smx_size = 224;
1396                 dev_priv->r600_sq_num_cf_insts = 2;
1397
1398                 dev_priv->r700_sx_num_of_sets = 7;
1399                 dev_priv->r700_sc_prim_fifo_size = 0xf9;
1400                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1401                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1402                 if (dev_priv->r600_sx_max_export_pos_size > 16) {
1403                         dev_priv->r600_sx_max_export_pos_size -= 16;
1404                         dev_priv->r600_sx_max_export_smx_size += 16;
1405                 }
1406                 break;
1407         case CHIP_RV710:
1408                 dev_priv->r600_max_pipes = 2;
1409                 dev_priv->r600_max_tile_pipes = 2;
1410                 dev_priv->r600_max_simds = 2;
1411                 dev_priv->r600_max_backends = 1;
1412                 dev_priv->r600_max_gprs = 256;
1413                 dev_priv->r600_max_threads = 192;
1414                 dev_priv->r600_max_stack_entries = 256;
1415                 dev_priv->r600_max_hw_contexts = 4;
1416                 dev_priv->r600_max_gs_threads = 8 * 2;
1417                 dev_priv->r600_sx_max_export_size = 128;
1418                 dev_priv->r600_sx_max_export_pos_size = 16;
1419                 dev_priv->r600_sx_max_export_smx_size = 112;
1420                 dev_priv->r600_sq_num_cf_insts = 1;
1421
1422                 dev_priv->r700_sx_num_of_sets = 7;
1423                 dev_priv->r700_sc_prim_fifo_size = 0x40;
1424                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1425                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1426                 break;
1427         case CHIP_RV740:
1428                 dev_priv->r600_max_pipes = 4;
1429                 dev_priv->r600_max_tile_pipes = 4;
1430                 dev_priv->r600_max_simds = 8;
1431                 dev_priv->r600_max_backends = 4;
1432                 dev_priv->r600_max_gprs = 256;
1433                 dev_priv->r600_max_threads = 248;
1434                 dev_priv->r600_max_stack_entries = 512;
1435                 dev_priv->r600_max_hw_contexts = 8;
1436                 dev_priv->r600_max_gs_threads = 16 * 2;
1437                 dev_priv->r600_sx_max_export_size = 256;
1438                 dev_priv->r600_sx_max_export_pos_size = 32;
1439                 dev_priv->r600_sx_max_export_smx_size = 224;
1440                 dev_priv->r600_sq_num_cf_insts = 2;
1441
1442                 dev_priv->r700_sx_num_of_sets = 7;
1443                 dev_priv->r700_sc_prim_fifo_size = 0x100;
1444                 dev_priv->r700_sc_hiz_tile_fifo_size = 0x30;
1445                 dev_priv->r700_sc_earlyz_tile_fifo_fize = 0x130;
1446
1447                 if (dev_priv->r600_sx_max_export_pos_size > 16) {
1448                         dev_priv->r600_sx_max_export_pos_size -= 16;
1449                         dev_priv->r600_sx_max_export_smx_size += 16;
1450                 }
1451                 break;
1452         default:
1453                 break;
1454         }
1455
1456         /* Initialize HDP */
1457         j = 0;
1458         for (i = 0; i < 32; i++) {
1459                 RADEON_WRITE((0x2c14 + j), 0x00000000);
1460                 RADEON_WRITE((0x2c18 + j), 0x00000000);
1461                 RADEON_WRITE((0x2c1c + j), 0x00000000);
1462                 RADEON_WRITE((0x2c20 + j), 0x00000000);
1463                 RADEON_WRITE((0x2c24 + j), 0x00000000);
1464                 j += 0x18;
1465         }
1466
1467         RADEON_WRITE(R600_GRBM_CNTL, R600_GRBM_READ_TIMEOUT(0xff));
1468
1469         /* setup tiling, simd, pipe config */
1470         mc_arb_ramcfg = RADEON_READ(R700_MC_ARB_RAMCFG);
1471
1472         switch (dev_priv->r600_max_tile_pipes) {
1473         case 1:
1474                 gb_tiling_config |= R600_PIPE_TILING(0);
1475                 break;
1476         case 2:
1477                 gb_tiling_config |= R600_PIPE_TILING(1);
1478                 break;
1479         case 4:
1480                 gb_tiling_config |= R600_PIPE_TILING(2);
1481                 break;
1482         case 8:
1483                 gb_tiling_config |= R600_PIPE_TILING(3);
1484                 break;
1485         default:
1486                 break;
1487         }
1488
1489         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV770)
1490                 gb_tiling_config |= R600_BANK_TILING(1);
1491         else
1492                 gb_tiling_config |= R600_BANK_TILING((mc_arb_ramcfg >> R700_NOOFBANK_SHIFT) & R700_NOOFBANK_MASK);
1493
1494         gb_tiling_config |= R600_GROUP_SIZE(0);
1495
1496         if (((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK) > 3) {
1497                 gb_tiling_config |= R600_ROW_TILING(3);
1498                 gb_tiling_config |= R600_SAMPLE_SPLIT(3);
1499         } else {
1500                 gb_tiling_config |=
1501                         R600_ROW_TILING(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
1502                 gb_tiling_config |=
1503                         R600_SAMPLE_SPLIT(((mc_arb_ramcfg >> R700_NOOFROWS_SHIFT) & R700_NOOFROWS_MASK));
1504         }
1505
1506         gb_tiling_config |= R600_BANK_SWAPS(1);
1507
1508         cc_rb_backend_disable = RADEON_READ(R600_CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1509         cc_rb_backend_disable |=
1510                 R600_BACKEND_DISABLE((R7XX_MAX_BACKENDS_MASK << dev_priv->r600_max_backends) & R7XX_MAX_BACKENDS_MASK);
1511
1512         cc_gc_shader_pipe_config = RADEON_READ(R600_CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
1513         cc_gc_shader_pipe_config |=
1514                 R600_INACTIVE_QD_PIPES((R7XX_MAX_PIPES_MASK << dev_priv->r600_max_pipes) & R7XX_MAX_PIPES_MASK);
1515         cc_gc_shader_pipe_config |=
1516                 R600_INACTIVE_SIMDS((R7XX_MAX_SIMDS_MASK << dev_priv->r600_max_simds) & R7XX_MAX_SIMDS_MASK);
1517
1518         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV740)
1519                 backend_map = 0x28;
1520         else
1521                 backend_map = r700_get_tile_pipe_to_backend_map(dev_priv,
1522                                                                 dev_priv->r600_max_tile_pipes,
1523                                                                 (R7XX_MAX_BACKENDS -
1524                                                                  r600_count_pipe_bits((cc_rb_backend_disable &
1525                                                                                        R7XX_MAX_BACKENDS_MASK) >> 16)),
1526                                                                 (cc_rb_backend_disable >> 16));
1527         gb_tiling_config |= R600_BACKEND_MAP(backend_map);
1528
1529         RADEON_WRITE(R600_GB_TILING_CONFIG,      gb_tiling_config);
1530         RADEON_WRITE(R600_DCP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
1531         RADEON_WRITE(R600_HDP_TILING_CONFIG,    (gb_tiling_config & 0xffff));
1532         if (gb_tiling_config & 0xc0) {
1533                 dev_priv->r600_group_size = 512;
1534         } else {
1535                 dev_priv->r600_group_size = 256;
1536         }
1537         dev_priv->r600_npipes = 1 << ((gb_tiling_config >> 1) & 0x7);
1538         if (gb_tiling_config & 0x30) {
1539                 dev_priv->r600_nbanks = 8;
1540         } else {
1541                 dev_priv->r600_nbanks = 4;
1542         }
1543
1544         RADEON_WRITE(R600_CC_RB_BACKEND_DISABLE,      cc_rb_backend_disable);
1545         RADEON_WRITE(R600_CC_GC_SHADER_PIPE_CONFIG,   cc_gc_shader_pipe_config);
1546         RADEON_WRITE(R600_GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1547
1548         RADEON_WRITE(R700_CC_SYS_RB_BACKEND_DISABLE, cc_rb_backend_disable);
1549         RADEON_WRITE(R700_CGTS_SYS_TCC_DISABLE, 0);
1550         RADEON_WRITE(R700_CGTS_TCC_DISABLE, 0);
1551         RADEON_WRITE(R700_CGTS_USER_SYS_TCC_DISABLE, 0);
1552         RADEON_WRITE(R700_CGTS_USER_TCC_DISABLE, 0);
1553
1554         num_qd_pipes =
1555                 R7XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & R600_INACTIVE_QD_PIPES_MASK) >> 8);
1556         RADEON_WRITE(R600_VGT_OUT_DEALLOC_CNTL, (num_qd_pipes * 4) & R600_DEALLOC_DIST_MASK);
1557         RADEON_WRITE(R600_VGT_VERTEX_REUSE_BLOCK_CNTL, ((num_qd_pipes * 4) - 2) & R600_VTX_REUSE_DEPTH_MASK);
1558
1559         /* set HW defaults for 3D engine */
1560         RADEON_WRITE(R600_CP_QUEUE_THRESHOLDS, (R600_ROQ_IB1_START(0x16) |
1561                                                 R600_ROQ_IB2_START(0x2b)));
1562
1563         RADEON_WRITE(R600_CP_MEQ_THRESHOLDS, R700_STQ_SPLIT(0x30));
1564
1565         ta_aux_cntl = RADEON_READ(R600_TA_CNTL_AUX);
1566         RADEON_WRITE(R600_TA_CNTL_AUX, ta_aux_cntl | R600_DISABLE_CUBE_ANISO);
1567
1568         sx_debug_1 = RADEON_READ(R700_SX_DEBUG_1);
1569         sx_debug_1 |= R700_ENABLE_NEW_SMX_ADDRESS;
1570         RADEON_WRITE(R700_SX_DEBUG_1, sx_debug_1);
1571
1572         smx_dc_ctl0 = RADEON_READ(R600_SMX_DC_CTL0);
1573         smx_dc_ctl0 &= ~R700_CACHE_DEPTH(0x1ff);
1574         smx_dc_ctl0 |= R700_CACHE_DEPTH((dev_priv->r700_sx_num_of_sets * 64) - 1);
1575         RADEON_WRITE(R600_SMX_DC_CTL0, smx_dc_ctl0);
1576
1577         if ((dev_priv->flags & RADEON_FAMILY_MASK) != CHIP_RV740)
1578                 RADEON_WRITE(R700_SMX_EVENT_CTL, (R700_ES_FLUSH_CTL(4) |
1579                                                   R700_GS_FLUSH_CTL(4) |
1580                                                   R700_ACK_FLUSH_CTL(3) |
1581                                                   R700_SYNC_FLUSH_CTL));
1582
1583         db_debug3 = RADEON_READ(R700_DB_DEBUG3);
1584         db_debug3 &= ~R700_DB_CLK_OFF_DELAY(0x1f);
1585         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1586         case CHIP_RV770:
1587         case CHIP_RV740:
1588                 db_debug3 |= R700_DB_CLK_OFF_DELAY(0x1f);
1589                 break;
1590         case CHIP_RV710:
1591         case CHIP_RV730:
1592         default:
1593                 db_debug3 |= R700_DB_CLK_OFF_DELAY(2);
1594                 break;
1595         }
1596         RADEON_WRITE(R700_DB_DEBUG3, db_debug3);
1597
1598         if ((dev_priv->flags & RADEON_FAMILY_MASK) != CHIP_RV770) {
1599                 db_debug4 = RADEON_READ(RV700_DB_DEBUG4);
1600                 db_debug4 |= RV700_DISABLE_TILE_COVERED_FOR_PS_ITER;
1601                 RADEON_WRITE(RV700_DB_DEBUG4, db_debug4);
1602         }
1603
1604         RADEON_WRITE(R600_SX_EXPORT_BUFFER_SIZES, (R600_COLOR_BUFFER_SIZE((dev_priv->r600_sx_max_export_size / 4) - 1) |
1605                                                    R600_POSITION_BUFFER_SIZE((dev_priv->r600_sx_max_export_pos_size / 4) - 1) |
1606                                                    R600_SMX_BUFFER_SIZE((dev_priv->r600_sx_max_export_smx_size / 4) - 1)));
1607
1608         RADEON_WRITE(R700_PA_SC_FIFO_SIZE_R7XX, (R700_SC_PRIM_FIFO_SIZE(dev_priv->r700_sc_prim_fifo_size) |
1609                                                  R700_SC_HIZ_TILE_FIFO_SIZE(dev_priv->r700_sc_hiz_tile_fifo_size) |
1610                                                  R700_SC_EARLYZ_TILE_FIFO_SIZE(dev_priv->r700_sc_earlyz_tile_fifo_fize)));
1611
1612         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1613
1614         RADEON_WRITE(R600_VGT_NUM_INSTANCES, 1);
1615
1616         RADEON_WRITE(R600_SPI_CONFIG_CNTL, R600_GPR_WRITE_PRIORITY(0));
1617
1618         RADEON_WRITE(R600_SPI_CONFIG_CNTL_1, R600_VTX_DONE_DELAY(4));
1619
1620         RADEON_WRITE(R600_CP_PERFMON_CNTL, 0);
1621
1622         sq_ms_fifo_sizes = (R600_CACHE_FIFO_SIZE(16 * dev_priv->r600_sq_num_cf_insts) |
1623                             R600_DONE_FIFO_HIWATER(0xe0) |
1624                             R600_ALU_UPDATE_FIFO_HIWATER(0x8));
1625         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1626         case CHIP_RV770:
1627         case CHIP_RV730:
1628         case CHIP_RV710:
1629                 sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x1);
1630                 break;
1631         case CHIP_RV740:
1632         default:
1633                 sq_ms_fifo_sizes |= R600_FETCH_FIFO_HIWATER(0x4);
1634                 break;
1635         }
1636         RADEON_WRITE(R600_SQ_MS_FIFO_SIZES, sq_ms_fifo_sizes);
1637
1638         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1639          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1640          */
1641         sq_config = RADEON_READ(R600_SQ_CONFIG);
1642         sq_config &= ~(R600_PS_PRIO(3) |
1643                        R600_VS_PRIO(3) |
1644                        R600_GS_PRIO(3) |
1645                        R600_ES_PRIO(3));
1646         sq_config |= (R600_DX9_CONSTS |
1647                       R600_VC_ENABLE |
1648                       R600_EXPORT_SRC_C |
1649                       R600_PS_PRIO(0) |
1650                       R600_VS_PRIO(1) |
1651                       R600_GS_PRIO(2) |
1652                       R600_ES_PRIO(3));
1653         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
1654                 /* no vertex cache */
1655                 sq_config &= ~R600_VC_ENABLE;
1656
1657         RADEON_WRITE(R600_SQ_CONFIG, sq_config);
1658
1659         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_1,  (R600_NUM_PS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
1660                                                     R600_NUM_VS_GPRS((dev_priv->r600_max_gprs * 24)/64) |
1661                                                     R600_NUM_CLAUSE_TEMP_GPRS(((dev_priv->r600_max_gprs * 24)/64)/2)));
1662
1663         RADEON_WRITE(R600_SQ_GPR_RESOURCE_MGMT_2,  (R600_NUM_GS_GPRS((dev_priv->r600_max_gprs * 7)/64) |
1664                                                     R600_NUM_ES_GPRS((dev_priv->r600_max_gprs * 7)/64)));
1665
1666         sq_thread_resource_mgmt = (R600_NUM_PS_THREADS((dev_priv->r600_max_threads * 4)/8) |
1667                                    R600_NUM_VS_THREADS((dev_priv->r600_max_threads * 2)/8) |
1668                                    R600_NUM_ES_THREADS((dev_priv->r600_max_threads * 1)/8));
1669         if (((dev_priv->r600_max_threads * 1) / 8) > dev_priv->r600_max_gs_threads)
1670                 sq_thread_resource_mgmt |= R600_NUM_GS_THREADS(dev_priv->r600_max_gs_threads);
1671         else
1672                 sq_thread_resource_mgmt |= R600_NUM_GS_THREADS((dev_priv->r600_max_gs_threads * 1)/8);
1673         RADEON_WRITE(R600_SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1674
1675         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_1, (R600_NUM_PS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
1676                                                      R600_NUM_VS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
1677
1678         RADEON_WRITE(R600_SQ_STACK_RESOURCE_MGMT_2, (R600_NUM_GS_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4) |
1679                                                      R600_NUM_ES_STACK_ENTRIES((dev_priv->r600_max_stack_entries * 1)/4)));
1680
1681         sq_dyn_gpr_size_simd_ab_0 = (R700_SIMDA_RING0((dev_priv->r600_max_gprs * 38)/64) |
1682                                      R700_SIMDA_RING1((dev_priv->r600_max_gprs * 38)/64) |
1683                                      R700_SIMDB_RING0((dev_priv->r600_max_gprs * 38)/64) |
1684                                      R700_SIMDB_RING1((dev_priv->r600_max_gprs * 38)/64));
1685
1686         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_0, sq_dyn_gpr_size_simd_ab_0);
1687         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_1, sq_dyn_gpr_size_simd_ab_0);
1688         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_2, sq_dyn_gpr_size_simd_ab_0);
1689         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_3, sq_dyn_gpr_size_simd_ab_0);
1690         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_4, sq_dyn_gpr_size_simd_ab_0);
1691         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_5, sq_dyn_gpr_size_simd_ab_0);
1692         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_6, sq_dyn_gpr_size_simd_ab_0);
1693         RADEON_WRITE(R700_SQ_DYN_GPR_SIZE_SIMD_AB_7, sq_dyn_gpr_size_simd_ab_0);
1694
1695         RADEON_WRITE(R700_PA_SC_FORCE_EOV_MAX_CNTS, (R700_FORCE_EOV_MAX_CLK_CNT(4095) |
1696                                                      R700_FORCE_EOV_MAX_REZ_CNT(255)));
1697
1698         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV710)
1699                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_TC_ONLY) |
1700                                                            R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
1701         else
1702                 RADEON_WRITE(R600_VGT_CACHE_INVALIDATION, (R600_CACHE_INVALIDATION(R600_VC_AND_TC) |
1703                                                            R700_AUTO_INVLD_EN(R700_ES_AND_GS_AUTO)));
1704
1705         switch (dev_priv->flags & RADEON_FAMILY_MASK) {
1706         case CHIP_RV770:
1707         case CHIP_RV730:
1708         case CHIP_RV740:
1709                 gs_prim_buffer_depth = 384;
1710                 break;
1711         case CHIP_RV710:
1712                 gs_prim_buffer_depth = 128;
1713                 break;
1714         default:
1715                 break;
1716         }
1717
1718         num_gs_verts_per_thread = dev_priv->r600_max_pipes * 16;
1719         vgt_gs_per_es = gs_prim_buffer_depth + num_gs_verts_per_thread;
1720         /* Max value for this is 256 */
1721         if (vgt_gs_per_es > 256)
1722                 vgt_gs_per_es = 256;
1723
1724         RADEON_WRITE(R600_VGT_ES_PER_GS, 128);
1725         RADEON_WRITE(R600_VGT_GS_PER_ES, vgt_gs_per_es);
1726         RADEON_WRITE(R600_VGT_GS_PER_VS, 2);
1727
1728         /* more default values. 2D/3D driver should adjust as needed */
1729         RADEON_WRITE(R600_VGT_GS_VERTEX_REUSE, 16);
1730         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE_STATE, 0);
1731         RADEON_WRITE(R600_VGT_STRMOUT_EN, 0);
1732         RADEON_WRITE(R600_SX_MISC, 0);
1733         RADEON_WRITE(R600_PA_SC_MODE_CNTL, 0);
1734         RADEON_WRITE(R700_PA_SC_EDGERULE, 0xaaaaaaaa);
1735         RADEON_WRITE(R600_PA_SC_AA_CONFIG, 0);
1736         RADEON_WRITE(R600_PA_SC_CLIPRECT_RULE, 0xffff);
1737         RADEON_WRITE(R600_PA_SC_LINE_STIPPLE, 0);
1738         RADEON_WRITE(R600_SPI_INPUT_Z, 0);
1739         RADEON_WRITE(R600_SPI_PS_IN_CONTROL_0, R600_NUM_INTERP(2));
1740         RADEON_WRITE(R600_CB_COLOR7_FRAG, 0);
1741
1742         /* clear render buffer base addresses */
1743         RADEON_WRITE(R600_CB_COLOR0_BASE, 0);
1744         RADEON_WRITE(R600_CB_COLOR1_BASE, 0);
1745         RADEON_WRITE(R600_CB_COLOR2_BASE, 0);
1746         RADEON_WRITE(R600_CB_COLOR3_BASE, 0);
1747         RADEON_WRITE(R600_CB_COLOR4_BASE, 0);
1748         RADEON_WRITE(R600_CB_COLOR5_BASE, 0);
1749         RADEON_WRITE(R600_CB_COLOR6_BASE, 0);
1750         RADEON_WRITE(R600_CB_COLOR7_BASE, 0);
1751
1752         RADEON_WRITE(R700_TCP_CNTL, 0);
1753
1754         hdp_host_path_cntl = RADEON_READ(R600_HDP_HOST_PATH_CNTL);
1755         RADEON_WRITE(R600_HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1756
1757         RADEON_WRITE(R600_PA_SC_MULTI_CHIP_CNTL, 0);
1758
1759         RADEON_WRITE(R600_PA_CL_ENHANCE, (R600_CLIP_VTX_REORDER_ENA |
1760                                           R600_NUM_CLIP_SEQ(3)));
1761
1762 }
1763
1764 static void r600_cp_init_ring_buffer(struct drm_device *dev,
1765                                        drm_radeon_private_t *dev_priv,
1766                                        struct drm_file *file_priv)
1767 {
1768         struct drm_radeon_master_private *master_priv;
1769         u32 ring_start;
1770         u64 rptr_addr;
1771
1772         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
1773                 r700_gfx_init(dev, dev_priv);
1774         else
1775                 r600_gfx_init(dev, dev_priv);
1776
1777         RADEON_WRITE(R600_GRBM_SOFT_RESET, R600_SOFT_RESET_CP);
1778         RADEON_READ(R600_GRBM_SOFT_RESET);
1779         DRM_MDELAY(15);
1780         RADEON_WRITE(R600_GRBM_SOFT_RESET, 0);
1781
1782
1783         /* Set ring buffer size */
1784 #ifdef __BIG_ENDIAN
1785         RADEON_WRITE(R600_CP_RB_CNTL,
1786                      R600_BUF_SWAP_32BIT |
1787                      R600_RB_NO_UPDATE |
1788                      (dev_priv->ring.rptr_update_l2qw << 8) |
1789                      dev_priv->ring.size_l2qw);
1790 #else
1791         RADEON_WRITE(R600_CP_RB_CNTL,
1792                      RADEON_RB_NO_UPDATE |
1793                      (dev_priv->ring.rptr_update_l2qw << 8) |
1794                      dev_priv->ring.size_l2qw);
1795 #endif
1796
1797         RADEON_WRITE(R600_CP_SEM_WAIT_TIMER, 0x0);
1798
1799         /* Set the write pointer delay */
1800         RADEON_WRITE(R600_CP_RB_WPTR_DELAY, 0);
1801
1802 #ifdef __BIG_ENDIAN
1803         RADEON_WRITE(R600_CP_RB_CNTL,
1804                      R600_BUF_SWAP_32BIT |
1805                      R600_RB_NO_UPDATE |
1806                      R600_RB_RPTR_WR_ENA |
1807                      (dev_priv->ring.rptr_update_l2qw << 8) |
1808                      dev_priv->ring.size_l2qw);
1809 #else
1810         RADEON_WRITE(R600_CP_RB_CNTL,
1811                      R600_RB_NO_UPDATE |
1812                      R600_RB_RPTR_WR_ENA |
1813                      (dev_priv->ring.rptr_update_l2qw << 8) |
1814                      dev_priv->ring.size_l2qw);
1815 #endif
1816
1817         /* Initialize the ring buffer's read and write pointers */
1818         RADEON_WRITE(R600_CP_RB_RPTR_WR, 0);
1819         RADEON_WRITE(R600_CP_RB_WPTR, 0);
1820         SET_RING_HEAD(dev_priv, 0);
1821         dev_priv->ring.tail = 0;
1822
1823 #if __OS_HAS_AGP
1824         if (dev_priv->flags & RADEON_IS_AGP) {
1825                 rptr_addr = dev_priv->ring_rptr->offset
1826                         - dev->agp->base +
1827                         dev_priv->gart_vm_start;
1828         } else
1829 #endif
1830         {
1831                 rptr_addr = dev_priv->ring_rptr->offset
1832                         - ((unsigned long) dev->sg->vaddr)
1833                         + dev_priv->gart_vm_start;
1834         }
1835         RADEON_WRITE(R600_CP_RB_RPTR_ADDR, (rptr_addr & 0xfffffffc));
1836         RADEON_WRITE(R600_CP_RB_RPTR_ADDR_HI, upper_32_bits(rptr_addr));
1837
1838 #ifdef __BIG_ENDIAN
1839         RADEON_WRITE(R600_CP_RB_CNTL,
1840                      RADEON_BUF_SWAP_32BIT |
1841                      (dev_priv->ring.rptr_update_l2qw << 8) |
1842                      dev_priv->ring.size_l2qw);
1843 #else
1844         RADEON_WRITE(R600_CP_RB_CNTL,
1845                      (dev_priv->ring.rptr_update_l2qw << 8) |
1846                      dev_priv->ring.size_l2qw);
1847 #endif
1848
1849 #if __OS_HAS_AGP
1850         if (dev_priv->flags & RADEON_IS_AGP) {
1851                 /* XXX */
1852                 radeon_write_agp_base(dev_priv, dev->agp->base);
1853
1854                 /* XXX */
1855                 radeon_write_agp_location(dev_priv,
1856                              (((dev_priv->gart_vm_start - 1 +
1857                                 dev_priv->gart_size) & 0xffff0000) |
1858                               (dev_priv->gart_vm_start >> 16)));
1859
1860                 ring_start = (dev_priv->cp_ring->offset
1861                               - dev->agp->base
1862                               + dev_priv->gart_vm_start);
1863         } else
1864 #endif
1865                 ring_start = (dev_priv->cp_ring->offset
1866                               - (unsigned long)dev->sg->vaddr>
1867                               + dev_priv->gart_vm_start);
1868
1869         RADEON_WRITE(R600_CP_RB_BASE, ring_start >> 8);
1870
1871         RADEON_WRITE(R600_CP_ME_CNTL, 0xff);
1872
1873         RADEON_WRITE(R600_CP_DEBUG, (1 << 27) | (1 << 28));
1874
1875         /* Initialize the scratch register pointer.  This will cause
1876          * the scratch register values to be written out to memory
1877          * whenever they are updated.
1878          *
1879          * We simply put this behind the ring read pointer, this works
1880          * with PCI GART as well as (whatever kind of) AGP GART
1881          */
1882         {
1883                 u64 scratch_addr;
1884
1885                 scratch_addr = RADEON_READ(R600_CP_RB_RPTR_ADDR) & 0xFFFFFFFC;
1886                 scratch_addr |= ((u64)RADEON_READ(R600_CP_RB_RPTR_ADDR_HI)) << 32;
1887                 scratch_addr += R600_SCRATCH_REG_OFFSET;
1888                 scratch_addr >>= 8;
1889                 scratch_addr &= 0xffffffff;
1890
1891                 RADEON_WRITE(R600_SCRATCH_ADDR, (uint32_t)scratch_addr);
1892         }
1893
1894         RADEON_WRITE(R600_SCRATCH_UMSK, 0x7);
1895
1896         /* Turn on bus mastering */
1897         radeon_enable_bm(dev_priv);
1898
1899         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(0), 0);
1900         RADEON_WRITE(R600_LAST_FRAME_REG, 0);
1901
1902         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(1), 0);
1903         RADEON_WRITE(R600_LAST_DISPATCH_REG, 0);
1904
1905         radeon_write_ring_rptr(dev_priv, R600_SCRATCHOFF(2), 0);
1906         RADEON_WRITE(R600_LAST_CLEAR_REG, 0);
1907
1908         /* reset sarea copies of these */
1909         master_priv = file_priv->masterp->driver_priv;
1910         if (master_priv->sarea_priv) {
1911                 master_priv->sarea_priv->last_frame = 0;
1912                 master_priv->sarea_priv->last_dispatch = 0;
1913                 master_priv->sarea_priv->last_clear = 0;
1914         }
1915
1916         r600_do_wait_for_idle(dev_priv);
1917
1918 }
1919
1920 int r600_do_cleanup_cp(struct drm_device *dev)
1921 {
1922         drm_radeon_private_t *dev_priv = dev->dev_private;
1923         DRM_DEBUG("\n");
1924
1925         /* Make sure interrupts are disabled here because the uninstall ioctl
1926          * may not have been called from userspace and after dev_private
1927          * is freed, it's too late.
1928          */
1929         if (dev->irq_enabled)
1930                 drm_irq_uninstall(dev);
1931
1932 #if __OS_HAS_AGP
1933         if (dev_priv->flags & RADEON_IS_AGP) {
1934                 if (dev_priv->cp_ring != NULL) {
1935                         drm_core_ioremapfree(dev_priv->cp_ring, dev);
1936                         dev_priv->cp_ring = NULL;
1937                 }
1938                 if (dev_priv->ring_rptr != NULL) {
1939                         drm_core_ioremapfree(dev_priv->ring_rptr, dev);
1940                         dev_priv->ring_rptr = NULL;
1941                 }
1942                 if (dev->agp_buffer_map != NULL) {
1943                         drm_core_ioremapfree(dev->agp_buffer_map, dev);
1944                         dev->agp_buffer_map = NULL;
1945                 }
1946         } else
1947 #endif
1948         {
1949
1950                 if (dev_priv->gart_info.bus_addr)
1951                         r600_page_table_cleanup(dev, &dev_priv->gart_info);
1952
1953                 if (dev_priv->gart_info.gart_table_location == DRM_ATI_GART_FB) {
1954                         drm_core_ioremapfree(&dev_priv->gart_info.mapping, dev);
1955                         dev_priv->gart_info.addr = NULL;
1956                 }
1957         }
1958         /* only clear to the start of flags */
1959         memset(dev_priv, 0, offsetof(drm_radeon_private_t, flags));
1960
1961         return 0;
1962 }
1963
1964 int r600_do_init_cp(struct drm_device *dev, drm_radeon_init_t *init,
1965                     struct drm_file *file_priv)
1966 {
1967         drm_radeon_private_t *dev_priv = dev->dev_private;
1968         struct drm_radeon_master_private *master_priv = file_priv->masterp->driver_priv;
1969
1970         DRM_DEBUG("\n");
1971
1972         lockinit(&dev_priv->cs_mutex, "drm__radeon_private__cs_mutex", 0,
1973                  LK_CANRECURSE);
1974         r600_cs_legacy_init();
1975         /* if we require new memory map but we don't have it fail */
1976         if ((dev_priv->flags & RADEON_NEW_MEMMAP) && !dev_priv->new_memmap) {
1977                 DRM_ERROR("Cannot initialise DRM on this card\nThis card requires a new X.org DDX for 3D\n");
1978                 r600_do_cleanup_cp(dev);
1979                 return -EINVAL;
1980         }
1981
1982         if (init->is_pci && (dev_priv->flags & RADEON_IS_AGP)) {
1983                 DRM_DEBUG("Forcing AGP card to PCI mode\n");
1984                 dev_priv->flags &= ~RADEON_IS_AGP;
1985                 /* The writeback test succeeds, but when writeback is enabled,
1986                  * the ring buffer read ptr update fails after first 128 bytes.
1987                  */
1988                 radeon_no_wb = 1;
1989         } else if (!(dev_priv->flags & (RADEON_IS_AGP | RADEON_IS_PCI | RADEON_IS_PCIE))
1990                  && !init->is_pci) {
1991                 DRM_DEBUG("Restoring AGP flag\n");
1992                 dev_priv->flags |= RADEON_IS_AGP;
1993         }
1994
1995         dev_priv->usec_timeout = init->usec_timeout;
1996         if (dev_priv->usec_timeout < 1 ||
1997             dev_priv->usec_timeout > RADEON_MAX_USEC_TIMEOUT) {
1998                 DRM_DEBUG("TIMEOUT problem!\n");
1999                 r600_do_cleanup_cp(dev);
2000                 return -EINVAL;
2001         }
2002
2003         /* Enable vblank on CRTC1 for older X servers
2004          */
2005         dev_priv->vblank_crtc = DRM_RADEON_VBLANK_CRTC1;
2006         dev_priv->do_boxes = 0;
2007         dev_priv->cp_mode = init->cp_mode;
2008
2009         /* We don't support anything other than bus-mastering ring mode,
2010          * but the ring can be in either AGP or PCI space for the ring
2011          * read pointer.
2012          */
2013         if ((init->cp_mode != RADEON_CSQ_PRIBM_INDDIS) &&
2014             (init->cp_mode != RADEON_CSQ_PRIBM_INDBM)) {
2015                 DRM_DEBUG("BAD cp_mode (%x)!\n", init->cp_mode);
2016                 r600_do_cleanup_cp(dev);
2017                 return -EINVAL;
2018         }
2019
2020         switch (init->fb_bpp) {
2021         case 16:
2022                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_RGB565;
2023                 break;
2024         case 32:
2025         default:
2026                 dev_priv->color_fmt = RADEON_COLOR_FORMAT_ARGB8888;
2027                 break;
2028         }
2029         dev_priv->front_offset = init->front_offset;
2030         dev_priv->front_pitch = init->front_pitch;
2031         dev_priv->back_offset = init->back_offset;
2032         dev_priv->back_pitch = init->back_pitch;
2033
2034         dev_priv->ring_offset = init->ring_offset;
2035         dev_priv->ring_rptr_offset = init->ring_rptr_offset;
2036         dev_priv->buffers_offset = init->buffers_offset;
2037         dev_priv->gart_textures_offset = init->gart_textures_offset;
2038
2039         master_priv->sarea = drm_getsarea(dev);
2040         if (!master_priv->sarea) {
2041                 DRM_ERROR("could not find sarea!\n");
2042                 r600_do_cleanup_cp(dev);
2043                 return -EINVAL;
2044         }
2045
2046         dev_priv->cp_ring = drm_core_findmap(dev, init->ring_offset);
2047         if (!dev_priv->cp_ring) {
2048                 DRM_ERROR("could not find cp ring region!\n");
2049                 r600_do_cleanup_cp(dev);
2050                 return -EINVAL;
2051         }
2052         dev_priv->ring_rptr = drm_core_findmap(dev, init->ring_rptr_offset);
2053         if (!dev_priv->ring_rptr) {
2054                 DRM_ERROR("could not find ring read pointer!\n");
2055                 r600_do_cleanup_cp(dev);
2056                 return -EINVAL;
2057         }
2058         dev->agp_buffer_token = init->buffers_offset;
2059         dev->agp_buffer_map = drm_core_findmap(dev, init->buffers_offset);
2060         if (!dev->agp_buffer_map) {
2061                 DRM_ERROR("could not find dma buffer region!\n");
2062                 r600_do_cleanup_cp(dev);
2063                 return -EINVAL;
2064         }
2065
2066         if (init->gart_textures_offset) {
2067                 dev_priv->gart_textures =
2068                     drm_core_findmap(dev, init->gart_textures_offset);
2069                 if (!dev_priv->gart_textures) {
2070                         DRM_ERROR("could not find GART texture region!\n");
2071                         r600_do_cleanup_cp(dev);
2072                         return -EINVAL;
2073                 }
2074         }
2075
2076 #if __OS_HAS_AGP
2077         /* XXX */
2078         if (dev_priv->flags & RADEON_IS_AGP) {
2079                 drm_core_ioremap_wc(dev_priv->cp_ring, dev);
2080                 drm_core_ioremap_wc(dev_priv->ring_rptr, dev);
2081                 drm_core_ioremap_wc(dev->agp_buffer_map, dev);
2082                 if (!dev_priv->cp_ring->handle ||
2083                     !dev_priv->ring_rptr->handle ||
2084                     !dev->agp_buffer_map->handle) {
2085                         DRM_ERROR("could not find ioremap agp regions!\n");
2086                         r600_do_cleanup_cp(dev);
2087                         return -EINVAL;
2088                 }
2089         } else
2090 #endif
2091         {
2092                 dev_priv->cp_ring->handle = (void *)(unsigned long)dev_priv->cp_ring->offset;
2093                 dev_priv->ring_rptr->handle =
2094                         (void *)(unsigned long)dev_priv->ring_rptr->offset;
2095                 dev->agp_buffer_map->handle =
2096                         (void *)(unsigned long)dev->agp_buffer_map->offset;
2097
2098                 DRM_DEBUG("dev_priv->cp_ring->handle %p\n",
2099                           dev_priv->cp_ring->handle);
2100                 DRM_DEBUG("dev_priv->ring_rptr->handle %p\n",
2101                           dev_priv->ring_rptr->handle);
2102                 DRM_DEBUG("dev->agp_buffer_map->handle %p\n",
2103                           dev->agp_buffer_map->handle);
2104         }
2105
2106         dev_priv->fb_location = (radeon_read_fb_location(dev_priv) & 0xffff) << 24;
2107         dev_priv->fb_size =
2108                 (((radeon_read_fb_location(dev_priv) & 0xffff0000u) << 8) + 0x1000000)
2109                 - dev_priv->fb_location;
2110
2111         dev_priv->front_pitch_offset = (((dev_priv->front_pitch / 64) << 22) |
2112                                         ((dev_priv->front_offset
2113                                           + dev_priv->fb_location) >> 10));
2114
2115         dev_priv->back_pitch_offset = (((dev_priv->back_pitch / 64) << 22) |
2116                                        ((dev_priv->back_offset
2117                                          + dev_priv->fb_location) >> 10));
2118
2119         dev_priv->depth_pitch_offset = (((dev_priv->depth_pitch / 64) << 22) |
2120                                         ((dev_priv->depth_offset
2121                                           + dev_priv->fb_location) >> 10));
2122
2123         dev_priv->gart_size = init->gart_size;
2124
2125         /* New let's set the memory map ... */
2126         if (dev_priv->new_memmap) {
2127                 u32 base = 0;
2128
2129                 DRM_INFO("Setting GART location based on new memory map\n");
2130
2131                 /* If using AGP, try to locate the AGP aperture at the same
2132                  * location in the card and on the bus, though we have to
2133                  * align it down.
2134                  */
2135 #if __OS_HAS_AGP
2136                 /* XXX */
2137                 if (dev_priv->flags & RADEON_IS_AGP) {
2138                         base = dev->agp->base;
2139                         /* Check if valid */
2140                         if ((base + dev_priv->gart_size - 1) >= dev_priv->fb_location &&
2141                             base < (dev_priv->fb_location + dev_priv->fb_size - 1)) {
2142                                 DRM_INFO("Can't use AGP base @0x%08lx, won't fit\n",
2143                                          dev->agp->base);
2144                                 base = 0;
2145                         }
2146                 }
2147 #endif
2148                 /* If not or if AGP is at 0 (Macs), try to put it elsewhere */
2149                 if (base == 0) {
2150                         base = dev_priv->fb_location + dev_priv->fb_size;
2151                         if (base < dev_priv->fb_location ||
2152                             ((base + dev_priv->gart_size) & 0xfffffffful) < base)
2153                                 base = dev_priv->fb_location
2154                                         - dev_priv->gart_size;
2155                 }
2156                 dev_priv->gart_vm_start = base & 0xffc00000u;
2157                 if (dev_priv->gart_vm_start != base)
2158                         DRM_INFO("GART aligned down from 0x%08x to 0x%08x\n",
2159                                  base, dev_priv->gart_vm_start);
2160         }
2161
2162 #if __OS_HAS_AGP
2163         /* XXX */
2164         if (dev_priv->flags & RADEON_IS_AGP)
2165                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
2166                                                  - dev->agp->base
2167                                                  + dev_priv->gart_vm_start);
2168         else
2169 #endif
2170                 dev_priv->gart_buffers_offset = (dev->agp_buffer_map->offset
2171                                                  - (unsigned long)dev->sg->vaddr
2172                                                  + dev_priv->gart_vm_start);
2173
2174         DRM_DEBUG("fb 0x%08x size %d\n",
2175                   (unsigned int) dev_priv->fb_location,
2176                   (unsigned int) dev_priv->fb_size);
2177         DRM_DEBUG("dev_priv->gart_size %d\n", dev_priv->gart_size);
2178         DRM_DEBUG("dev_priv->gart_vm_start 0x%08x\n",
2179                   (unsigned int) dev_priv->gart_vm_start);
2180         DRM_DEBUG("dev_priv->gart_buffers_offset 0x%08lx\n",
2181                   dev_priv->gart_buffers_offset);
2182
2183         dev_priv->ring.start = (u32 *) dev_priv->cp_ring->handle;
2184         dev_priv->ring.end = ((u32 *) dev_priv->cp_ring->handle
2185                               + init->ring_size / sizeof(u32));
2186         dev_priv->ring.size = init->ring_size;
2187         dev_priv->ring.size_l2qw = drm_order(init->ring_size / 8);
2188
2189         dev_priv->ring.rptr_update = /* init->rptr_update */ 4096;
2190         dev_priv->ring.rptr_update_l2qw = drm_order(/* init->rptr_update */ 4096 / 8);
2191
2192         dev_priv->ring.fetch_size = /* init->fetch_size */ 32;
2193         dev_priv->ring.fetch_size_l2ow = drm_order(/* init->fetch_size */ 32 / 16);
2194
2195         dev_priv->ring.tail_mask = (dev_priv->ring.size / sizeof(u32)) - 1;
2196
2197         dev_priv->ring.high_mark = RADEON_RING_HIGH_MARK;
2198
2199 #if __OS_HAS_AGP
2200         if (dev_priv->flags & RADEON_IS_AGP) {
2201                 /* XXX turn off pcie gart */
2202         } else
2203 #endif
2204         {
2205                 dev_priv->gart_info.table_mask = DMA_BIT_MASK(32);
2206                 /* if we have an offset set from userspace */
2207                 if (!dev_priv->pcigart_offset_set) {
2208                         DRM_ERROR("Need gart offset from userspace\n");
2209                         r600_do_cleanup_cp(dev);
2210                         return -EINVAL;
2211                 }
2212
2213                 DRM_DEBUG("Using gart offset 0x%08lx\n", dev_priv->pcigart_offset);
2214
2215                 dev_priv->gart_info.bus_addr =
2216                         dev_priv->pcigart_offset + dev_priv->fb_location;
2217                 dev_priv->gart_info.mapping.offset =
2218                         dev_priv->pcigart_offset + dev_priv->fb_aper_offset;
2219                 dev_priv->gart_info.mapping.size =
2220                         dev_priv->gart_info.table_size;
2221
2222                 drm_core_ioremap_wc(&dev_priv->gart_info.mapping, dev);
2223                 if (!dev_priv->gart_info.mapping.handle) {
2224                         DRM_ERROR("ioremap failed.\n");
2225                         r600_do_cleanup_cp(dev);
2226                         return -EINVAL;
2227                 }
2228
2229                 dev_priv->gart_info.addr =
2230                         dev_priv->gart_info.mapping.handle;
2231
2232                 DRM_DEBUG("Setting phys_pci_gart to %p %08lX\n",
2233                           dev_priv->gart_info.addr,
2234                           dev_priv->pcigart_offset);
2235
2236                 if (!r600_page_table_init(dev)) {
2237                         DRM_ERROR("Failed to init GART table\n");
2238                         r600_do_cleanup_cp(dev);
2239                         return -EINVAL;
2240                 }
2241
2242                 if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
2243                         r700_vm_init(dev);
2244                 else
2245                         r600_vm_init(dev);
2246         }
2247
2248         if (!dev_priv->me_fw || !dev_priv->pfp_fw) {
2249                 int err = r600_cp_init_microcode(dev_priv);
2250                 if (err) {
2251                         DRM_ERROR("Failed to load firmware!\n");
2252                         r600_do_cleanup_cp(dev);
2253                         return err;
2254                 }
2255         }
2256         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770))
2257                 r700_cp_load_microcode(dev_priv);
2258         else
2259                 r600_cp_load_microcode(dev_priv);
2260
2261         r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
2262
2263         dev_priv->last_buf = 0;
2264
2265         r600_do_engine_reset(dev);
2266         r600_test_writeback(dev_priv);
2267
2268         return 0;
2269 }
2270
2271 int r600_do_resume_cp(struct drm_device *dev, struct drm_file *file_priv)
2272 {
2273         drm_radeon_private_t *dev_priv = dev->dev_private;
2274
2275         DRM_DEBUG("\n");
2276         if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)) {
2277                 r700_vm_init(dev);
2278                 r700_cp_load_microcode(dev_priv);
2279         } else {
2280                 r600_vm_init(dev);
2281                 r600_cp_load_microcode(dev_priv);
2282         }
2283         r600_cp_init_ring_buffer(dev, dev_priv, file_priv);
2284         r600_do_engine_reset(dev);
2285
2286         return 0;
2287 }
2288
2289 /* Wait for the CP to go idle.
2290  */
2291 int r600_do_cp_idle(drm_radeon_private_t *dev_priv)
2292 {
2293         RING_LOCALS;
2294         DRM_DEBUG("\n");
2295
2296         BEGIN_RING(5);
2297         OUT_RING(CP_PACKET3(R600_IT_EVENT_WRITE, 0));
2298         OUT_RING(R600_CACHE_FLUSH_AND_INV_EVENT);
2299         /* wait for 3D idle clean */
2300         OUT_RING(CP_PACKET3(R600_IT_SET_CONFIG_REG, 1));
2301         OUT_RING((R600_WAIT_UNTIL - R600_SET_CONFIG_REG_OFFSET) >> 2);
2302         OUT_RING(RADEON_WAIT_3D_IDLE | RADEON_WAIT_3D_IDLECLEAN);
2303
2304         ADVANCE_RING();
2305         COMMIT_RING();
2306
2307         return r600_do_wait_for_idle(dev_priv);
2308 }
2309
2310 /* Start the Command Processor.
2311  */
2312 void r600_do_cp_start(drm_radeon_private_t *dev_priv)
2313 {
2314         u32 cp_me;
2315         RING_LOCALS;
2316         DRM_DEBUG("\n");
2317
2318         BEGIN_RING(7);
2319         OUT_RING(CP_PACKET3(R600_IT_ME_INITIALIZE, 5));
2320         OUT_RING(0x00000001);
2321         if (((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV770))
2322                 OUT_RING(0x00000003);
2323         else
2324                 OUT_RING(0x00000000);
2325         OUT_RING((dev_priv->r600_max_hw_contexts - 1));
2326         OUT_RING(R600_ME_INITIALIZE_DEVICE_ID(1));
2327         OUT_RING(0x00000000);
2328         OUT_RING(0x00000000);
2329         ADVANCE_RING();
2330         COMMIT_RING();
2331
2332         /* set the mux and reset the halt bit */
2333         cp_me = 0xff;
2334         RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
2335
2336         dev_priv->cp_running = 1;
2337
2338 }
2339
2340 void r600_do_cp_reset(drm_radeon_private_t *dev_priv)
2341 {
2342         u32 cur_read_ptr;
2343         DRM_DEBUG("\n");
2344
2345         cur_read_ptr = RADEON_READ(R600_CP_RB_RPTR);
2346         RADEON_WRITE(R600_CP_RB_WPTR, cur_read_ptr);
2347         SET_RING_HEAD(dev_priv, cur_read_ptr);
2348         dev_priv->ring.tail = cur_read_ptr;
2349 }
2350
2351 void r600_do_cp_stop(drm_radeon_private_t *dev_priv)
2352 {
2353         uint32_t cp_me;
2354
2355         DRM_DEBUG("\n");
2356
2357         cp_me = 0xff | R600_CP_ME_HALT;
2358
2359         RADEON_WRITE(R600_CP_ME_CNTL, cp_me);
2360
2361         dev_priv->cp_running = 0;
2362 }
2363
2364 int r600_cp_dispatch_indirect(struct drm_device *dev,
2365                               struct drm_buf *buf, int start, int end)
2366 {
2367         drm_radeon_private_t *dev_priv = dev->dev_private;
2368         RING_LOCALS;
2369
2370         if (start != end) {
2371                 unsigned long offset = (dev_priv->gart_buffers_offset
2372                                         + buf->offset + start);
2373                 int dwords = (end - start + 3) / sizeof(u32);
2374
2375                 DRM_DEBUG("dwords:%d\n", dwords);
2376                 DRM_DEBUG("offset 0x%lx\n", offset);
2377
2378
2379                 /* Indirect buffer data must be a multiple of 16 dwords.
2380                  * pad the data with a Type-2 CP packet.
2381                  */
2382                 while (dwords & 0xf) {
2383                         u32 *data = (u32 *)
2384                             ((char *)dev->agp_buffer_map->handle
2385                              + buf->offset + start);
2386                         data[dwords++] = RADEON_CP_PACKET2;
2387                 }
2388
2389                 /* Fire off the indirect buffer */
2390                 BEGIN_RING(4);
2391                 OUT_RING(CP_PACKET3(R600_IT_INDIRECT_BUFFER, 2));
2392                 OUT_RING((offset & 0xfffffffc));
2393                 OUT_RING((upper_32_bits(offset) & 0xff));
2394                 OUT_RING(dwords);
2395                 ADVANCE_RING();
2396         }
2397
2398         return 0;
2399 }
2400
2401 void r600_cp_dispatch_swap(struct drm_device *dev, struct drm_file *file_priv)
2402 {
2403         drm_radeon_private_t *dev_priv = dev->dev_private;
2404         struct drm_master *master = file_priv->masterp;
2405         struct drm_radeon_master_private *master_priv = master->driver_priv;
2406         drm_radeon_sarea_t *sarea_priv = master_priv->sarea_priv;
2407         int nbox = sarea_priv->nbox;
2408         struct drm_clip_rect *pbox = sarea_priv->boxes;
2409         int i, cpp, src_pitch, dst_pitch;
2410         uint64_t src, dst;
2411         RING_LOCALS;
2412         DRM_DEBUG("\n");
2413
2414         if (dev_priv->color_fmt == RADEON_COLOR_FORMAT_ARGB8888)
2415                 cpp = 4;
2416         else
2417                 cpp = 2;
2418
2419         if (sarea_priv->pfCurrentPage == 0) {
2420                 src_pitch = dev_priv->back_pitch;
2421                 dst_pitch = dev_priv->front_pitch;
2422                 src = dev_priv->back_offset + dev_priv->fb_location;
2423                 dst = dev_priv->front_offset + dev_priv->fb_location;
2424         } else {
2425                 src_pitch = dev_priv->front_pitch;
2426                 dst_pitch = dev_priv->back_pitch;
2427                 src = dev_priv->front_offset + dev_priv->fb_location;
2428                 dst = dev_priv->back_offset + dev_priv->fb_location;
2429         }
2430
2431         if (r600_prepare_blit_copy(dev, file_priv)) {
2432                 DRM_ERROR("unable to allocate vertex buffer for swap buffer\n");
2433                 return;
2434         }
2435         for (i = 0; i < nbox; i++) {
2436                 int x = pbox[i].x1;
2437                 int y = pbox[i].y1;
2438                 int w = pbox[i].x2 - x;
2439                 int h = pbox[i].y2 - y;
2440
2441                 DRM_DEBUG("%d,%d-%d,%d\n", x, y, w, h);
2442
2443                 r600_blit_swap(dev,
2444                                src, dst,
2445                                x, y, x, y, w, h,
2446                                src_pitch, dst_pitch, cpp);
2447         }
2448         r600_done_blit_copy(dev);
2449
2450         /* Increment the frame counter.  The client-side 3D driver must
2451          * throttle the framerate by waiting for this value before
2452          * performing the swapbuffer ioctl.
2453          */
2454         sarea_priv->last_frame++;
2455
2456         BEGIN_RING(3);
2457         R600_FRAME_AGE(sarea_priv->last_frame);
2458         ADVANCE_RING();
2459 }
2460
2461 int r600_cp_dispatch_texture(struct drm_device *dev,
2462                              struct drm_file *file_priv,
2463                              drm_radeon_texture_t *tex,
2464                              drm_radeon_tex_image_t *image)
2465 {
2466         drm_radeon_private_t *dev_priv = dev->dev_private;
2467         struct drm_buf *buf;
2468         u32 *buffer;
2469         const u8 __user *data;
2470         int size, pass_size;
2471         u64 src_offset, dst_offset;
2472
2473         if (!radeon_check_offset(dev_priv, tex->offset)) {
2474                 DRM_ERROR("Invalid destination offset\n");
2475                 return -EINVAL;
2476         }
2477
2478         /* this might fail for zero-sized uploads - are those illegal? */
2479         if (!radeon_check_offset(dev_priv, tex->offset + tex->height * tex->pitch - 1)) {
2480                 DRM_ERROR("Invalid final destination offset\n");
2481                 return -EINVAL;
2482         }
2483
2484         size = tex->height * tex->pitch;
2485
2486         if (size == 0)
2487                 return 0;
2488
2489         dst_offset = tex->offset;
2490
2491         if (r600_prepare_blit_copy(dev, file_priv)) {
2492                 DRM_ERROR("unable to allocate vertex buffer for swap buffer\n");
2493                 return -EAGAIN;
2494         }
2495         do {
2496                 data = (const u8 __user *)image->data;
2497                 pass_size = size;
2498
2499                 buf = radeon_freelist_get(dev);
2500                 if (!buf) {
2501                         DRM_DEBUG("EAGAIN\n");
2502                         if (DRM_COPY_TO_USER(tex->image, image, sizeof(*image)))
2503                                 return -EFAULT;
2504                         return -EAGAIN;
2505                 }
2506
2507                 if (pass_size > buf->total)
2508                         pass_size = buf->total;
2509
2510                 /* Dispatch the indirect buffer.
2511                  */
2512                 buffer =
2513                     (u32 *) ((char *)dev->agp_buffer_map->handle + buf->offset);
2514
2515                 if (DRM_COPY_FROM_USER(buffer, data, pass_size)) {
2516                         DRM_ERROR("EFAULT on pad, %d bytes\n", pass_size);
2517                         return -EFAULT;
2518                 }
2519
2520                 buf->file_priv = file_priv;
2521                 buf->used = pass_size;
2522                 src_offset = dev_priv->gart_buffers_offset + buf->offset;
2523
2524                 r600_blit_copy(dev, src_offset, dst_offset, pass_size);
2525
2526                 radeon_cp_discard_buffer(dev, file_priv->masterp, buf);
2527
2528                 /* Update the input parameters for next time */
2529                 image->data = (const u8 __user *)image->data + pass_size;
2530                 dst_offset += pass_size;
2531                 size -= pass_size;
2532         } while (size > 0);
2533         r600_done_blit_copy(dev);
2534
2535         return 0;
2536 }
2537
2538 /*
2539  * Legacy cs ioctl
2540  */
2541 static u32 radeon_cs_id_get(struct drm_radeon_private *radeon)
2542 {
2543         /* FIXME: check if wrap affect last reported wrap & sequence */
2544         radeon->cs_id_scnt = (radeon->cs_id_scnt + 1) & 0x00FFFFFF;
2545         if (!radeon->cs_id_scnt) {
2546                 /* increment wrap counter */
2547                 radeon->cs_id_wcnt += 0x01000000;
2548                 /* valid sequence counter start at 1 */
2549                 radeon->cs_id_scnt = 1;
2550         }
2551         return (radeon->cs_id_scnt | radeon->cs_id_wcnt);
2552 }
2553
2554 static void r600_cs_id_emit(drm_radeon_private_t *dev_priv, u32 *id)
2555 {
2556         RING_LOCALS;
2557
2558         *id = radeon_cs_id_get(dev_priv);
2559
2560         /* SCRATCH 2 */
2561         BEGIN_RING(3);
2562         R600_CLEAR_AGE(*id);
2563         ADVANCE_RING();
2564         COMMIT_RING();
2565 }
2566
2567 static int r600_ib_get(struct drm_device *dev,
2568                         struct drm_file *fpriv,
2569                         struct drm_buf **buffer)
2570 {
2571         struct drm_buf *buf;
2572
2573         *buffer = NULL;
2574         buf = radeon_freelist_get(dev);
2575         if (!buf) {
2576                 return -EBUSY;
2577         }
2578         buf->file_priv = fpriv;
2579         *buffer = buf;
2580         return 0;
2581 }
2582
2583 static void r600_ib_free(struct drm_device *dev, struct drm_buf *buf,
2584                         struct drm_file *fpriv, int l, int r)
2585 {
2586         drm_radeon_private_t *dev_priv = dev->dev_private;
2587
2588         if (buf) {
2589                 if (!r)
2590                         r600_cp_dispatch_indirect(dev, buf, 0, l * 4);
2591                 radeon_cp_discard_buffer(dev, fpriv->masterp, buf);
2592                 COMMIT_RING();
2593         }
2594 }
2595
2596 int r600_cs_legacy_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
2597 {
2598         struct drm_radeon_private *dev_priv = dev->dev_private;
2599         struct drm_radeon_cs *cs = data;
2600         struct drm_buf *buf;
2601         unsigned family;
2602         int l, r = 0;
2603         u32 *ib, cs_id = 0;
2604
2605         if (dev_priv == NULL) {
2606                 DRM_ERROR("called with no initialization\n");
2607                 return -EINVAL;
2608         }
2609         family = dev_priv->flags & RADEON_FAMILY_MASK;
2610         if (family < CHIP_R600) {
2611                 DRM_ERROR("cs ioctl valid only for R6XX & R7XX in legacy mode\n");
2612                 return -EINVAL;
2613         }
2614         lockmgr(&dev_priv->cs_mutex, LK_EXCLUSIVE);
2615         /* get ib */
2616         l = 0;
2617         r = r600_ib_get(dev, fpriv, &buf);
2618         if (r) {
2619                 DRM_ERROR("ib_get failed\n");
2620                 goto out;
2621         }
2622         ib = (u32 *)((uintptr_t)dev->agp_buffer_map->handle + buf->offset);
2623         /* now parse command stream */
2624         r = r600_cs_legacy(dev, data,  fpriv, family, ib, &l);
2625         if (r) {
2626                 goto out;
2627         }
2628
2629 out:
2630         r600_ib_free(dev, buf, fpriv, l, r);
2631         /* emit cs id sequence */
2632         r600_cs_id_emit(dev_priv, &cs_id);
2633         cs->cs_id = cs_id;
2634         lockmgr(&dev_priv->cs_mutex, LK_RELEASE);
2635         return r;
2636 }
2637
2638 void r600_cs_legacy_get_tiling_conf(struct drm_device *dev, u32 *npipes, u32 *nbanks, u32 *group_size)
2639 {
2640         struct drm_radeon_private *dev_priv = dev->dev_private;
2641
2642         *npipes = dev_priv->r600_npipes;
2643         *nbanks = dev_priv->r600_nbanks;
2644         *group_size = dev_priv->r600_group_size;
2645 }