sys/dev/drm/radeon_cs.c

   1 /*-
   2  * Copyright 2008 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Jerome Glisse <glisse@freedesktop.org>
  26  * __FBSDID("$FreeBSD: src/sys/dev/drm/radeon_cs.c,v 1.2 2009/09/28 22:41:28 rnoland Exp $");
  27  */
  28
  29 #include "dev/drm/drmP.h"
  30 #include "dev/drm/radeon_drm.h"
  31 #include "dev/drm/radeon_drv.h"
  32
  33 /* regs */
  34 #define AVIVO_D1MODE_VLINE_START_END                           0x6538
  35 #define AVIVO_D2MODE_VLINE_START_END                           0x6d38
  36 #define R600_CP_COHER_BASE                                     0x85f8
  37 #define R600_DB_DEPTH_BASE                                     0x2800c
  38 #define R600_CB_COLOR0_BASE                                    0x28040
  39 #define R600_CB_COLOR1_BASE                                    0x28044
  40 #define R600_CB_COLOR2_BASE                                    0x28048
  41 #define R600_CB_COLOR3_BASE                                    0x2804c
  42 #define R600_CB_COLOR4_BASE                                    0x28050
  43 #define R600_CB_COLOR5_BASE                                    0x28054
  44 #define R600_CB_COLOR6_BASE                                    0x28058
  45 #define R600_CB_COLOR7_BASE                                    0x2805c
  46 #define R600_SQ_PGM_START_FS                                   0x28894
  47 #define R600_SQ_PGM_START_ES                                   0x28880
  48 #define R600_SQ_PGM_START_VS                                   0x28858
  49 #define R600_SQ_PGM_START_GS                                   0x2886c
  50 #define R600_SQ_PGM_START_PS                                   0x28840
  51 #define R600_VGT_DMA_BASE                                      0x287e8
  52 #define R600_VGT_DMA_BASE_HI                                   0x287e4
  53 #define R600_VGT_STRMOUT_BASE_OFFSET_0                         0x28b10
  54 #define R600_VGT_STRMOUT_BASE_OFFSET_1                         0x28b14
  55 #define R600_VGT_STRMOUT_BASE_OFFSET_2                         0x28b18
  56 #define R600_VGT_STRMOUT_BASE_OFFSET_3                         0x28b1c
  57 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_0                      0x28b44
  58 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_1                      0x28b48
  59 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_2                      0x28b4c
  60 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_3                      0x28b50
  61 #define R600_VGT_STRMOUT_BUFFER_BASE_0                         0x28ad8
  62 #define R600_VGT_STRMOUT_BUFFER_BASE_1                         0x28ae8
  63 #define R600_VGT_STRMOUT_BUFFER_BASE_2                         0x28af8
  64 #define R600_VGT_STRMOUT_BUFFER_BASE_3                         0x28b08
  65 #define R600_VGT_STRMOUT_BUFFER_OFFSET_0                       0x28adc
  66 #define R600_VGT_STRMOUT_BUFFER_OFFSET_1                       0x28aec
  67 #define R600_VGT_STRMOUT_BUFFER_OFFSET_2                       0x28afc
  68 #define R600_VGT_STRMOUT_BUFFER_OFFSET_3                       0x28b0c
  69
  70 /* resource type */
  71 #define R600_SQ_TEX_VTX_INVALID_TEXTURE                        0x0
  72 #define R600_SQ_TEX_VTX_INVALID_BUFFER                         0x1
  73 #define R600_SQ_TEX_VTX_VALID_TEXTURE                          0x2
  74 #define R600_SQ_TEX_VTX_VALID_BUFFER                           0x3
  75
  76 /* packet 3 type offsets */
  77 #define R600_SET_CONFIG_REG_OFFSET                             0x00008000
  78 #define R600_SET_CONFIG_REG_END                                0x0000ac00
  79 #define R600_SET_CONTEXT_REG_OFFSET                            0x00028000
  80 #define R600_SET_CONTEXT_REG_END                               0x00029000
  81 #define R600_SET_ALU_CONST_OFFSET                              0x00030000
  82 #define R600_SET_ALU_CONST_END                                 0x00032000
  83 #define R600_SET_RESOURCE_OFFSET                               0x00038000
  84 #define R600_SET_RESOURCE_END                                  0x0003c000
  85 #define R600_SET_SAMPLER_OFFSET                                0x0003c000
  86 #define R600_SET_SAMPLER_END                                   0x0003cff0
  87 #define R600_SET_CTL_CONST_OFFSET                              0x0003cff0
  88 #define R600_SET_CTL_CONST_END                                 0x0003e200
  89 #define R600_SET_LOOP_CONST_OFFSET                             0x0003e200
  90 #define R600_SET_LOOP_CONST_END                                0x0003e380
  91 #define R600_SET_BOOL_CONST_OFFSET                             0x0003e380
  92 #define R600_SET_BOOL_CONST_END                                0x00040000
  93
  94 /* Packet 3 types */
  95 #define R600_IT_INDIRECT_BUFFER_END               0x00001700
  96 #define R600_IT_SET_PREDICATION                   0x00002000
  97 #define R600_IT_REG_RMW                           0x00002100
  98 #define R600_IT_COND_EXEC                         0x00002200
  99 #define R600_IT_PRED_EXEC                         0x00002300
 100 #define R600_IT_START_3D_CMDBUF                   0x00002400
 101 #define R600_IT_DRAW_INDEX_2                      0x00002700
 102 #define R600_IT_CONTEXT_CONTROL                   0x00002800
 103 #define R600_IT_DRAW_INDEX_IMMD_BE                0x00002900
 104 #define R600_IT_INDEX_TYPE                        0x00002A00
 105 #define R600_IT_DRAW_INDEX                        0x00002B00
 106 #define R600_IT_DRAW_INDEX_AUTO                   0x00002D00
 107 #define R600_IT_DRAW_INDEX_IMMD                   0x00002E00
 108 #define R600_IT_NUM_INSTANCES                     0x00002F00
 109 #define R600_IT_STRMOUT_BUFFER_UPDATE             0x00003400
 110 #define R600_IT_INDIRECT_BUFFER_MP                0x00003800
 111 #define R600_IT_MEM_SEMAPHORE                     0x00003900
 112 #define R600_IT_MPEG_INDEX                        0x00003A00
 113 #define R600_IT_WAIT_REG_MEM                      0x00003C00
 114 #define R600_IT_MEM_WRITE                         0x00003D00
 115 #define R600_IT_INDIRECT_BUFFER                   0x00003200
 116 #define R600_IT_CP_INTERRUPT                      0x00004000
 117 #define R600_IT_SURFACE_SYNC                      0x00004300
 118 #define R600_IT_ME_INITIALIZE                     0x00004400
 119 #define R600_IT_COND_WRITE                        0x00004500
 120 #define R600_IT_EVENT_WRITE                       0x00004600
 121 #define R600_IT_EVENT_WRITE_EOP                   0x00004700
 122 #define R600_IT_ONE_REG_WRITE                     0x00005700
 123 #define R600_IT_SET_CONFIG_REG                    0x00006800
 124 #define R600_IT_SET_CONTEXT_REG                   0x00006900
 125 #define R600_IT_SET_ALU_CONST                     0x00006A00
 126 #define R600_IT_SET_BOOL_CONST                    0x00006B00
 127 #define R600_IT_SET_LOOP_CONST                    0x00006C00
 128 #define R600_IT_SET_RESOURCE                      0x00006D00
 129 #define R600_IT_SET_SAMPLER                       0x00006E00
 130 #define R600_IT_SET_CTL_CONST                     0x00006F00
 131 #define R600_IT_SURFACE_BASE_UPDATE               0x00007300
 132
 133 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 134 {
 135         struct drm_radeon_cs_parser parser;
 136         struct drm_radeon_private *dev_priv = dev->dev_private;
 137         struct drm_radeon_cs *cs = data;
 138         uint32_t cs_id;
 139         struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
 140         uint64_t *chunk_array;
 141         uint64_t *chunk_array_ptr;
 142         long size;
 143         int r, i;
 144
 145         DRM_SPINLOCK(&dev_priv->cs.cs_mutex);
 146         /* set command stream id to 0 which is fake id */
 147         cs_id = 0;
 148         cs->cs_id = cs_id;
 149
 150         if (dev_priv == NULL) {
 151                 DRM_ERROR("called with no initialization\n");
 152                 DRM_SPINUNLOCK(&dev_priv->cs.cs_mutex);
 153                 return -EINVAL;
 154         }
 155         if (!cs->num_chunks) {
 156                 DRM_SPINUNLOCK(&dev_priv->cs.cs_mutex);
 157                 return 0;
 158         }
 159
 160
 161         chunk_array = drm_calloc(cs->num_chunks, sizeof(uint64_t), DRM_MEM_DRIVER);
 162         if (!chunk_array) {
 163                 DRM_SPINUNLOCK(&dev_priv->cs.cs_mutex);
 164                 return -ENOMEM;
 165         }
 166
 167         chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
 168
 169         if (DRM_COPY_FROM_USER(chunk_array, chunk_array_ptr, sizeof(uint64_t)*cs->num_chunks)) {
 170                 r = -EFAULT;
 171                 goto out;
 172         }
 173
 174         parser.dev = dev;
 175         parser.file_priv = fpriv;
 176         parser.reloc_index = -1;
 177         parser.ib_index = -1;
 178         parser.num_chunks = cs->num_chunks;
 179         /* copy out the chunk headers */
 180         parser.chunks = drm_calloc(parser.num_chunks, sizeof(struct drm_radeon_kernel_chunk), DRM_MEM_DRIVER);
 181         if (!parser.chunks) {
 182                 r = -ENOMEM;
 183                 goto out;
 184         }
 185
 186         for (i = 0; i < parser.num_chunks; i++) {
 187                 struct drm_radeon_cs_chunk user_chunk;
 188
 189                 chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
 190
 191                 if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr, sizeof(struct drm_radeon_cs_chunk))){
 192                         r = -EFAULT;
 193                         goto out;
 194                 }
 195                 parser.chunks[i].chunk_id = user_chunk.chunk_id;
 196
 197                 if (parser.chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS)
 198                         parser.reloc_index = i;
 199
 200                 if (parser.chunks[i].chunk_id == RADEON_CHUNK_ID_IB)
 201                         parser.ib_index = i;
 202
 203                 if (parser.chunks[i].chunk_id == RADEON_CHUNK_ID_OLD) {
 204                         parser.ib_index = i;
 205                         parser.reloc_index = -1;
 206                 }
 207
 208                 parser.chunks[i].length_dw = user_chunk.length_dw;
 209                 parser.chunks[i].chunk_data = (uint32_t *)(unsigned long)user_chunk.chunk_data;
 210
 211                 parser.chunks[i].kdata = NULL;
 212                 size = parser.chunks[i].length_dw * sizeof(uint32_t);
 213
 214                 switch(parser.chunks[i].chunk_id) {
 215                 case RADEON_CHUNK_ID_IB:
 216                 case RADEON_CHUNK_ID_OLD:
 217                         if (size == 0) {
 218                                 r = -EINVAL;
 219                                 goto out;
 220                         }
 221                 case RADEON_CHUNK_ID_RELOCS:
 222                         if (size) {
 223                                 parser.chunks[i].kdata = drm_alloc(size, DRM_MEM_DRIVER);
 224                                 if (!parser.chunks[i].kdata) {
 225                                         r = -ENOMEM;
 226                                         goto out;
 227                                 }
 228
 229                                 if (DRM_COPY_FROM_USER(parser.chunks[i].kdata, parser.chunks[i].chunk_data, size)) {
 230                                         r = -EFAULT;
 231                                         goto out;
 232                                 }
 233                         } else
 234                                 parser.chunks[i].kdata = NULL;
 235                         break;
 236                 default:
 237                         break;
 238                 }
 239                 DRM_DEBUG("chunk %d %d %d %p\n", i, parser.chunks[i].chunk_id, parser.chunks[i].length_dw,
 240                           parser.chunks[i].chunk_data);
 241         }
 242
 243         if (parser.chunks[parser.ib_index].length_dw > (16 * 1024)) {
 244                 DRM_ERROR("cs->dwords too big: %d\n", parser.chunks[parser.ib_index].length_dw);
 245                 r = -EINVAL;
 246                 goto out;
 247         }
 248
 249         /* get ib */
 250         r = dev_priv->cs.ib_get(&parser);
 251         if (r) {
 252                 DRM_ERROR("ib_get failed\n");
 253                 goto out;
 254         }
 255
 256         /* now parse command stream */
 257         r = dev_priv->cs.parse(&parser);
 258         if (r) {
 259                 goto out;
 260         }
 261
 262 out:
 263         dev_priv->cs.ib_free(&parser, r);
 264
 265         /* emit cs id sequence */
 266         dev_priv->cs.id_emit(&parser, &cs_id);
 267
 268         cs->cs_id = cs_id;
 269
 270         DRM_SPINUNLOCK(&dev_priv->cs.cs_mutex);
 271
 272         for (i = 0; i < parser.num_chunks; i++) {
 273                 if (parser.chunks[i].kdata)
 274                         drm_free(parser.chunks[i].kdata, parser.chunks[i].length_dw * sizeof(uint32_t), DRM_MEM_DRIVER);
 275         }
 276
 277         drm_free(parser.chunks, sizeof(struct drm_radeon_kernel_chunk)*parser.num_chunks, DRM_MEM_DRIVER);
 278         drm_free(chunk_array, sizeof(uint64_t)*parser.num_chunks, DRM_MEM_DRIVER);
 279
 280         return r;
 281 }
 282
 283 /* for non-mm */
 284 static int r600_nomm_relocate(struct drm_radeon_cs_parser *parser, uint32_t *reloc, uint64_t *offset)
 285 {
 286         struct drm_device *dev = parser->dev;
 287         drm_radeon_private_t *dev_priv = dev->dev_private;
 288         struct drm_radeon_kernel_chunk *reloc_chunk = &parser->chunks[parser->reloc_index];
 289         uint32_t offset_dw = reloc[1];
 290
 291         //DRM_INFO("reloc: 0x%08x 0x%08x\n", reloc[0], reloc[1]);
 292         //DRM_INFO("length: %d\n", reloc_chunk->length_dw);
 293
 294         if (!reloc_chunk->kdata)
 295                 return -EINVAL;
 296
 297         if (offset_dw > reloc_chunk->length_dw) {
 298                 DRM_ERROR("Offset larger than chunk 0x%x %d\n", offset_dw, reloc_chunk->length_dw);
 299                 return -EINVAL;
 300         }
 301
 302         /* 40 bit addr */
 303         *offset = reloc_chunk->kdata[offset_dw + 3];
 304         *offset <<= 32;
 305         *offset |= reloc_chunk->kdata[offset_dw + 0];
 306
 307         //DRM_INFO("offset 0x%lx\n", *offset);
 308
 309         if (!radeon_check_offset(dev_priv, *offset)) {
 310                 DRM_ERROR("bad offset! 0x%lx\n", (unsigned long)*offset);
 311                 return -EINVAL;
 312         }
 313
 314         return 0;
 315 }
 316
 317 static inline int r600_cs_packet0(struct drm_radeon_cs_parser *parser, uint32_t *offset_dw_p)
 318 {
 319         uint32_t hdr, num_dw, reg;
 320         int count_dw = 1;
 321         int ret = 0;
 322         uint32_t offset_dw = *offset_dw_p;
 323         int incr = 2;
 324
 325         hdr = parser->chunks[parser->ib_index].kdata[offset_dw];
 326         num_dw = ((hdr & RADEON_CP_PACKET_COUNT_MASK) >> 16) + 2;
 327         reg = (hdr & 0xffff) << 2;
 328
 329         while (count_dw < num_dw) {
 330                 switch (reg) {
 331                 case AVIVO_D1MODE_VLINE_START_END:
 332                 case AVIVO_D2MODE_VLINE_START_END:
 333                         break;
 334                 default:
 335                         ret = -EINVAL;
 336                         DRM_ERROR("bad packet 0 reg: 0x%08x\n", reg);
 337                         break;
 338                 }
 339                 if (ret)
 340                         break;
 341                 count_dw++;
 342                 reg += 4;
 343         }
 344         *offset_dw_p += incr;
 345         return ret;
 346 }
 347
 348 static inline int r600_cs_packet3(struct drm_radeon_cs_parser *parser, uint32_t *offset_dw_p)
 349 {
 350         struct drm_device *dev = parser->dev;
 351         drm_radeon_private_t *dev_priv = dev->dev_private;
 352         uint32_t hdr, num_dw, start_reg, end_reg, reg;
 353         uint32_t *reloc;
 354         uint64_t offset;
 355         int ret = 0;
 356         uint32_t offset_dw = *offset_dw_p;
 357         int incr = 2;
 358         int i;
 359         struct drm_radeon_kernel_chunk *ib_chunk;
 360
 361         ib_chunk = &parser->chunks[parser->ib_index];
 362
 363         hdr = ib_chunk->kdata[offset_dw];
 364         num_dw = ((hdr & RADEON_CP_PACKET_COUNT_MASK) >> 16) + 2;
 365
 366         /* just the ones we use for now, add more later */
 367         switch (hdr & 0xff00) {
 368         case R600_IT_START_3D_CMDBUF:
 369                 //DRM_INFO("R600_IT_START_3D_CMDBUF\n");
 370                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 371                         ret = -EINVAL;
 372                 if (num_dw != 2)
 373                         ret = -EINVAL;
 374                 if (ret)
 375                         DRM_ERROR("bad START_3D\n");
 376                 break;
 377         case R600_IT_CONTEXT_CONTROL:
 378                 //DRM_INFO("R600_IT_CONTEXT_CONTROL\n");
 379                 if (num_dw != 3)
 380                         ret = -EINVAL;
 381                 if (ret)
 382                         DRM_ERROR("bad CONTEXT_CONTROL\n");
 383                 break;
 384         case R600_IT_INDEX_TYPE:
 385         case R600_IT_NUM_INSTANCES:
 386                 //DRM_INFO("R600_IT_INDEX_TYPE/R600_IT_NUM_INSTANCES\n");
 387                 if (num_dw != 2)
 388                         ret = -EINVAL;
 389                 if (ret)
 390                         DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES\n");
 391                 break;
 392         case R600_IT_DRAW_INDEX:
 393                 //DRM_INFO("R600_IT_DRAW_INDEX\n");
 394                 if (num_dw != 5) {
 395                         ret = -EINVAL;
 396                         DRM_ERROR("bad DRAW_INDEX\n");
 397                         break;
 398                 }
 399                 reloc = ib_chunk->kdata + offset_dw + num_dw;
 400                 ret = dev_priv->cs.relocate(parser, reloc, &offset);
 401                 if (ret) {
 402                         DRM_ERROR("bad DRAW_INDEX\n");
 403                         break;
 404                 }
 405                 ib_chunk->kdata[offset_dw + 1] += (offset & 0xffffffff);
 406                 ib_chunk->kdata[offset_dw + 2] += (upper_32_bits(offset) & 0xff);
 407                 break;
 408         case R600_IT_DRAW_INDEX_AUTO:
 409                 //DRM_INFO("R600_IT_DRAW_INDEX_AUTO\n");
 410                 if (num_dw != 3)
 411                         ret = -EINVAL;
 412                 if (ret)
 413                         DRM_ERROR("bad DRAW_INDEX_AUTO\n");
 414                 break;
 415         case R600_IT_DRAW_INDEX_IMMD_BE:
 416         case R600_IT_DRAW_INDEX_IMMD:
 417                 //DRM_INFO("R600_IT_DRAW_INDEX_IMMD\n");
 418                 if (num_dw < 4)
 419                         ret = -EINVAL;
 420                 if (ret)
 421                         DRM_ERROR("bad DRAW_INDEX_IMMD\n");
 422                 break;
 423         case R600_IT_WAIT_REG_MEM:
 424                 //DRM_INFO("R600_IT_WAIT_REG_MEM\n");
 425                 if (num_dw != 7)
 426                         ret = -EINVAL;
 427                 /* bit 4 is reg (0) or mem (1) */
 428                 if (ib_chunk->kdata[offset_dw + 1] & 0x10) {
 429                         reloc = ib_chunk->kdata + offset_dw + num_dw;
 430                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 431                         if (ret) {
 432                                 DRM_ERROR("bad WAIT_REG_MEM\n");
 433                                 break;
 434                         }
 435                         ib_chunk->kdata[offset_dw + 2] += (offset & 0xffffffff);
 436                         ib_chunk->kdata[offset_dw + 3] += (upper_32_bits(offset) & 0xff);
 437                 }
 438                 if (ret)
 439                         DRM_ERROR("bad WAIT_REG_MEM\n");
 440                 break;
 441         case R600_IT_SURFACE_SYNC:
 442                 //DRM_INFO("R600_IT_SURFACE_SYNC\n");
 443                 if (num_dw != 5)
 444                         ret = -EINVAL;
 445                 /* 0xffffffff/0x0 is flush all cache flag */
 446                 else if ((ib_chunk->kdata[offset_dw + 2] == 0xffffffff) &&
 447                          (ib_chunk->kdata[offset_dw + 3] == 0))
 448                         ret = 0;
 449                 else {
 450                         reloc = ib_chunk->kdata + offset_dw + num_dw;
 451                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 452                         if (ret) {
 453                                 DRM_ERROR("bad SURFACE_SYNC\n");
 454                                 break;
 455                         }
 456                         ib_chunk->kdata[offset_dw + 3] += ((offset >> 8) & 0xffffffff);
 457                 }
 458                 break;
 459         case R600_IT_EVENT_WRITE:
 460                 //DRM_INFO("R600_IT_EVENT_WRITE\n");
 461                 if ((num_dw != 4) && (num_dw != 2))
 462                         ret = -EINVAL;
 463                 if (num_dw > 2) {
 464                         reloc = ib_chunk->kdata + offset_dw + num_dw;
 465                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 466                         if (ret) {
 467                                 DRM_ERROR("bad EVENT_WRITE\n");
 468                                 break;
 469                         }
 470                         ib_chunk->kdata[offset_dw + 2] += (offset & 0xffffffff);
 471                         ib_chunk->kdata[offset_dw + 3] += (upper_32_bits(offset) & 0xff);
 472                 }
 473                 if (ret)
 474                         DRM_ERROR("bad EVENT_WRITE\n");
 475                 break;
 476         case R600_IT_EVENT_WRITE_EOP:
 477                 //DRM_INFO("R600_IT_EVENT_WRITE_EOP\n");
 478                 if (num_dw != 6) {
 479                         ret = -EINVAL;
 480                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
 481                         break;
 482                 }
 483                 reloc = ib_chunk->kdata + offset_dw + num_dw;
 484                 ret = dev_priv->cs.relocate(parser, reloc, &offset);
 485                 if (ret) {
 486                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
 487                         break;
 488                 }
 489                 ib_chunk->kdata[offset_dw + 2] += (offset & 0xffffffff);
 490                 ib_chunk->kdata[offset_dw + 3] += (upper_32_bits(offset) & 0xff);
 491                 break;
 492         case R600_IT_SET_CONFIG_REG:
 493                 //DRM_INFO("R600_IT_SET_CONFIG_REG\n");
 494                 start_reg = (ib_chunk->kdata[offset_dw + 1] << 2) + R600_SET_CONFIG_REG_OFFSET;
 495                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 496                 if ((start_reg < R600_SET_CONFIG_REG_OFFSET) ||
 497                     (start_reg >= R600_SET_CONFIG_REG_END) ||
 498                     (end_reg >= R600_SET_CONFIG_REG_END))
 499                         ret = -EINVAL;
 500                 else {
 501                         for (i = 0; i < (num_dw - 2); i++) {
 502                                 reg = start_reg + (4 * i);
 503                                 switch (reg) {
 504                                 case R600_CP_COHER_BASE:
 505                                         /* use R600_IT_SURFACE_SYNC */
 506                                         ret = -EINVAL;
 507                                         break;
 508                                 default:
 509                                         break;
 510                                 }
 511                                 if (ret)
 512                                         break;
 513                         }
 514                 }
 515                 if (ret)
 516                         DRM_ERROR("bad SET_CONFIG_REG\n");
 517                 break;
 518         case R600_IT_SET_CONTEXT_REG:
 519                 //DRM_INFO("R600_IT_SET_CONTEXT_REG\n");
 520                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 521                 start_reg += R600_SET_CONTEXT_REG_OFFSET;
 522                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 523                 if ((start_reg < R600_SET_CONTEXT_REG_OFFSET) ||
 524                     (start_reg >= R600_SET_CONTEXT_REG_END) ||
 525                     (end_reg >= R600_SET_CONTEXT_REG_END))
 526                         ret = -EINVAL;
 527                 else {
 528                         for (i = 0; i < (num_dw - 2); i++) {
 529                                 reg = start_reg + (4 * i);
 530                                 switch (reg) {
 531                                 case R600_DB_DEPTH_BASE:
 532                                 case R600_CB_COLOR0_BASE:
 533                                 case R600_CB_COLOR1_BASE:
 534                                 case R600_CB_COLOR2_BASE:
 535                                 case R600_CB_COLOR3_BASE:
 536                                 case R600_CB_COLOR4_BASE:
 537                                 case R600_CB_COLOR5_BASE:
 538                                 case R600_CB_COLOR6_BASE:
 539                                 case R600_CB_COLOR7_BASE:
 540                                 case R600_SQ_PGM_START_FS:
 541                                 case R600_SQ_PGM_START_ES:
 542                                 case R600_SQ_PGM_START_VS:
 543                                 case R600_SQ_PGM_START_GS:
 544                                 case R600_SQ_PGM_START_PS:
 545                                         //DRM_INFO("reg: 0x%08x\n", reg);
 546                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 2);
 547                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 548                                         if (ret) {
 549                                                 DRM_ERROR("bad SET_CONTEXT_REG\n");
 550                                                 break;
 551                                         }
 552                                         ib_chunk->kdata[offset_dw + 2 + i] +=
 553                                                 ((offset >> 8) & 0xffffffff);
 554                                         break;
 555                                 case R600_VGT_DMA_BASE:
 556                                 case R600_VGT_DMA_BASE_HI:
 557                                         /* These should be handled by DRAW_INDEX packet 3 */
 558                                 case R600_VGT_STRMOUT_BASE_OFFSET_0:
 559                                 case R600_VGT_STRMOUT_BASE_OFFSET_1:
 560                                 case R600_VGT_STRMOUT_BASE_OFFSET_2:
 561                                 case R600_VGT_STRMOUT_BASE_OFFSET_3:
 562                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_0:
 563                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_1:
 564                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_2:
 565                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_3:
 566                                 case R600_VGT_STRMOUT_BUFFER_BASE_0:
 567                                 case R600_VGT_STRMOUT_BUFFER_BASE_1:
 568                                 case R600_VGT_STRMOUT_BUFFER_BASE_2:
 569                                 case R600_VGT_STRMOUT_BUFFER_BASE_3:
 570                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_0:
 571                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_1:
 572                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_2:
 573                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_3:
 574                                         /* These should be handled by STRMOUT_BUFFER packet 3 */
 575                                         DRM_ERROR("bad context reg: 0x%08x\n", reg);
 576                                         ret = -EINVAL;
 577                                         break;
 578                                 default:
 579                                         break;
 580                                 }
 581                                 if (ret)
 582                                         break;
 583                         }
 584                 }
 585                 if (ret)
 586                         DRM_ERROR("bad SET_CONTEXT_REG\n");
 587                 break;
 588         case R600_IT_SET_RESOURCE:
 589                 //DRM_INFO("R600_IT_SET_RESOURCE\n");
 590                 if ((num_dw - 2) % 7)
 591                         ret = -EINVAL;
 592                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 593                 start_reg += R600_SET_RESOURCE_OFFSET;
 594                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 595                 if ((start_reg < R600_SET_RESOURCE_OFFSET) ||
 596                     (start_reg >= R600_SET_RESOURCE_END) ||
 597                     (end_reg >= R600_SET_RESOURCE_END))
 598                         ret = -EINVAL;
 599                 else {
 600                         for (i = 0; i < ((num_dw - 2) / 7); i++) {
 601                                 switch ((ib_chunk->kdata[offset_dw + (i * 7) + 6 + 2] & 0xc0000000) >> 30) {
 602                                 case R600_SQ_TEX_VTX_INVALID_TEXTURE:
 603                                 case R600_SQ_TEX_VTX_INVALID_BUFFER:
 604                                 default:
 605                                         ret = -EINVAL;
 606                                         break;
 607                                 case R600_SQ_TEX_VTX_VALID_TEXTURE:
 608                                         /* tex base */
 609                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 4);
 610                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 611                                         if (ret)
 612                                                 break;
 613                                         ib_chunk->kdata[offset_dw + (i * 7) + 2 + 2] +=
 614                                                 ((offset >> 8) & 0xffffffff);
 615                                         /* tex mip base */
 616                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 4) + 2;
 617                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 618                                         if (ret)
 619                                                 break;
 620                                         ib_chunk->kdata[offset_dw + (i * 7) + 3 + 2] +=
 621                                                 ((offset >> 8) & 0xffffffff);
 622                                         break;
 623                                 case R600_SQ_TEX_VTX_VALID_BUFFER:
 624                                         /* vtx base */
 625                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 2);
 626                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 627                                         if (ret)
 628                                                 break;
 629                                         ib_chunk->kdata[offset_dw + (i * 7) + 0 + 2] += (offset & 0xffffffff);
 630                                         ib_chunk->kdata[offset_dw + (i * 7) + 2 + 2] += (upper_32_bits(offset) & 0xff);
 631                                         break;
 632                                 }
 633                                 if (ret)
 634                                         break;
 635                         }
 636                 }
 637                 if (ret)
 638                         DRM_ERROR("bad SET_RESOURCE\n");
 639                 break;
 640         case R600_IT_SET_ALU_CONST:
 641                 //DRM_INFO("R600_IT_SET_ALU_CONST\n");
 642                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 643                 start_reg += R600_SET_ALU_CONST_OFFSET;
 644                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 645                 if ((start_reg < R600_SET_ALU_CONST_OFFSET) ||
 646                     (start_reg >= R600_SET_ALU_CONST_END) ||
 647                     (end_reg >= R600_SET_ALU_CONST_END))
 648                         ret = -EINVAL;
 649                 if (ret)
 650                         DRM_ERROR("bad SET_ALU_CONST\n");
 651                 break;
 652         case R600_IT_SET_BOOL_CONST:
 653                 //DRM_INFO("R600_IT_SET_BOOL_CONST\n");
 654                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 655                 start_reg += R600_SET_BOOL_CONST_OFFSET;
 656                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 657                 if ((start_reg < R600_SET_BOOL_CONST_OFFSET) ||
 658                     (start_reg >= R600_SET_BOOL_CONST_END) ||
 659                     (end_reg >= R600_SET_BOOL_CONST_END))
 660                         ret = -EINVAL;
 661                 if (ret)
 662                         DRM_ERROR("bad SET_BOOL_CONST\n");
 663                 break;
 664         case R600_IT_SET_LOOP_CONST:
 665                 //DRM_INFO("R600_IT_SET_LOOP_CONST\n");
 666                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 667                 start_reg += R600_SET_LOOP_CONST_OFFSET;
 668                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 669                 if ((start_reg < R600_SET_LOOP_CONST_OFFSET) ||
 670                     (start_reg >= R600_SET_LOOP_CONST_END) ||
 671                     (end_reg >= R600_SET_LOOP_CONST_END))
 672                         ret = -EINVAL;
 673                 if (ret)
 674                         DRM_ERROR("bad SET_LOOP_CONST\n");
 675                 break;
 676         case R600_IT_SET_CTL_CONST:
 677                 //DRM_INFO("R600_IT_SET_CTL_CONST\n");
 678                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 679                 start_reg += R600_SET_CTL_CONST_OFFSET;
 680                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 681                 if ((start_reg < R600_SET_CTL_CONST_OFFSET) ||
 682                     (start_reg >= R600_SET_CTL_CONST_END) ||
 683                     (end_reg >= R600_SET_CTL_CONST_END))
 684                         ret = -EINVAL;
 685                 if (ret)
 686                         DRM_ERROR("bad SET_CTL_CONST\n");
 687                 break;
 688         case R600_IT_SET_SAMPLER:
 689                 //DRM_INFO("R600_IT_SET_SAMPLER\n");
 690                 if ((num_dw - 2) % 3)
 691                         ret = -EINVAL;
 692                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 693                 start_reg += R600_SET_SAMPLER_OFFSET;
 694                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 695                 if ((start_reg < R600_SET_SAMPLER_OFFSET) ||
 696                     (start_reg >= R600_SET_SAMPLER_END) ||
 697                     (end_reg >= R600_SET_SAMPLER_END))
 698                         ret = -EINVAL;
 699                 if (ret)
 700                         DRM_ERROR("bad SET_SAMPLER\n");
 701                 break;
 702         case R600_IT_SURFACE_BASE_UPDATE:
 703                 //DRM_INFO("R600_IT_SURFACE_BASE_UPDATE\n");
 704                 if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) ||
 705                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600))
 706                         ret = -EINVAL;
 707                 if (num_dw != 2)
 708                         ret = -EINVAL;
 709                 if (ret)
 710                         DRM_ERROR("bad SURFACE_BASE_UPDATE\n");
 711                 break;
 712         case RADEON_CP_NOP:
 713                 //DRM_INFO("NOP: %d\n", ib_chunk->kdata[offset_dw + 1]);
 714                 break;
 715         default:
 716                 DRM_ERROR("invalid packet 3 0x%08x\n", 0xff00);
 717                 ret = -EINVAL;
 718                 break;
 719         }
 720
 721         *offset_dw_p += incr;
 722         return ret;
 723 }
 724
 725 static int r600_cs_parse(struct drm_radeon_cs_parser *parser)
 726 {
 727         volatile int rb;
 728         struct drm_radeon_kernel_chunk *ib_chunk;
 729         /* scan the packet for various things */
 730         int count_dw = 0, size_dw;
 731         int ret = 0;
 732
 733         ib_chunk = &parser->chunks[parser->ib_index];
 734         size_dw = ib_chunk->length_dw;
 735
 736         while (count_dw < size_dw && ret == 0) {
 737                 int hdr = ib_chunk->kdata[count_dw];
 738                 int num_dw = (hdr & RADEON_CP_PACKET_COUNT_MASK) >> 16;
 739
 740                 switch (hdr & RADEON_CP_PACKET_MASK) {
 741                 case RADEON_CP_PACKET0:
 742                         ret = r600_cs_packet0(parser, &count_dw);
 743                         break;
 744                 case RADEON_CP_PACKET1:
 745                         ret = -EINVAL;
 746                         break;
 747                 case RADEON_CP_PACKET2:
 748                         DRM_DEBUG("Packet 2\n");
 749                         num_dw += 1;
 750                         break;
 751                 case RADEON_CP_PACKET3:
 752                         ret = r600_cs_packet3(parser, &count_dw);
 753                         break;
 754                 }
 755
 756                 count_dw += num_dw;
 757         }
 758
 759         if (ret)
 760                 return ret;
 761
 762
 763         /* copy the packet into the IB */
 764         memcpy(parser->ib, ib_chunk->kdata, ib_chunk->length_dw * sizeof(uint32_t));
 765
 766         /* read back last byte to flush WC buffers */
 767         rb = readl(((vm_offset_t)parser->ib + (ib_chunk->length_dw-1) * sizeof(uint32_t)));
 768
 769         return 0;
 770 }
 771
 772 static uint32_t radeon_cs_id_get(struct drm_radeon_private *radeon)
 773 {
 774         /* FIXME: protect with a spinlock */
 775         /* FIXME: check if wrap affect last reported wrap & sequence */
 776         radeon->cs.id_scnt = (radeon->cs.id_scnt + 1) & 0x00FFFFFF;
 777         if (!radeon->cs.id_scnt) {
 778                 /* increment wrap counter */
 779                 radeon->cs.id_wcnt += 0x01000000;
 780                 /* valid sequence counter start at 1 */
 781                 radeon->cs.id_scnt = 1;
 782         }
 783         return (radeon->cs.id_scnt | radeon->cs.id_wcnt);
 784 }
 785
 786 static void r600_cs_id_emit(struct drm_radeon_cs_parser *parser, uint32_t *id)
 787 {
 788         drm_radeon_private_t *dev_priv = parser->dev->dev_private;
 789         RING_LOCALS;
 790
 791         //dev_priv->irq_emitted = radeon_update_breadcrumb(parser->dev);
 792
 793         *id = radeon_cs_id_get(dev_priv);
 794
 795         /* SCRATCH 2 */
 796         BEGIN_RING(3);
 797         R600_CLEAR_AGE(*id);
 798         ADVANCE_RING();
 799         COMMIT_RING();
 800 }
 801
 802 static uint32_t r600_cs_id_last_get(struct drm_device *dev)
 803 {
 804         //drm_radeon_private_t *dev_priv = dev->dev_private;
 805
 806         //return GET_R600_SCRATCH(dev_priv, 2);
 807         return 0;
 808 }
 809
 810 static int r600_ib_get(struct drm_radeon_cs_parser *parser)
 811 {
 812         struct drm_device *dev = parser->dev;
 813         drm_radeon_private_t *dev_priv = dev->dev_private;
 814         struct drm_buf *buf;
 815
 816         buf = radeon_freelist_get(dev);
 817         if (!buf) {
 818                 dev_priv->cs_buf = NULL;
 819                 return -EBUSY;
 820         }
 821         buf->file_priv = parser->file_priv;
 822         dev_priv->cs_buf = buf;
 823         parser->ib = (void *)((vm_offset_t)dev->agp_buffer_map->handle +
 824             buf->offset);
 825
 826         return 0;
 827 }
 828
 829 static void r600_ib_free(struct drm_radeon_cs_parser *parser, int error)
 830 {
 831         struct drm_device *dev = parser->dev;
 832         drm_radeon_private_t *dev_priv = dev->dev_private;
 833         struct drm_buf *buf = dev_priv->cs_buf;
 834
 835         if (buf) {
 836                 if (!error)
 837                         r600_cp_dispatch_indirect(dev, buf, 0,
 838                                                   parser->chunks[parser->ib_index].length_dw * sizeof(uint32_t));
 839                 radeon_cp_discard_buffer(dev, buf);
 840                 COMMIT_RING();
 841         }
 842 }
 843
 844 int r600_cs_init(struct drm_device *dev)
 845 {
 846         drm_radeon_private_t *dev_priv = dev->dev_private;
 847
 848         dev_priv->cs.ib_get = r600_ib_get;
 849         dev_priv->cs.ib_free = r600_ib_free;
 850         dev_priv->cs.id_emit = r600_cs_id_emit;
 851         dev_priv->cs.id_last_get = r600_cs_id_last_get;
 852         dev_priv->cs.parse = r600_cs_parse;
 853         dev_priv->cs.relocate = r600_nomm_relocate;
 854         return 0;
 855 }