sys/dev/drm/radeon_cs.c

   1 /*-
   2  * Copyright 2008 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Jerome Glisse <glisse@freedesktop.org>
  26  * __FBSDID("$FreeBSD: src/sys/dev/drm/radeon_cs.c,v 1.2 2009/09/28 22:41:28 rnoland Exp $");
  27  */
  28
  29 #include "dev/drm/drmP.h"
  30 #include "dev/drm/radeon_drm.h"
  31 #include "dev/drm/radeon_drv.h"
  32
  33 /* regs */
  34 #define AVIVO_D1MODE_VLINE_START_END                           0x6538
  35 #define AVIVO_D2MODE_VLINE_START_END                           0x6d38
  36 #define R600_CP_COHER_BASE                                     0x85f8
  37 #define R600_DB_DEPTH_BASE                                     0x2800c
  38 #define R600_CB_COLOR0_BASE                                    0x28040
  39 #define R600_CB_COLOR1_BASE                                    0x28044
  40 #define R600_CB_COLOR2_BASE                                    0x28048
  41 #define R600_CB_COLOR3_BASE                                    0x2804c
  42 #define R600_CB_COLOR4_BASE                                    0x28050
  43 #define R600_CB_COLOR5_BASE                                    0x28054
  44 #define R600_CB_COLOR6_BASE                                    0x28058
  45 #define R600_CB_COLOR7_BASE                                    0x2805c
  46 #define R600_SQ_PGM_START_FS                                   0x28894
  47 #define R600_SQ_PGM_START_ES                                   0x28880
  48 #define R600_SQ_PGM_START_VS                                   0x28858
  49 #define R600_SQ_PGM_START_GS                                   0x2886c
  50 #define R600_SQ_PGM_START_PS                                   0x28840
  51 #define R600_VGT_DMA_BASE                                      0x287e8
  52 #define R600_VGT_DMA_BASE_HI                                   0x287e4
  53 #define R600_VGT_STRMOUT_BASE_OFFSET_0                         0x28b10
  54 #define R600_VGT_STRMOUT_BASE_OFFSET_1                         0x28b14
  55 #define R600_VGT_STRMOUT_BASE_OFFSET_2                         0x28b18
  56 #define R600_VGT_STRMOUT_BASE_OFFSET_3                         0x28b1c
  57 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_0                      0x28b44
  58 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_1                      0x28b48
  59 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_2                      0x28b4c
  60 #define R600_VGT_STRMOUT_BASE_OFFSET_HI_3                      0x28b50
  61 #define R600_VGT_STRMOUT_BUFFER_BASE_0                         0x28ad8
  62 #define R600_VGT_STRMOUT_BUFFER_BASE_1                         0x28ae8
  63 #define R600_VGT_STRMOUT_BUFFER_BASE_2                         0x28af8
  64 #define R600_VGT_STRMOUT_BUFFER_BASE_3                         0x28b08
  65 #define R600_VGT_STRMOUT_BUFFER_OFFSET_0                       0x28adc
  66 #define R600_VGT_STRMOUT_BUFFER_OFFSET_1                       0x28aec
  67 #define R600_VGT_STRMOUT_BUFFER_OFFSET_2                       0x28afc
  68 #define R600_VGT_STRMOUT_BUFFER_OFFSET_3                       0x28b0c
  69
  70 /* resource type */
  71 #define R600_SQ_TEX_VTX_INVALID_TEXTURE                        0x0
  72 #define R600_SQ_TEX_VTX_INVALID_BUFFER                         0x1
  73 #define R600_SQ_TEX_VTX_VALID_TEXTURE                          0x2
  74 #define R600_SQ_TEX_VTX_VALID_BUFFER                           0x3
  75
  76 /* packet 3 type offsets */
  77 #define R600_SET_CONFIG_REG_OFFSET                             0x00008000
  78 #define R600_SET_CONFIG_REG_END                                0x0000ac00
  79 #define R600_SET_CONTEXT_REG_OFFSET                            0x00028000
  80 #define R600_SET_CONTEXT_REG_END                               0x00029000
  81 #define R600_SET_ALU_CONST_OFFSET                              0x00030000
  82 #define R600_SET_ALU_CONST_END                                 0x00032000
  83 #define R600_SET_RESOURCE_OFFSET                               0x00038000
  84 #define R600_SET_RESOURCE_END                                  0x0003c000
  85 #define R600_SET_SAMPLER_OFFSET                                0x0003c000
  86 #define R600_SET_SAMPLER_END                                   0x0003cff0
  87 #define R600_SET_CTL_CONST_OFFSET                              0x0003cff0
  88 #define R600_SET_CTL_CONST_END                                 0x0003e200
  89 #define R600_SET_LOOP_CONST_OFFSET                             0x0003e200
  90 #define R600_SET_LOOP_CONST_END                                0x0003e380
  91 #define R600_SET_BOOL_CONST_OFFSET                             0x0003e380
  92 #define R600_SET_BOOL_CONST_END                                0x00040000
  93
  94 /* Packet 3 types */
  95 #define R600_IT_INDIRECT_BUFFER_END               0x00001700
  96 #define R600_IT_SET_PREDICATION                   0x00002000
  97 #define R600_IT_REG_RMW                           0x00002100
  98 #define R600_IT_COND_EXEC                         0x00002200
  99 #define R600_IT_PRED_EXEC                         0x00002300
 100 #define R600_IT_START_3D_CMDBUF                   0x00002400
 101 #define R600_IT_DRAW_INDEX_2                      0x00002700
 102 #define R600_IT_CONTEXT_CONTROL                   0x00002800
 103 #define R600_IT_DRAW_INDEX_IMMD_BE                0x00002900
 104 #define R600_IT_INDEX_TYPE                        0x00002A00
 105 #define R600_IT_DRAW_INDEX                        0x00002B00
 106 #define R600_IT_DRAW_INDEX_AUTO                   0x00002D00
 107 #define R600_IT_DRAW_INDEX_IMMD                   0x00002E00
 108 #define R600_IT_NUM_INSTANCES                     0x00002F00
 109 #define R600_IT_STRMOUT_BUFFER_UPDATE             0x00003400
 110 #define R600_IT_INDIRECT_BUFFER_MP                0x00003800
 111 #define R600_IT_MEM_SEMAPHORE                     0x00003900
 112 #define R600_IT_MPEG_INDEX                        0x00003A00
 113 #define R600_IT_WAIT_REG_MEM                      0x00003C00
 114 #define R600_IT_MEM_WRITE                         0x00003D00
 115 #define R600_IT_INDIRECT_BUFFER                   0x00003200
 116 #define R600_IT_CP_INTERRUPT                      0x00004000
 117 #define R600_IT_SURFACE_SYNC                      0x00004300
 118 #define R600_IT_ME_INITIALIZE                     0x00004400
 119 #define R600_IT_COND_WRITE                        0x00004500
 120 #define R600_IT_EVENT_WRITE                       0x00004600
 121 #define R600_IT_EVENT_WRITE_EOP                   0x00004700
 122 #define R600_IT_ONE_REG_WRITE                     0x00005700
 123 #define R600_IT_SET_CONFIG_REG                    0x00006800
 124 #define R600_IT_SET_CONTEXT_REG                   0x00006900
 125 #define R600_IT_SET_ALU_CONST                     0x00006A00
 126 #define R600_IT_SET_BOOL_CONST                    0x00006B00
 127 #define R600_IT_SET_LOOP_CONST                    0x00006C00
 128 #define R600_IT_SET_RESOURCE                      0x00006D00
 129 #define R600_IT_SET_SAMPLER                       0x00006E00
 130 #define R600_IT_SET_CTL_CONST                     0x00006F00
 131 #define R600_IT_SURFACE_BASE_UPDATE               0x00007300
 132
 133 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *fpriv)
 134 {
 135         struct drm_radeon_cs_parser parser;
 136         struct drm_radeon_private *dev_priv = dev->dev_private;
 137         struct drm_radeon_cs *cs = data;
 138         uint32_t cs_id;
 139         struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
 140         uint64_t *chunk_array;
 141         uint64_t *chunk_array_ptr;
 142         long size;
 143         int r, i;
 144
 145         if (dev_priv == NULL) {
 146                 DRM_ERROR("called with no initialization\n");
 147                 return -EINVAL;
 148         }
 149         DRM_SPINLOCK(&dev_priv->cs.cs_mutex);
 150         /* set command stream id to 0 which is fake id */
 151         cs_id = 0;
 152         cs->cs_id = cs_id;
 153
 154         if (!cs->num_chunks) {
 155                 DRM_SPINUNLOCK(&dev_priv->cs.cs_mutex);
 156                 return 0;
 157         }
 158
 159
 160         chunk_array = drm_calloc(cs->num_chunks, sizeof(uint64_t), DRM_MEM_DRIVER);
 161         if (!chunk_array) {
 162                 DRM_SPINUNLOCK(&dev_priv->cs.cs_mutex);
 163                 return -ENOMEM;
 164         }
 165
 166         chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
 167
 168         if (DRM_COPY_FROM_USER(chunk_array, chunk_array_ptr, sizeof(uint64_t)*cs->num_chunks)) {
 169                 r = -EFAULT;
 170                 goto out;
 171         }
 172
 173         parser.dev = dev;
 174         parser.file_priv = fpriv;
 175         parser.reloc_index = -1;
 176         parser.ib_index = -1;
 177         parser.num_chunks = cs->num_chunks;
 178         /* copy out the chunk headers */
 179         parser.chunks = drm_calloc(parser.num_chunks, sizeof(struct drm_radeon_kernel_chunk), DRM_MEM_DRIVER);
 180         if (!parser.chunks) {
 181                 r = -ENOMEM;
 182                 goto out;
 183         }
 184
 185         for (i = 0; i < parser.num_chunks; i++) {
 186                 struct drm_radeon_cs_chunk user_chunk;
 187
 188                 chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
 189
 190                 if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr, sizeof(struct drm_radeon_cs_chunk))){
 191                         r = -EFAULT;
 192                         goto out;
 193                 }
 194                 parser.chunks[i].chunk_id = user_chunk.chunk_id;
 195
 196                 if (parser.chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS)
 197                         parser.reloc_index = i;
 198
 199                 if (parser.chunks[i].chunk_id == RADEON_CHUNK_ID_IB)
 200                         parser.ib_index = i;
 201
 202                 if (parser.chunks[i].chunk_id == RADEON_CHUNK_ID_OLD) {
 203                         parser.ib_index = i;
 204                         parser.reloc_index = -1;
 205                 }
 206
 207                 parser.chunks[i].length_dw = user_chunk.length_dw;
 208                 parser.chunks[i].chunk_data = (uint32_t *)(unsigned long)user_chunk.chunk_data;
 209
 210                 parser.chunks[i].kdata = NULL;
 211                 size = parser.chunks[i].length_dw * sizeof(uint32_t);
 212
 213                 switch(parser.chunks[i].chunk_id) {
 214                 case RADEON_CHUNK_ID_IB:
 215                 case RADEON_CHUNK_ID_OLD:
 216                         if (size == 0) {
 217                                 r = -EINVAL;
 218                                 goto out;
 219                         }
 220                 case RADEON_CHUNK_ID_RELOCS:
 221                         if (size) {
 222                                 parser.chunks[i].kdata = drm_alloc(size, DRM_MEM_DRIVER);
 223                                 if (!parser.chunks[i].kdata) {
 224                                         r = -ENOMEM;
 225                                         goto out;
 226                                 }
 227
 228                                 if (DRM_COPY_FROM_USER(parser.chunks[i].kdata, parser.chunks[i].chunk_data, size)) {
 229                                         r = -EFAULT;
 230                                         goto out;
 231                                 }
 232                         } else
 233                                 parser.chunks[i].kdata = NULL;
 234                         break;
 235                 default:
 236                         break;
 237                 }
 238                 DRM_DEBUG("chunk %d %d %d %p\n", i, parser.chunks[i].chunk_id, parser.chunks[i].length_dw,
 239                           parser.chunks[i].chunk_data);
 240         }
 241
 242         if (parser.chunks[parser.ib_index].length_dw > (16 * 1024)) {
 243                 DRM_ERROR("cs->dwords too big: %d\n", parser.chunks[parser.ib_index].length_dw);
 244                 r = -EINVAL;
 245                 goto out;
 246         }
 247
 248         /* get ib */
 249         r = dev_priv->cs.ib_get(&parser);
 250         if (r) {
 251                 DRM_ERROR("ib_get failed\n");
 252                 goto out;
 253         }
 254
 255         /* now parse command stream */
 256         r = dev_priv->cs.parse(&parser);
 257         if (r) {
 258                 goto out;
 259         }
 260
 261 out:
 262         dev_priv->cs.ib_free(&parser, r);
 263
 264         /* emit cs id sequence */
 265         dev_priv->cs.id_emit(&parser, &cs_id);
 266
 267         cs->cs_id = cs_id;
 268
 269         DRM_SPINUNLOCK(&dev_priv->cs.cs_mutex);
 270
 271         for (i = 0; i < parser.num_chunks; i++) {
 272                 if (parser.chunks[i].kdata)
 273                         drm_free(parser.chunks[i].kdata, parser.chunks[i].length_dw * sizeof(uint32_t), DRM_MEM_DRIVER);
 274         }
 275
 276         drm_free(parser.chunks, sizeof(struct drm_radeon_kernel_chunk)*parser.num_chunks, DRM_MEM_DRIVER);
 277         drm_free(chunk_array, sizeof(uint64_t)*parser.num_chunks, DRM_MEM_DRIVER);
 278
 279         return r;
 280 }
 281
 282 /* for non-mm */
 283 static int r600_nomm_relocate(struct drm_radeon_cs_parser *parser, uint32_t *reloc, uint64_t *offset)
 284 {
 285         struct drm_device *dev = parser->dev;
 286         drm_radeon_private_t *dev_priv = dev->dev_private;
 287         struct drm_radeon_kernel_chunk *reloc_chunk = &parser->chunks[parser->reloc_index];
 288         uint32_t offset_dw = reloc[1];
 289
 290         //DRM_INFO("reloc: 0x%08x 0x%08x\n", reloc[0], reloc[1]);
 291         //DRM_INFO("length: %d\n", reloc_chunk->length_dw);
 292
 293         if (!reloc_chunk->kdata)
 294                 return -EINVAL;
 295
 296         if (offset_dw > reloc_chunk->length_dw) {
 297                 DRM_ERROR("Offset larger than chunk 0x%x %d\n", offset_dw, reloc_chunk->length_dw);
 298                 return -EINVAL;
 299         }
 300
 301         /* 40 bit addr */
 302         *offset = reloc_chunk->kdata[offset_dw + 3];
 303         *offset <<= 32;
 304         *offset |= reloc_chunk->kdata[offset_dw + 0];
 305
 306         //DRM_INFO("offset 0x%lx\n", *offset);
 307
 308         if (!radeon_check_offset(dev_priv, *offset)) {
 309                 DRM_ERROR("bad offset! 0x%lx\n", (unsigned long)*offset);
 310                 return -EINVAL;
 311         }
 312
 313         return 0;
 314 }
 315
 316 static inline int r600_cs_packet0(struct drm_radeon_cs_parser *parser, uint32_t *offset_dw_p)
 317 {
 318         uint32_t hdr, num_dw, reg;
 319         int count_dw = 1;
 320         int ret = 0;
 321         uint32_t offset_dw = *offset_dw_p;
 322         int incr = 2;
 323
 324         hdr = parser->chunks[parser->ib_index].kdata[offset_dw];
 325         num_dw = ((hdr & RADEON_CP_PACKET_COUNT_MASK) >> 16) + 2;
 326         reg = (hdr & 0xffff) << 2;
 327
 328         while (count_dw < num_dw) {
 329                 switch (reg) {
 330                 case AVIVO_D1MODE_VLINE_START_END:
 331                 case AVIVO_D2MODE_VLINE_START_END:
 332                         break;
 333                 default:
 334                         ret = -EINVAL;
 335                         DRM_ERROR("bad packet 0 reg: 0x%08x\n", reg);
 336                         break;
 337                 }
 338                 if (ret)
 339                         break;
 340                 count_dw++;
 341                 reg += 4;
 342         }
 343         *offset_dw_p += incr;
 344         return ret;
 345 }
 346
 347 static inline int r600_cs_packet3(struct drm_radeon_cs_parser *parser, uint32_t *offset_dw_p)
 348 {
 349         struct drm_device *dev = parser->dev;
 350         drm_radeon_private_t *dev_priv = dev->dev_private;
 351         uint32_t hdr, num_dw, start_reg, end_reg, reg;
 352         uint32_t *reloc;
 353         uint64_t offset;
 354         int ret = 0;
 355         uint32_t offset_dw = *offset_dw_p;
 356         int incr = 2;
 357         int i;
 358         struct drm_radeon_kernel_chunk *ib_chunk;
 359
 360         ib_chunk = &parser->chunks[parser->ib_index];
 361
 362         hdr = ib_chunk->kdata[offset_dw];
 363         num_dw = ((hdr & RADEON_CP_PACKET_COUNT_MASK) >> 16) + 2;
 364
 365         /* just the ones we use for now, add more later */
 366         switch (hdr & 0xff00) {
 367         case R600_IT_START_3D_CMDBUF:
 368                 //DRM_INFO("R600_IT_START_3D_CMDBUF\n");
 369                 if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770)
 370                         ret = -EINVAL;
 371                 if (num_dw != 2)
 372                         ret = -EINVAL;
 373                 if (ret)
 374                         DRM_ERROR("bad START_3D\n");
 375                 break;
 376         case R600_IT_CONTEXT_CONTROL:
 377                 //DRM_INFO("R600_IT_CONTEXT_CONTROL\n");
 378                 if (num_dw != 3)
 379                         ret = -EINVAL;
 380                 if (ret)
 381                         DRM_ERROR("bad CONTEXT_CONTROL\n");
 382                 break;
 383         case R600_IT_INDEX_TYPE:
 384         case R600_IT_NUM_INSTANCES:
 385                 //DRM_INFO("R600_IT_INDEX_TYPE/R600_IT_NUM_INSTANCES\n");
 386                 if (num_dw != 2)
 387                         ret = -EINVAL;
 388                 if (ret)
 389                         DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES\n");
 390                 break;
 391         case R600_IT_DRAW_INDEX:
 392                 //DRM_INFO("R600_IT_DRAW_INDEX\n");
 393                 if (num_dw != 5) {
 394                         ret = -EINVAL;
 395                         DRM_ERROR("bad DRAW_INDEX\n");
 396                         break;
 397                 }
 398                 reloc = ib_chunk->kdata + offset_dw + num_dw;
 399                 ret = dev_priv->cs.relocate(parser, reloc, &offset);
 400                 if (ret) {
 401                         DRM_ERROR("bad DRAW_INDEX\n");
 402                         break;
 403                 }
 404                 ib_chunk->kdata[offset_dw + 1] += (offset & 0xffffffff);
 405                 ib_chunk->kdata[offset_dw + 2] += (upper_32_bits(offset) & 0xff);
 406                 break;
 407         case R600_IT_DRAW_INDEX_AUTO:
 408                 //DRM_INFO("R600_IT_DRAW_INDEX_AUTO\n");
 409                 if (num_dw != 3)
 410                         ret = -EINVAL;
 411                 if (ret)
 412                         DRM_ERROR("bad DRAW_INDEX_AUTO\n");
 413                 break;
 414         case R600_IT_DRAW_INDEX_IMMD_BE:
 415         case R600_IT_DRAW_INDEX_IMMD:
 416                 //DRM_INFO("R600_IT_DRAW_INDEX_IMMD\n");
 417                 if (num_dw < 4)
 418                         ret = -EINVAL;
 419                 if (ret)
 420                         DRM_ERROR("bad DRAW_INDEX_IMMD\n");
 421                 break;
 422         case R600_IT_WAIT_REG_MEM:
 423                 //DRM_INFO("R600_IT_WAIT_REG_MEM\n");
 424                 if (num_dw != 7)
 425                         ret = -EINVAL;
 426                 /* bit 4 is reg (0) or mem (1) */
 427                 if (ib_chunk->kdata[offset_dw + 1] & 0x10) {
 428                         reloc = ib_chunk->kdata + offset_dw + num_dw;
 429                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 430                         if (ret) {
 431                                 DRM_ERROR("bad WAIT_REG_MEM\n");
 432                                 break;
 433                         }
 434                         ib_chunk->kdata[offset_dw + 2] += (offset & 0xffffffff);
 435                         ib_chunk->kdata[offset_dw + 3] += (upper_32_bits(offset) & 0xff);
 436                 }
 437                 if (ret)
 438                         DRM_ERROR("bad WAIT_REG_MEM\n");
 439                 break;
 440         case R600_IT_SURFACE_SYNC:
 441                 //DRM_INFO("R600_IT_SURFACE_SYNC\n");
 442                 if (num_dw != 5)
 443                         ret = -EINVAL;
 444                 /* 0xffffffff/0x0 is flush all cache flag */
 445                 else if ((ib_chunk->kdata[offset_dw + 2] == 0xffffffff) &&
 446                          (ib_chunk->kdata[offset_dw + 3] == 0))
 447                         ret = 0;
 448                 else {
 449                         reloc = ib_chunk->kdata + offset_dw + num_dw;
 450                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 451                         if (ret) {
 452                                 DRM_ERROR("bad SURFACE_SYNC\n");
 453                                 break;
 454                         }
 455                         ib_chunk->kdata[offset_dw + 3] += ((offset >> 8) & 0xffffffff);
 456                 }
 457                 break;
 458         case R600_IT_EVENT_WRITE:
 459                 //DRM_INFO("R600_IT_EVENT_WRITE\n");
 460                 if ((num_dw != 4) && (num_dw != 2))
 461                         ret = -EINVAL;
 462                 if (num_dw > 2) {
 463                         reloc = ib_chunk->kdata + offset_dw + num_dw;
 464                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 465                         if (ret) {
 466                                 DRM_ERROR("bad EVENT_WRITE\n");
 467                                 break;
 468                         }
 469                         ib_chunk->kdata[offset_dw + 2] += (offset & 0xffffffff);
 470                         ib_chunk->kdata[offset_dw + 3] += (upper_32_bits(offset) & 0xff);
 471                 }
 472                 if (ret)
 473                         DRM_ERROR("bad EVENT_WRITE\n");
 474                 break;
 475         case R600_IT_EVENT_WRITE_EOP:
 476                 //DRM_INFO("R600_IT_EVENT_WRITE_EOP\n");
 477                 if (num_dw != 6) {
 478                         ret = -EINVAL;
 479                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
 480                         break;
 481                 }
 482                 reloc = ib_chunk->kdata + offset_dw + num_dw;
 483                 ret = dev_priv->cs.relocate(parser, reloc, &offset);
 484                 if (ret) {
 485                         DRM_ERROR("bad EVENT_WRITE_EOP\n");
 486                         break;
 487                 }
 488                 ib_chunk->kdata[offset_dw + 2] += (offset & 0xffffffff);
 489                 ib_chunk->kdata[offset_dw + 3] += (upper_32_bits(offset) & 0xff);
 490                 break;
 491         case R600_IT_SET_CONFIG_REG:
 492                 //DRM_INFO("R600_IT_SET_CONFIG_REG\n");
 493                 start_reg = (ib_chunk->kdata[offset_dw + 1] << 2) + R600_SET_CONFIG_REG_OFFSET;
 494                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 495                 if ((start_reg < R600_SET_CONFIG_REG_OFFSET) ||
 496                     (start_reg >= R600_SET_CONFIG_REG_END) ||
 497                     (end_reg >= R600_SET_CONFIG_REG_END))
 498                         ret = -EINVAL;
 499                 else {
 500                         for (i = 0; i < (num_dw - 2); i++) {
 501                                 reg = start_reg + (4 * i);
 502                                 switch (reg) {
 503                                 case R600_CP_COHER_BASE:
 504                                         /* use R600_IT_SURFACE_SYNC */
 505                                         ret = -EINVAL;
 506                                         break;
 507                                 default:
 508                                         break;
 509                                 }
 510                                 if (ret)
 511                                         break;
 512                         }
 513                 }
 514                 if (ret)
 515                         DRM_ERROR("bad SET_CONFIG_REG\n");
 516                 break;
 517         case R600_IT_SET_CONTEXT_REG:
 518                 //DRM_INFO("R600_IT_SET_CONTEXT_REG\n");
 519                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 520                 start_reg += R600_SET_CONTEXT_REG_OFFSET;
 521                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 522                 if ((start_reg < R600_SET_CONTEXT_REG_OFFSET) ||
 523                     (start_reg >= R600_SET_CONTEXT_REG_END) ||
 524                     (end_reg >= R600_SET_CONTEXT_REG_END))
 525                         ret = -EINVAL;
 526                 else {
 527                         for (i = 0; i < (num_dw - 2); i++) {
 528                                 reg = start_reg + (4 * i);
 529                                 switch (reg) {
 530                                 case R600_DB_DEPTH_BASE:
 531                                 case R600_CB_COLOR0_BASE:
 532                                 case R600_CB_COLOR1_BASE:
 533                                 case R600_CB_COLOR2_BASE:
 534                                 case R600_CB_COLOR3_BASE:
 535                                 case R600_CB_COLOR4_BASE:
 536                                 case R600_CB_COLOR5_BASE:
 537                                 case R600_CB_COLOR6_BASE:
 538                                 case R600_CB_COLOR7_BASE:
 539                                 case R600_SQ_PGM_START_FS:
 540                                 case R600_SQ_PGM_START_ES:
 541                                 case R600_SQ_PGM_START_VS:
 542                                 case R600_SQ_PGM_START_GS:
 543                                 case R600_SQ_PGM_START_PS:
 544                                         //DRM_INFO("reg: 0x%08x\n", reg);
 545                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 2);
 546                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 547                                         if (ret) {
 548                                                 DRM_ERROR("bad SET_CONTEXT_REG\n");
 549                                                 break;
 550                                         }
 551                                         ib_chunk->kdata[offset_dw + 2 + i] +=
 552                                                 ((offset >> 8) & 0xffffffff);
 553                                         break;
 554                                 case R600_VGT_DMA_BASE:
 555                                 case R600_VGT_DMA_BASE_HI:
 556                                         /* These should be handled by DRAW_INDEX packet 3 */
 557                                 case R600_VGT_STRMOUT_BASE_OFFSET_0:
 558                                 case R600_VGT_STRMOUT_BASE_OFFSET_1:
 559                                 case R600_VGT_STRMOUT_BASE_OFFSET_2:
 560                                 case R600_VGT_STRMOUT_BASE_OFFSET_3:
 561                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_0:
 562                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_1:
 563                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_2:
 564                                 case R600_VGT_STRMOUT_BASE_OFFSET_HI_3:
 565                                 case R600_VGT_STRMOUT_BUFFER_BASE_0:
 566                                 case R600_VGT_STRMOUT_BUFFER_BASE_1:
 567                                 case R600_VGT_STRMOUT_BUFFER_BASE_2:
 568                                 case R600_VGT_STRMOUT_BUFFER_BASE_3:
 569                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_0:
 570                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_1:
 571                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_2:
 572                                 case R600_VGT_STRMOUT_BUFFER_OFFSET_3:
 573                                         /* These should be handled by STRMOUT_BUFFER packet 3 */
 574                                         DRM_ERROR("bad context reg: 0x%08x\n", reg);
 575                                         ret = -EINVAL;
 576                                         break;
 577                                 default:
 578                                         break;
 579                                 }
 580                                 if (ret)
 581                                         break;
 582                         }
 583                 }
 584                 if (ret)
 585                         DRM_ERROR("bad SET_CONTEXT_REG\n");
 586                 break;
 587         case R600_IT_SET_RESOURCE:
 588                 //DRM_INFO("R600_IT_SET_RESOURCE\n");
 589                 if ((num_dw - 2) % 7)
 590                         ret = -EINVAL;
 591                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 592                 start_reg += R600_SET_RESOURCE_OFFSET;
 593                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 594                 if ((start_reg < R600_SET_RESOURCE_OFFSET) ||
 595                     (start_reg >= R600_SET_RESOURCE_END) ||
 596                     (end_reg >= R600_SET_RESOURCE_END))
 597                         ret = -EINVAL;
 598                 else {
 599                         for (i = 0; i < ((num_dw - 2) / 7); i++) {
 600                                 switch ((ib_chunk->kdata[offset_dw + (i * 7) + 6 + 2] & 0xc0000000) >> 30) {
 601                                 case R600_SQ_TEX_VTX_INVALID_TEXTURE:
 602                                 case R600_SQ_TEX_VTX_INVALID_BUFFER:
 603                                 default:
 604                                         ret = -EINVAL;
 605                                         break;
 606                                 case R600_SQ_TEX_VTX_VALID_TEXTURE:
 607                                         /* tex base */
 608                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 4);
 609                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 610                                         if (ret)
 611                                                 break;
 612                                         ib_chunk->kdata[offset_dw + (i * 7) + 2 + 2] +=
 613                                                 ((offset >> 8) & 0xffffffff);
 614                                         /* tex mip base */
 615                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 4) + 2;
 616                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 617                                         if (ret)
 618                                                 break;
 619                                         ib_chunk->kdata[offset_dw + (i * 7) + 3 + 2] +=
 620                                                 ((offset >> 8) & 0xffffffff);
 621                                         break;
 622                                 case R600_SQ_TEX_VTX_VALID_BUFFER:
 623                                         /* vtx base */
 624                                         reloc = ib_chunk->kdata + offset_dw + num_dw + (i * 2);
 625                                         ret = dev_priv->cs.relocate(parser, reloc, &offset);
 626                                         if (ret)
 627                                                 break;
 628                                         ib_chunk->kdata[offset_dw + (i * 7) + 0 + 2] += (offset & 0xffffffff);
 629                                         ib_chunk->kdata[offset_dw + (i * 7) + 2 + 2] += (upper_32_bits(offset) & 0xff);
 630                                         break;
 631                                 }
 632                                 if (ret)
 633                                         break;
 634                         }
 635                 }
 636                 if (ret)
 637                         DRM_ERROR("bad SET_RESOURCE\n");
 638                 break;
 639         case R600_IT_SET_ALU_CONST:
 640                 //DRM_INFO("R600_IT_SET_ALU_CONST\n");
 641                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 642                 start_reg += R600_SET_ALU_CONST_OFFSET;
 643                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 644                 if ((start_reg < R600_SET_ALU_CONST_OFFSET) ||
 645                     (start_reg >= R600_SET_ALU_CONST_END) ||
 646                     (end_reg >= R600_SET_ALU_CONST_END))
 647                         ret = -EINVAL;
 648                 if (ret)
 649                         DRM_ERROR("bad SET_ALU_CONST\n");
 650                 break;
 651         case R600_IT_SET_BOOL_CONST:
 652                 //DRM_INFO("R600_IT_SET_BOOL_CONST\n");
 653                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 654                 start_reg += R600_SET_BOOL_CONST_OFFSET;
 655                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 656                 if ((start_reg < R600_SET_BOOL_CONST_OFFSET) ||
 657                     (start_reg >= R600_SET_BOOL_CONST_END) ||
 658                     (end_reg >= R600_SET_BOOL_CONST_END))
 659                         ret = -EINVAL;
 660                 if (ret)
 661                         DRM_ERROR("bad SET_BOOL_CONST\n");
 662                 break;
 663         case R600_IT_SET_LOOP_CONST:
 664                 //DRM_INFO("R600_IT_SET_LOOP_CONST\n");
 665                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 666                 start_reg += R600_SET_LOOP_CONST_OFFSET;
 667                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 668                 if ((start_reg < R600_SET_LOOP_CONST_OFFSET) ||
 669                     (start_reg >= R600_SET_LOOP_CONST_END) ||
 670                     (end_reg >= R600_SET_LOOP_CONST_END))
 671                         ret = -EINVAL;
 672                 if (ret)
 673                         DRM_ERROR("bad SET_LOOP_CONST\n");
 674                 break;
 675         case R600_IT_SET_CTL_CONST:
 676                 //DRM_INFO("R600_IT_SET_CTL_CONST\n");
 677                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 678                 start_reg += R600_SET_CTL_CONST_OFFSET;
 679                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 680                 if ((start_reg < R600_SET_CTL_CONST_OFFSET) ||
 681                     (start_reg >= R600_SET_CTL_CONST_END) ||
 682                     (end_reg >= R600_SET_CTL_CONST_END))
 683                         ret = -EINVAL;
 684                 if (ret)
 685                         DRM_ERROR("bad SET_CTL_CONST\n");
 686                 break;
 687         case R600_IT_SET_SAMPLER:
 688                 //DRM_INFO("R600_IT_SET_SAMPLER\n");
 689                 if ((num_dw - 2) % 3)
 690                         ret = -EINVAL;
 691                 start_reg = ib_chunk->kdata[offset_dw + 1] << 2;
 692                 start_reg += R600_SET_SAMPLER_OFFSET;
 693                 end_reg = 4 * (num_dw - 2) + start_reg - 4;
 694                 if ((start_reg < R600_SET_SAMPLER_OFFSET) ||
 695                     (start_reg >= R600_SET_SAMPLER_END) ||
 696                     (end_reg >= R600_SET_SAMPLER_END))
 697                         ret = -EINVAL;
 698                 if (ret)
 699                         DRM_ERROR("bad SET_SAMPLER\n");
 700                 break;
 701         case R600_IT_SURFACE_BASE_UPDATE:
 702                 //DRM_INFO("R600_IT_SURFACE_BASE_UPDATE\n");
 703                 if (((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV770) ||
 704                     ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_R600))
 705                         ret = -EINVAL;
 706                 if (num_dw != 2)
 707                         ret = -EINVAL;
 708                 if (ret)
 709                         DRM_ERROR("bad SURFACE_BASE_UPDATE\n");
 710                 break;
 711         case RADEON_CP_NOP:
 712                 //DRM_INFO("NOP: %d\n", ib_chunk->kdata[offset_dw + 1]);
 713                 break;
 714         default:
 715                 DRM_ERROR("invalid packet 3 0x%08x\n", 0xff00);
 716                 ret = -EINVAL;
 717                 break;
 718         }
 719
 720         *offset_dw_p += incr;
 721         return ret;
 722 }
 723
 724 static int r600_cs_parse(struct drm_radeon_cs_parser *parser)
 725 {
 726         volatile int rb;
 727         struct drm_radeon_kernel_chunk *ib_chunk;
 728         /* scan the packet for various things */
 729         int count_dw = 0, size_dw;
 730         int ret = 0;
 731
 732         ib_chunk = &parser->chunks[parser->ib_index];
 733         size_dw = ib_chunk->length_dw;
 734
 735         while (count_dw < size_dw && ret == 0) {
 736                 int hdr = ib_chunk->kdata[count_dw];
 737                 int num_dw = (hdr & RADEON_CP_PACKET_COUNT_MASK) >> 16;
 738
 739                 switch (hdr & RADEON_CP_PACKET_MASK) {
 740                 case RADEON_CP_PACKET0:
 741                         ret = r600_cs_packet0(parser, &count_dw);
 742                         break;
 743                 case RADEON_CP_PACKET1:
 744                         ret = -EINVAL;
 745                         break;
 746                 case RADEON_CP_PACKET2:
 747                         DRM_DEBUG("Packet 2\n");
 748                         num_dw += 1;
 749                         break;
 750                 case RADEON_CP_PACKET3:
 751                         ret = r600_cs_packet3(parser, &count_dw);
 752                         break;
 753                 }
 754
 755                 count_dw += num_dw;
 756         }
 757
 758         if (ret)
 759                 return ret;
 760
 761
 762         /* copy the packet into the IB */
 763         memcpy(parser->ib, ib_chunk->kdata, ib_chunk->length_dw * sizeof(uint32_t));
 764
 765         /* read back last byte to flush WC buffers */
 766         rb = readl(((vm_offset_t)parser->ib + (ib_chunk->length_dw-1) * sizeof(uint32_t)));
 767
 768         return 0;
 769 }
 770
 771 static uint32_t radeon_cs_id_get(struct drm_radeon_private *radeon)
 772 {
 773         /* FIXME: protect with a spinlock */
 774         /* FIXME: check if wrap affect last reported wrap & sequence */
 775         radeon->cs.id_scnt = (radeon->cs.id_scnt + 1) & 0x00FFFFFF;
 776         if (!radeon->cs.id_scnt) {
 777                 /* increment wrap counter */
 778                 radeon->cs.id_wcnt += 0x01000000;
 779                 /* valid sequence counter start at 1 */
 780                 radeon->cs.id_scnt = 1;
 781         }
 782         return (radeon->cs.id_scnt | radeon->cs.id_wcnt);
 783 }
 784
 785 static void r600_cs_id_emit(struct drm_radeon_cs_parser *parser, uint32_t *id)
 786 {
 787         drm_radeon_private_t *dev_priv = parser->dev->dev_private;
 788         RING_LOCALS;
 789
 790         //dev_priv->irq_emitted = radeon_update_breadcrumb(parser->dev);
 791
 792         *id = radeon_cs_id_get(dev_priv);
 793
 794         /* SCRATCH 2 */
 795         BEGIN_RING(3);
 796         R600_CLEAR_AGE(*id);
 797         ADVANCE_RING();
 798         COMMIT_RING();
 799 }
 800
 801 static uint32_t r600_cs_id_last_get(struct drm_device *dev)
 802 {
 803         //drm_radeon_private_t *dev_priv = dev->dev_private;
 804
 805         //return GET_R600_SCRATCH(dev_priv, 2);
 806         return 0;
 807 }
 808
 809 static int r600_ib_get(struct drm_radeon_cs_parser *parser)
 810 {
 811         struct drm_device *dev = parser->dev;
 812         drm_radeon_private_t *dev_priv = dev->dev_private;
 813         struct drm_buf *buf;
 814
 815         buf = radeon_freelist_get(dev);
 816         if (!buf) {
 817                 dev_priv->cs_buf = NULL;
 818                 return -EBUSY;
 819         }
 820         buf->file_priv = parser->file_priv;
 821         dev_priv->cs_buf = buf;
 822         parser->ib = (void *)((vm_offset_t)dev->agp_buffer_map->handle +
 823             buf->offset);
 824
 825         return 0;
 826 }
 827
 828 static void r600_ib_free(struct drm_radeon_cs_parser *parser, int error)
 829 {
 830         struct drm_device *dev = parser->dev;
 831         drm_radeon_private_t *dev_priv = dev->dev_private;
 832         struct drm_buf *buf = dev_priv->cs_buf;
 833
 834         if (buf) {
 835                 if (!error)
 836                         r600_cp_dispatch_indirect(dev, buf, 0,
 837                                                   parser->chunks[parser->ib_index].length_dw * sizeof(uint32_t));
 838                 radeon_cp_discard_buffer(dev, buf);
 839                 COMMIT_RING();
 840         }
 841 }
 842
 843 int r600_cs_init(struct drm_device *dev)
 844 {
 845         drm_radeon_private_t *dev_priv = dev->dev_private;
 846
 847         dev_priv->cs.ib_get = r600_ib_get;
 848         dev_priv->cs.ib_free = r600_ib_free;
 849         dev_priv->cs.id_emit = r600_cs_id_emit;
 850         dev_priv->cs.id_last_get = r600_cs_id_last_get;
 851         dev_priv->cs.parse = r600_cs_parse;
 852         dev_priv->cs.relocate = r600_nomm_relocate;
 853         return 0;
 854 }