sys/dev/drm/radeon/r300_cmdbuf.c

   1 /* r300_cmdbuf.c -- Command buffer emission for R300 -*- linux-c -*-
   2  *
   3  * Copyright (C) The Weather Channel, Inc.  2002.
   4  * Copyright (C) 2004 Nicolai Haehnle.
   5  * All Rights Reserved.
   6  *
   7  * The Weather Channel (TM) funded Tungsten Graphics to develop the
   8  * initial release of the Radeon 8500 driver under the XFree86 license.
   9  * This notice must be preserved.
  10  *
  11  * Permission is hereby granted, free of charge, to any person obtaining a
  12  * copy of this software and associated documentation files (the "Software"),
  13  * to deal in the Software without restriction, including without limitation
  14  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  15  * and/or sell copies of the Software, and to permit persons to whom the
  16  * Software is furnished to do so, subject to the following conditions:
  17  *
  18  * The above copyright notice and this permission notice (including the next
  19  * paragraph) shall be included in all copies or substantial portions of the
  20  * Software.
  21  *
  22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  23  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  25  * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  26  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  27  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  28  * DEALINGS IN THE SOFTWARE.
  29  *
  30  * Authors:
  31  *    Nicolai Haehnle <prefect_@gmx.net>
  32  */
  33
  34 #include "dev/drm/drmP.h"
  35 #include "dev/drm/drm.h"
  36 #include "dev/drm/radeon_drm.h"
  37 #include "radeon_drv.h"
  38 #include "r300_reg.h"
  39
  40 #define R300_SIMULTANEOUS_CLIPRECTS             4
  41
  42 /* Values for R300_RE_CLIPRECT_CNTL depending on the number of cliprects
  43  */
  44 static const int r300_cliprect_cntl[4] = {
  45         0xAAAA,
  46         0xEEEE,
  47         0xFEFE,
  48         0xFFFE
  49 };
  50
  51 /**
  52  * Emit up to R300_SIMULTANEOUS_CLIPRECTS cliprects from the given command
  53  * buffer, starting with index n.
  54  */
  55 static int r300_emit_cliprects(drm_radeon_private_t *dev_priv,
  56                                drm_radeon_kcmd_buffer_t *cmdbuf, int n)
  57 {
  58         struct drm_clip_rect box;
  59         int nr;
  60         int i;
  61         RING_LOCALS;
  62
  63         nr = cmdbuf->nbox - n;
  64         if (nr > R300_SIMULTANEOUS_CLIPRECTS)
  65                 nr = R300_SIMULTANEOUS_CLIPRECTS;
  66
  67         DRM_DEBUG("%i cliprects\n", nr);
  68
  69         if (nr) {
  70                 BEGIN_RING(6 + nr * 2);
  71                 OUT_RING(CP_PACKET0(R300_RE_CLIPRECT_TL_0, nr * 2 - 1));
  72
  73                 for (i = 0; i < nr; ++i) {
  74                         if (DRM_COPY_FROM_USER_UNCHECKED
  75                             (&box, &cmdbuf->boxes[n + i], sizeof(box))) {
  76                                 DRM_ERROR("copy cliprect faulted\n");
  77                                 return -EFAULT;
  78                         }
  79
  80                         box.x2--; /* Hardware expects inclusive bottom-right corner */
  81                         box.y2--;
  82
  83                         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
  84                                 box.x1 = (box.x1) &
  85                                         R300_CLIPRECT_MASK;
  86                                 box.y1 = (box.y1) &
  87                                         R300_CLIPRECT_MASK;
  88                                 box.x2 = (box.x2) &
  89                                         R300_CLIPRECT_MASK;
  90                                 box.y2 = (box.y2) &
  91                                         R300_CLIPRECT_MASK;
  92                         } else {
  93                                 box.x1 = (box.x1 + R300_CLIPRECT_OFFSET) &
  94                                         R300_CLIPRECT_MASK;
  95                                 box.y1 = (box.y1 + R300_CLIPRECT_OFFSET) &
  96                                         R300_CLIPRECT_MASK;
  97                                 box.x2 = (box.x2 + R300_CLIPRECT_OFFSET) &
  98                                         R300_CLIPRECT_MASK;
  99                                 box.y2 = (box.y2 + R300_CLIPRECT_OFFSET) &
 100                                         R300_CLIPRECT_MASK;
 101                         }
 102
 103                         OUT_RING((box.x1 << R300_CLIPRECT_X_SHIFT) |
 104                                  (box.y1 << R300_CLIPRECT_Y_SHIFT));
 105                         OUT_RING((box.x2 << R300_CLIPRECT_X_SHIFT) |
 106                                  (box.y2 << R300_CLIPRECT_Y_SHIFT));
 107
 108                 }
 109
 110                 OUT_RING_REG(R300_RE_CLIPRECT_CNTL, r300_cliprect_cntl[nr - 1]);
 111
 112                 /* TODO/SECURITY: Force scissors to a safe value, otherwise the
 113                  * client might be able to trample over memory.
 114                  * The impact should be very limited, but I'd rather be safe than
 115                  * sorry.
 116                  */
 117                 OUT_RING(CP_PACKET0(R300_RE_SCISSORS_TL, 1));
 118                 OUT_RING(0);
 119                 OUT_RING(R300_SCISSORS_X_MASK | R300_SCISSORS_Y_MASK);
 120                 ADVANCE_RING();
 121         } else {
 122                 /* Why we allow zero cliprect rendering:
 123                  * There are some commands in a command buffer that must be submitted
 124                  * even when there are no cliprects, e.g. DMA buffer discard
 125                  * or state setting (though state setting could be avoided by
 126                  * simulating a loss of context).
 127                  *
 128                  * Now since the cmdbuf interface is so chaotic right now (and is
 129                  * bound to remain that way for a bit until things settle down),
 130                  * it is basically impossible to filter out the commands that are
 131                  * necessary and those that aren't.
 132                  *
 133                  * So I choose the safe way and don't do any filtering at all;
 134                  * instead, I simply set up the engine so that all rendering
 135                  * can't produce any fragments.
 136                  */
 137                 BEGIN_RING(2);
 138                 OUT_RING_REG(R300_RE_CLIPRECT_CNTL, 0);
 139                 ADVANCE_RING();
 140         }
 141
 142         /* flus cache and wait idle clean after cliprect change */
 143         BEGIN_RING(2);
 144         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 145         OUT_RING(R300_RB3D_DC_FLUSH);
 146         ADVANCE_RING();
 147         BEGIN_RING(2);
 148         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 149         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 150         ADVANCE_RING();
 151         /* set flush flag */
 152         dev_priv->track_flush |= RADEON_FLUSH_EMITED;
 153
 154         return 0;
 155 }
 156
 157 static u8 r300_reg_flags[0x10000 >> 2];
 158
 159 void r300_init_reg_flags(struct drm_device *dev)
 160 {
 161         int i;
 162         drm_radeon_private_t *dev_priv = dev->dev_private;
 163
 164         memset(r300_reg_flags, 0, 0x10000 >> 2);
 165 #define ADD_RANGE_MARK(reg, count,mark) \
 166                 for(i=((reg)>>2);i<((reg)>>2)+(count);i++)\
 167                         r300_reg_flags[i]|=(mark);
 168
 169 #define MARK_SAFE               1
 170 #define MARK_CHECK_OFFSET       2
 171
 172 #define ADD_RANGE(reg, count)   ADD_RANGE_MARK(reg, count, MARK_SAFE)
 173
 174         /* these match cmducs() command in r300_driver/r300/r300_cmdbuf.c */
 175         ADD_RANGE(R300_SE_VPORT_XSCALE, 6);
 176         ADD_RANGE(R300_VAP_CNTL, 1);
 177         ADD_RANGE(R300_SE_VTE_CNTL, 2);
 178         ADD_RANGE(0x2134, 2);
 179         ADD_RANGE(R300_VAP_CNTL_STATUS, 1);
 180         ADD_RANGE(R300_VAP_INPUT_CNTL_0, 2);
 181         ADD_RANGE(0x21DC, 1);
 182         ADD_RANGE(R300_VAP_UNKNOWN_221C, 1);
 183         ADD_RANGE(R300_VAP_CLIP_X_0, 4);
 184         ADD_RANGE(R300_VAP_PVS_STATE_FLUSH_REG, 1);
 185         ADD_RANGE(R300_VAP_UNKNOWN_2288, 1);
 186         ADD_RANGE(R300_VAP_OUTPUT_VTX_FMT_0, 2);
 187         ADD_RANGE(R300_VAP_PVS_CNTL_1, 3);
 188         ADD_RANGE(R300_GB_ENABLE, 1);
 189         ADD_RANGE(R300_GB_MSPOS0, 5);
 190         ADD_RANGE(R300_TX_INVALTAGS, 1);
 191         ADD_RANGE(R300_TX_ENABLE, 1);
 192         ADD_RANGE(0x4200, 4);
 193         ADD_RANGE(0x4214, 1);
 194         ADD_RANGE(R300_RE_POINTSIZE, 1);
 195         ADD_RANGE(0x4230, 3);
 196         ADD_RANGE(R300_RE_LINE_CNT, 1);
 197         ADD_RANGE(R300_RE_UNK4238, 1);
 198         ADD_RANGE(0x4260, 3);
 199         ADD_RANGE(R300_RE_SHADE, 4);
 200         ADD_RANGE(R300_RE_POLYGON_MODE, 5);
 201         ADD_RANGE(R300_RE_ZBIAS_CNTL, 1);
 202         ADD_RANGE(R300_RE_ZBIAS_T_FACTOR, 4);
 203         ADD_RANGE(R300_RE_OCCLUSION_CNTL, 1);
 204         ADD_RANGE(R300_RE_CULL_CNTL, 1);
 205         ADD_RANGE(0x42C0, 2);
 206         ADD_RANGE(R300_RS_CNTL_0, 2);
 207
 208         ADD_RANGE(R300_SU_REG_DEST, 1);
 209         if ((dev_priv->flags & RADEON_FAMILY_MASK) == CHIP_RV530)
 210                 ADD_RANGE(RV530_FG_ZBREG_DEST, 1);
 211
 212         ADD_RANGE(R300_SC_HYPERZ, 2);
 213         ADD_RANGE(0x43E8, 1);
 214
 215         ADD_RANGE(0x46A4, 5);
 216
 217         ADD_RANGE(R300_RE_FOG_STATE, 1);
 218         ADD_RANGE(R300_FOG_COLOR_R, 3);
 219         ADD_RANGE(R300_PP_ALPHA_TEST, 2);
 220         ADD_RANGE(0x4BD8, 1);
 221         ADD_RANGE(R300_PFS_PARAM_0_X, 64);
 222         ADD_RANGE(0x4E00, 1);
 223         ADD_RANGE(R300_RB3D_CBLEND, 2);
 224         ADD_RANGE(R300_RB3D_COLORMASK, 1);
 225         ADD_RANGE(R300_RB3D_BLEND_COLOR, 3);
 226         ADD_RANGE_MARK(R300_RB3D_COLOROFFSET0, 1, MARK_CHECK_OFFSET);   /* check offset */
 227         ADD_RANGE(R300_RB3D_COLORPITCH0, 1);
 228         ADD_RANGE(0x4E50, 9);
 229         ADD_RANGE(0x4E88, 1);
 230         ADD_RANGE(0x4EA0, 2);
 231         ADD_RANGE(R300_ZB_CNTL, 3);
 232         ADD_RANGE(R300_ZB_FORMAT, 4);
 233         ADD_RANGE_MARK(R300_ZB_DEPTHOFFSET, 1, MARK_CHECK_OFFSET);      /* check offset */
 234         ADD_RANGE(R300_ZB_DEPTHPITCH, 1);
 235         ADD_RANGE(R300_ZB_DEPTHCLEARVALUE, 1);
 236         ADD_RANGE(R300_ZB_ZMASK_OFFSET, 5);
 237         ADD_RANGE(R300_ZB_HIZ_OFFSET, 5);
 238         ADD_RANGE(R300_ZB_ZPASS_DATA, 1);
 239         ADD_RANGE_MARK(R300_ZB_ZPASS_ADDR, 1, MARK_CHECK_OFFSET);       /* check offset */
 240         ADD_RANGE(R300_ZB_DEPTHXY_OFFSET, 1)
 241
 242         ADD_RANGE(R300_TX_FILTER_0, 16);
 243         ADD_RANGE(R300_TX_FILTER1_0, 16);
 244         ADD_RANGE(R300_TX_SIZE_0, 16);
 245         ADD_RANGE(R300_TX_FORMAT_0, 16);
 246         ADD_RANGE(R300_TX_PITCH_0, 16);
 247         /* Texture offset is dangerous and needs more checking */
 248         ADD_RANGE_MARK(R300_TX_OFFSET_0, 16, MARK_CHECK_OFFSET);
 249         ADD_RANGE(R300_TX_CHROMA_KEY_0, 16);
 250         ADD_RANGE(R300_TX_BORDER_COLOR_0, 16);
 251
 252         /* Sporadic registers used as primitives are emitted */
 253         ADD_RANGE(R300_ZB_ZCACHE_CTLSTAT, 1);
 254         ADD_RANGE(R300_RB3D_DSTCACHE_CTLSTAT, 1);
 255         ADD_RANGE(R300_VAP_INPUT_ROUTE_0_0, 8);
 256         ADD_RANGE(R300_VAP_INPUT_ROUTE_1_0, 8);
 257
 258         if ((dev_priv->flags & RADEON_FAMILY_MASK) >= CHIP_RV515) {
 259                 ADD_RANGE(R500_VAP_INDEX_OFFSET, 1);
 260                 ADD_RANGE(R500_US_CONFIG, 2);
 261                 ADD_RANGE(R500_US_CODE_ADDR, 3);
 262                 ADD_RANGE(R500_US_FC_CTRL, 1);
 263                 ADD_RANGE(R500_RS_IP_0, 16);
 264                 ADD_RANGE(R500_RS_INST_0, 16);
 265                 ADD_RANGE(R500_RB3D_COLOR_CLEAR_VALUE_AR, 2);
 266                 ADD_RANGE(R500_RB3D_CONSTANT_COLOR_AR, 2);
 267                 ADD_RANGE(R500_ZB_FIFO_SIZE, 2);
 268         } else {
 269                 ADD_RANGE(R300_PFS_CNTL_0, 3);
 270                 ADD_RANGE(R300_PFS_NODE_0, 4);
 271                 ADD_RANGE(R300_PFS_TEXI_0, 64);
 272                 ADD_RANGE(R300_PFS_INSTR0_0, 64);
 273                 ADD_RANGE(R300_PFS_INSTR1_0, 64);
 274                 ADD_RANGE(R300_PFS_INSTR2_0, 64);
 275                 ADD_RANGE(R300_PFS_INSTR3_0, 64);
 276                 ADD_RANGE(R300_RS_INTERP_0, 8);
 277                 ADD_RANGE(R300_RS_ROUTE_0, 8);
 278
 279         }
 280 }
 281
 282 static __inline__ int r300_check_range(unsigned reg, int count)
 283 {
 284         int i;
 285         if (reg & ~0xffff)
 286                 return -1;
 287         for (i = (reg >> 2); i < (reg >> 2) + count; i++)
 288                 if (r300_reg_flags[i] != MARK_SAFE)
 289                         return 1;
 290         return 0;
 291 }
 292
 293 static __inline__ int r300_emit_carefully_checked_packet0(drm_radeon_private_t *
 294                                                           dev_priv,
 295                                                           drm_radeon_kcmd_buffer_t
 296                                                           * cmdbuf,
 297                                                           drm_r300_cmd_header_t
 298                                                           header)
 299 {
 300         int reg;
 301         int sz;
 302         int i;
 303         int values[64];
 304         RING_LOCALS;
 305
 306         sz = header.packet0.count;
 307         reg = (header.packet0.reghi << 8) | header.packet0.reglo;
 308
 309         if ((sz > 64) || (sz < 0)) {
 310                 DRM_ERROR
 311                     ("Cannot emit more than 64 values at a time (reg=%04x sz=%d)\n",
 312                      reg, sz);
 313                 return -EINVAL;
 314         }
 315         for (i = 0; i < sz; i++) {
 316                 values[i] = ((int *)cmdbuf->buf)[i];
 317                 switch (r300_reg_flags[(reg >> 2) + i]) {
 318                 case MARK_SAFE:
 319                         break;
 320                 case MARK_CHECK_OFFSET:
 321                         if (!radeon_check_offset(dev_priv, (u32) values[i])) {
 322                                 DRM_ERROR
 323                                     ("Offset failed range check (reg=%04x sz=%d)\n",
 324                                      reg, sz);
 325                                 return -EINVAL;
 326                         }
 327                         break;
 328                 default:
 329                         DRM_ERROR("Register %04x failed check as flag=%02x\n",
 330                                   reg + i * 4, r300_reg_flags[(reg >> 2) + i]);
 331                         return -EINVAL;
 332                 }
 333         }
 334
 335         BEGIN_RING(1 + sz);
 336         OUT_RING(CP_PACKET0(reg, sz - 1));
 337         OUT_RING_TABLE(values, sz);
 338         ADVANCE_RING();
 339
 340         cmdbuf->buf += sz * 4;
 341         cmdbuf->bufsz -= sz * 4;
 342
 343         return 0;
 344 }
 345
 346 /**
 347  * Emits a packet0 setting arbitrary registers.
 348  * Called by r300_do_cp_cmdbuf.
 349  *
 350  * Note that checks are performed on contents and addresses of the registers
 351  */
 352 static __inline__ int r300_emit_packet0(drm_radeon_private_t *dev_priv,
 353                                         drm_radeon_kcmd_buffer_t *cmdbuf,
 354                                         drm_r300_cmd_header_t header)
 355 {
 356         int reg;
 357         int sz;
 358         RING_LOCALS;
 359
 360         sz = header.packet0.count;
 361         reg = (header.packet0.reghi << 8) | header.packet0.reglo;
 362
 363         DRM_DEBUG("R300_CMD_PACKET0: reg %04x, sz %d\n", reg, sz);
 364         if (!sz)
 365                 return 0;
 366
 367         if (sz * 4 > cmdbuf->bufsz)
 368                 return -EINVAL;
 369
 370         if (reg + sz * 4 >= 0x10000) {
 371                 DRM_ERROR("No such registers in hardware reg=%04x sz=%d\n", reg,
 372                           sz);
 373                 return -EINVAL;
 374         }
 375
 376         if (r300_check_range(reg, sz)) {
 377                 /* go and check everything */
 378                 return r300_emit_carefully_checked_packet0(dev_priv, cmdbuf,
 379                                                            header);
 380         }
 381         /* the rest of the data is safe to emit, whatever the values the user passed */
 382
 383         BEGIN_RING(1 + sz);
 384         OUT_RING(CP_PACKET0(reg, sz - 1));
 385         OUT_RING_TABLE((int *)cmdbuf->buf, sz);
 386         ADVANCE_RING();
 387
 388         cmdbuf->buf += sz * 4;
 389         cmdbuf->bufsz -= sz * 4;
 390
 391         return 0;
 392 }
 393
 394 /**
 395  * Uploads user-supplied vertex program instructions or parameters onto
 396  * the graphics card.
 397  * Called by r300_do_cp_cmdbuf.
 398  */
 399 static __inline__ int r300_emit_vpu(drm_radeon_private_t *dev_priv,
 400                                     drm_radeon_kcmd_buffer_t *cmdbuf,
 401                                     drm_r300_cmd_header_t header)
 402 {
 403         int sz;
 404         int addr;
 405         RING_LOCALS;
 406
 407         sz = header.vpu.count;
 408         addr = (header.vpu.adrhi << 8) | header.vpu.adrlo;
 409
 410         if (!sz)
 411                 return 0;
 412         if (sz * 16 > cmdbuf->bufsz)
 413                 return -EINVAL;
 414
 415         /* VAP is very sensitive so we purge cache before we program it
 416          * and we also flush its state before & after */
 417         BEGIN_RING(6);
 418         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 419         OUT_RING(R300_RB3D_DC_FLUSH);
 420         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 421         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 422         OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
 423         OUT_RING(0);
 424         ADVANCE_RING();
 425         /* set flush flag */
 426         dev_priv->track_flush |= RADEON_FLUSH_EMITED;
 427
 428         BEGIN_RING(3 + sz * 4);
 429         OUT_RING_REG(R300_VAP_PVS_UPLOAD_ADDRESS, addr);
 430         OUT_RING(CP_PACKET0_TABLE(R300_VAP_PVS_UPLOAD_DATA, sz * 4 - 1));
 431         OUT_RING_TABLE((int *)cmdbuf->buf, sz * 4);
 432         ADVANCE_RING();
 433
 434         BEGIN_RING(2);
 435         OUT_RING(CP_PACKET0(R300_VAP_PVS_STATE_FLUSH_REG, 0));
 436         OUT_RING(0);
 437         ADVANCE_RING();
 438
 439         cmdbuf->buf += sz * 16;
 440         cmdbuf->bufsz -= sz * 16;
 441
 442         return 0;
 443 }
 444
 445 /**
 446  * Emit a clear packet from userspace.
 447  * Called by r300_emit_packet3.
 448  */
 449 static __inline__ int r300_emit_clear(drm_radeon_private_t *dev_priv,
 450                                       drm_radeon_kcmd_buffer_t *cmdbuf)
 451 {
 452         RING_LOCALS;
 453
 454         if (8 * 4 > cmdbuf->bufsz)
 455                 return -EINVAL;
 456
 457         BEGIN_RING(10);
 458         OUT_RING(CP_PACKET3(R200_3D_DRAW_IMMD_2, 8));
 459         OUT_RING(R300_PRIM_TYPE_POINT | R300_PRIM_WALK_RING |
 460                  (1 << R300_PRIM_NUM_VERTICES_SHIFT));
 461         OUT_RING_TABLE((int *)cmdbuf->buf, 8);
 462         ADVANCE_RING();
 463
 464         BEGIN_RING(4);
 465         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 466         OUT_RING(R300_RB3D_DC_FLUSH);
 467         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 468         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 469         ADVANCE_RING();
 470         /* set flush flag */
 471         dev_priv->track_flush |= RADEON_FLUSH_EMITED;
 472
 473         cmdbuf->buf += 8 * 4;
 474         cmdbuf->bufsz -= 8 * 4;
 475
 476         return 0;
 477 }
 478
 479 static __inline__ int r300_emit_3d_load_vbpntr(drm_radeon_private_t *dev_priv,
 480                                                drm_radeon_kcmd_buffer_t *cmdbuf,
 481                                                u32 header)
 482 {
 483         int count, i, k;
 484 #define MAX_ARRAY_PACKET  64
 485         u32 payload[MAX_ARRAY_PACKET];
 486         u32 narrays;
 487         RING_LOCALS;
 488
 489         count = (header >> 16) & 0x3fff;
 490
 491         if ((count + 1) > MAX_ARRAY_PACKET) {
 492                 DRM_ERROR("Too large payload in 3D_LOAD_VBPNTR (count=%d)\n",
 493                           count);
 494                 return -EINVAL;
 495         }
 496         memset(payload, 0, MAX_ARRAY_PACKET * 4);
 497         memcpy(payload, cmdbuf->buf + 4, (count + 1) * 4);
 498
 499         /* carefully check packet contents */
 500
 501         narrays = payload[0];
 502         k = 0;
 503         i = 1;
 504         while ((k < narrays) && (i < (count + 1))) {
 505                 i++;            /* skip attribute field */
 506                 if (!radeon_check_offset(dev_priv, payload[i])) {
 507                         DRM_ERROR
 508                             ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
 509                              k, i);
 510                         return -EINVAL;
 511                 }
 512                 k++;
 513                 i++;
 514                 if (k == narrays)
 515                         break;
 516                 /* have one more to process, they come in pairs */
 517                 if (!radeon_check_offset(dev_priv, payload[i])) {
 518                         DRM_ERROR
 519                             ("Offset failed range check (k=%d i=%d) while processing 3D_LOAD_VBPNTR packet.\n",
 520                              k, i);
 521                         return -EINVAL;
 522                 }
 523                 k++;
 524                 i++;
 525         }
 526         /* do the counts match what we expect ? */
 527         if ((k != narrays) || (i != (count + 1))) {
 528                 DRM_ERROR
 529                     ("Malformed 3D_LOAD_VBPNTR packet (k=%d i=%d narrays=%d count+1=%d).\n",
 530                      k, i, narrays, count + 1);
 531                 return -EINVAL;
 532         }
 533
 534         /* all clear, output packet */
 535
 536         BEGIN_RING(count + 2);
 537         OUT_RING(header);
 538         OUT_RING_TABLE(payload, count + 1);
 539         ADVANCE_RING();
 540
 541         cmdbuf->buf += (count + 2) * 4;
 542         cmdbuf->bufsz -= (count + 2) * 4;
 543
 544         return 0;
 545 }
 546
 547 static __inline__ int r300_emit_bitblt_multi(drm_radeon_private_t *dev_priv,
 548                                              drm_radeon_kcmd_buffer_t *cmdbuf)
 549 {
 550         u32 *cmd = (u32 *) cmdbuf->buf;
 551         int count, ret;
 552         RING_LOCALS;
 553
 554         count=(cmd[0]>>16) & 0x3fff;
 555
 556         if (cmd[0] & 0x8000) {
 557                 u32 offset;
 558
 559                 if (cmd[1] & (RADEON_GMC_SRC_PITCH_OFFSET_CNTL
 560                               | RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 561                         offset = cmd[2] << 10;
 562                         ret = !radeon_check_offset(dev_priv, offset);
 563                         if (ret) {
 564                                 DRM_ERROR("Invalid bitblt first offset is %08X\n", offset);
 565                                 return -EINVAL;
 566                         }
 567                 }
 568
 569                 if ((cmd[1] & RADEON_GMC_SRC_PITCH_OFFSET_CNTL) &&
 570                     (cmd[1] & RADEON_GMC_DST_PITCH_OFFSET_CNTL)) {
 571                         offset = cmd[3] << 10;
 572                         ret = !radeon_check_offset(dev_priv, offset);
 573                         if (ret) {
 574                                 DRM_ERROR("Invalid bitblt second offset is %08X\n", offset);
 575                                 return -EINVAL;
 576                         }
 577
 578                 }
 579         }
 580
 581         BEGIN_RING(count+2);
 582         OUT_RING(cmd[0]);
 583         OUT_RING_TABLE((int *)(cmdbuf->buf + 4), count + 1);
 584         ADVANCE_RING();
 585
 586         cmdbuf->buf += (count+2)*4;
 587         cmdbuf->bufsz -= (count+2)*4;
 588
 589         return 0;
 590 }
 591
 592 static __inline__ int r300_emit_draw_indx_2(drm_radeon_private_t *dev_priv,
 593                                             drm_radeon_kcmd_buffer_t *cmdbuf)
 594 {
 595         u32 *cmd;
 596         int count;
 597         int expected_count;
 598         RING_LOCALS;
 599
 600         cmd = (u32 *) cmdbuf->buf;
 601         count = (cmd[0]>>16) & 0x3fff;
 602         expected_count = cmd[1] >> 16;
 603         if (!(cmd[1] & R300_VAP_VF_CNTL__INDEX_SIZE_32bit))
 604                 expected_count = (expected_count+1)/2;
 605
 606         if (count && count != expected_count) {
 607                 DRM_ERROR("3D_DRAW_INDX_2: packet size %i, expected %i\n",
 608                         count, expected_count);
 609                 return -EINVAL;
 610         }
 611
 612         BEGIN_RING(count+2);
 613         OUT_RING(cmd[0]);
 614         OUT_RING_TABLE((int *)(cmdbuf->buf + 4), count + 1);
 615         ADVANCE_RING();
 616
 617         cmdbuf->buf += (count+2)*4;
 618         cmdbuf->bufsz -= (count+2)*4;
 619
 620         if (!count) {
 621                 drm_r300_cmd_header_t header;
 622
 623                 if (cmdbuf->bufsz < 4*4 + sizeof(header)) {
 624                         DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER, but stream is too short.\n");
 625                         return -EINVAL;
 626                 }
 627
 628                 header.u = *(unsigned int *)cmdbuf->buf;
 629
 630                 cmdbuf->buf += sizeof(header);
 631                 cmdbuf->bufsz -= sizeof(header);
 632                 cmd = (u32 *) cmdbuf->buf;
 633
 634                 if (header.header.cmd_type != R300_CMD_PACKET3 ||
 635                     header.packet3.packet != R300_CMD_PACKET3_RAW ||
 636                     cmd[0] != CP_PACKET3(RADEON_CP_INDX_BUFFER, 2)) {
 637                         DRM_ERROR("3D_DRAW_INDX_2: expect subsequent INDX_BUFFER.\n");
 638                         return -EINVAL;
 639                 }
 640
 641                 if ((cmd[1] & 0x8000ffff) != 0x80000810) {
 642                         DRM_ERROR("Invalid indx_buffer reg address %08X\n", cmd[1]);
 643                         return -EINVAL;
 644                 }
 645                 if (!radeon_check_offset(dev_priv, cmd[2])) {
 646                         DRM_ERROR("Invalid indx_buffer offset is %08X\n", cmd[2]);
 647                         return -EINVAL;
 648                 }
 649                 if (cmd[3] != expected_count) {
 650                         DRM_ERROR("INDX_BUFFER: buffer size %i, expected %i\n",
 651                                 cmd[3], expected_count);
 652                         return -EINVAL;
 653                 }
 654
 655                 BEGIN_RING(4);
 656                 OUT_RING(cmd[0]);
 657                 OUT_RING_TABLE((int *)(cmdbuf->buf + 4), 3);
 658                 ADVANCE_RING();
 659
 660                 cmdbuf->buf += 4*4;
 661                 cmdbuf->bufsz -= 4*4;
 662         }
 663
 664         return 0;
 665 }
 666
 667 static __inline__ int r300_emit_raw_packet3(drm_radeon_private_t *dev_priv,
 668                                             drm_radeon_kcmd_buffer_t *cmdbuf)
 669 {
 670         u32 header;
 671         int count;
 672         RING_LOCALS;
 673
 674         if (4 > cmdbuf->bufsz)
 675                 return -EINVAL;
 676
 677         /* Fixme !! This simply emits a packet without much checking.
 678            We need to be smarter. */
 679
 680         /* obtain first word - actual packet3 header */
 681         header = *(u32 *) cmdbuf->buf;
 682
 683         /* Is it packet 3 ? */
 684         if ((header >> 30) != 0x3) {
 685                 DRM_ERROR("Not a packet3 header (0x%08x)\n", header);
 686                 return -EINVAL;
 687         }
 688
 689         count = (header >> 16) & 0x3fff;
 690
 691         /* Check again now that we know how much data to expect */
 692         if ((count + 2) * 4 > cmdbuf->bufsz) {
 693                 DRM_ERROR
 694                     ("Expected packet3 of length %d but have only %d bytes left\n",
 695                      (count + 2) * 4, cmdbuf->bufsz);
 696                 return -EINVAL;
 697         }
 698
 699         /* Is it a packet type we know about ? */
 700         switch (header & 0xff00) {
 701         case RADEON_3D_LOAD_VBPNTR:     /* load vertex array pointers */
 702                 return r300_emit_3d_load_vbpntr(dev_priv, cmdbuf, header);
 703
 704         case RADEON_CNTL_BITBLT_MULTI:
 705                 return r300_emit_bitblt_multi(dev_priv, cmdbuf);
 706
 707         case RADEON_CP_INDX_BUFFER:
 708                 DRM_ERROR("packet3 INDX_BUFFER without preceding 3D_DRAW_INDX_2 is illegal.\n");
 709                 return -EINVAL;
 710         case RADEON_CP_3D_DRAW_IMMD_2:
 711                 /* triggers drawing using in-packet vertex data */
 712         case RADEON_CP_3D_DRAW_VBUF_2:
 713                 /* triggers drawing of vertex buffers setup elsewhere */
 714                 dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
 715                                            RADEON_PURGE_EMITED);
 716                 break;
 717         case RADEON_CP_3D_DRAW_INDX_2:
 718                 /* triggers drawing using indices to vertex buffer */
 719                 /* whenever we send vertex we clear flush & purge */
 720                 dev_priv->track_flush &= ~(RADEON_FLUSH_EMITED |
 721                                            RADEON_PURGE_EMITED);
 722                 return r300_emit_draw_indx_2(dev_priv, cmdbuf);
 723         case RADEON_WAIT_FOR_IDLE:
 724         case RADEON_CP_NOP:
 725                 /* these packets are safe */
 726                 break;
 727         default:
 728                 DRM_ERROR("Unknown packet3 header (0x%08x)\n", header);
 729                 return -EINVAL;
 730         }
 731
 732         BEGIN_RING(count + 2);
 733         OUT_RING(header);
 734         OUT_RING_TABLE((int *)(cmdbuf->buf + 4), count + 1);
 735         ADVANCE_RING();
 736
 737         cmdbuf->buf += (count + 2) * 4;
 738         cmdbuf->bufsz -= (count + 2) * 4;
 739
 740         return 0;
 741 }
 742
 743 /**
 744  * Emit a rendering packet3 from userspace.
 745  * Called by r300_do_cp_cmdbuf.
 746  */
 747 static __inline__ int r300_emit_packet3(drm_radeon_private_t *dev_priv,
 748                                         drm_radeon_kcmd_buffer_t *cmdbuf,
 749                                         drm_r300_cmd_header_t header)
 750 {
 751         int n;
 752         int ret;
 753         char *orig_buf = cmdbuf->buf;
 754         int orig_bufsz = cmdbuf->bufsz;
 755
 756         /* This is a do-while-loop so that we run the interior at least once,
 757          * even if cmdbuf->nbox is 0. Compare r300_emit_cliprects for rationale.
 758          */
 759         n = 0;
 760         do {
 761                 if (cmdbuf->nbox > R300_SIMULTANEOUS_CLIPRECTS) {
 762                         ret = r300_emit_cliprects(dev_priv, cmdbuf, n);
 763                         if (ret)
 764                                 return ret;
 765
 766                         cmdbuf->buf = orig_buf;
 767                         cmdbuf->bufsz = orig_bufsz;
 768                 }
 769
 770                 switch (header.packet3.packet) {
 771                 case R300_CMD_PACKET3_CLEAR:
 772                         DRM_DEBUG("R300_CMD_PACKET3_CLEAR\n");
 773                         ret = r300_emit_clear(dev_priv, cmdbuf);
 774                         if (ret) {
 775                                 DRM_ERROR("r300_emit_clear failed\n");
 776                                 return ret;
 777                         }
 778                         break;
 779
 780                 case R300_CMD_PACKET3_RAW:
 781                         DRM_DEBUG("R300_CMD_PACKET3_RAW\n");
 782                         ret = r300_emit_raw_packet3(dev_priv, cmdbuf);
 783                         if (ret) {
 784                                 DRM_ERROR("r300_emit_raw_packet3 failed\n");
 785                                 return ret;
 786                         }
 787                         break;
 788
 789                 default:
 790                         DRM_ERROR("bad packet3 type %i at %p\n",
 791                                   header.packet3.packet,
 792                                   cmdbuf->buf - sizeof(header));
 793                         return -EINVAL;
 794                 }
 795
 796                 n += R300_SIMULTANEOUS_CLIPRECTS;
 797         } while (n < cmdbuf->nbox);
 798
 799         return 0;
 800 }
 801
 802 /* Some of the R300 chips seem to be extremely touchy about the two registers
 803  * that are configured in r300_pacify.
 804  * Among the worst offenders seems to be the R300 ND (0x4E44): When userspace
 805  * sends a command buffer that contains only state setting commands and a
 806  * vertex program/parameter upload sequence, this will eventually lead to a
 807  * lockup, unless the sequence is bracketed by calls to r300_pacify.
 808  * So we should take great care to *always* call r300_pacify before
 809  * *anything* 3D related, and again afterwards. This is what the
 810  * call bracket in r300_do_cp_cmdbuf is for.
 811  */
 812
 813 /**
 814  * Emit the sequence to pacify R300.
 815  */
 816 static __inline__ void r300_pacify(drm_radeon_private_t *dev_priv)
 817 {
 818         uint32_t cache_z, cache_3d, cache_2d;
 819         RING_LOCALS;
 820
 821         cache_z = R300_ZC_FLUSH;
 822         cache_2d = R300_RB2D_DC_FLUSH;
 823         cache_3d = R300_RB3D_DC_FLUSH;
 824         if (!(dev_priv->track_flush & RADEON_PURGE_EMITED)) {
 825                 /* we can purge, primitive where draw since last purge */
 826                 cache_z |= R300_ZC_FREE;
 827                 cache_2d |= R300_RB2D_DC_FREE;
 828                 cache_3d |= R300_RB3D_DC_FREE;
 829         }
 830
 831         /* flush & purge zbuffer */
 832         BEGIN_RING(2);
 833         OUT_RING(CP_PACKET0(R300_ZB_ZCACHE_CTLSTAT, 0));
 834         OUT_RING(cache_z);
 835         ADVANCE_RING();
 836         /* flush & purge 3d */
 837         BEGIN_RING(2);
 838         OUT_RING(CP_PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
 839         OUT_RING(cache_3d);
 840         ADVANCE_RING();
 841         /* flush & purge texture */
 842         BEGIN_RING(2);
 843         OUT_RING(CP_PACKET0(R300_TX_INVALTAGS, 0));
 844         OUT_RING(0);
 845         ADVANCE_RING();
 846         /* FIXME: is this one really needed ? */
 847         BEGIN_RING(2);
 848         OUT_RING(CP_PACKET0(R300_RB3D_AARESOLVE_CTL, 0));
 849         OUT_RING(0);
 850         ADVANCE_RING();
 851         BEGIN_RING(2);
 852         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 853         OUT_RING(RADEON_WAIT_3D_IDLECLEAN);
 854         ADVANCE_RING();
 855         /* flush & purge 2d through E2 as RB2D will trigger lockup */
 856         BEGIN_RING(4);
 857         OUT_RING(CP_PACKET0(R300_DSTCACHE_CTLSTAT, 0));
 858         OUT_RING(cache_2d);
 859         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 860         OUT_RING(RADEON_WAIT_2D_IDLECLEAN |
 861                  RADEON_WAIT_HOST_IDLECLEAN);
 862         ADVANCE_RING();
 863         /* set flush & purge flags */
 864         dev_priv->track_flush |= RADEON_FLUSH_EMITED | RADEON_PURGE_EMITED;
 865 }
 866
 867 /**
 868  * Called by r300_do_cp_cmdbuf to update the internal buffer age and state.
 869  * The actual age emit is done by r300_do_cp_cmdbuf, which is why you must
 870  * be careful about how this function is called.
 871  */
 872 static void r300_discard_buffer(struct drm_device * dev, struct drm_buf * buf)
 873 {
 874         drm_radeon_private_t *dev_priv = dev->dev_private;
 875         drm_radeon_buf_priv_t *buf_priv = buf->dev_private;
 876
 877         buf_priv->age = ++dev_priv->sarea_priv->last_dispatch;
 878         buf->pending = 1;
 879         buf->used = 0;
 880 }
 881
 882 static void r300_cmd_wait(drm_radeon_private_t * dev_priv,
 883                           drm_r300_cmd_header_t header)
 884 {
 885         u32 wait_until;
 886         RING_LOCALS;
 887
 888         if (!header.wait.flags)
 889                 return;
 890
 891         wait_until = 0;
 892
 893         switch(header.wait.flags) {
 894         case R300_WAIT_2D:
 895                 wait_until = RADEON_WAIT_2D_IDLE;
 896                 break;
 897         case R300_WAIT_3D:
 898                 wait_until = RADEON_WAIT_3D_IDLE;
 899                 break;
 900         case R300_NEW_WAIT_2D_3D:
 901                 wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_3D_IDLE;
 902                 break;
 903         case R300_NEW_WAIT_2D_2D_CLEAN:
 904                 wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
 905                 break;
 906         case R300_NEW_WAIT_3D_3D_CLEAN:
 907                 wait_until = RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
 908                 break;
 909         case R300_NEW_WAIT_2D_2D_CLEAN_3D_3D_CLEAN:
 910                 wait_until = RADEON_WAIT_2D_IDLE|RADEON_WAIT_2D_IDLECLEAN;
 911                 wait_until |= RADEON_WAIT_3D_IDLE|RADEON_WAIT_3D_IDLECLEAN;
 912                 break;
 913         default:
 914                 return;
 915         }
 916
 917         BEGIN_RING(2);
 918         OUT_RING(CP_PACKET0(RADEON_WAIT_UNTIL, 0));
 919         OUT_RING(wait_until);
 920         ADVANCE_RING();
 921 }
 922
 923 static int r300_scratch(drm_radeon_private_t *dev_priv,
 924                         drm_radeon_kcmd_buffer_t *cmdbuf,
 925                         drm_r300_cmd_header_t header)
 926 {
 927         u32 *ref_age_base;
 928         u32 i, buf_idx, h_pending;
 929         RING_LOCALS;
 930
 931         if (cmdbuf->bufsz < sizeof(uint64_t) + header.scratch.n_bufs * sizeof(buf_idx) ) {
 932                 return -EINVAL;
 933         }
 934
 935         if (header.scratch.reg >= 5) {
 936                 return -EINVAL;
 937         }
 938
 939         dev_priv->scratch_ages[header.scratch.reg] ++;
 940
 941         ref_age_base = (u32 *)(unsigned long)*((uint64_t *)cmdbuf->buf);
 942
 943         cmdbuf->buf += sizeof(uint64_t);
 944         cmdbuf->bufsz -= sizeof(uint64_t);
 945
 946         for (i=0; i < header.scratch.n_bufs; i++) {
 947                 buf_idx = *(u32 *)cmdbuf->buf;
 948                 buf_idx *= 2; /* 8 bytes per buf */
 949
 950                 if (DRM_COPY_TO_USER(ref_age_base + buf_idx, &dev_priv->scratch_ages[header.scratch.reg], sizeof(u32))) {
 951                         return -EINVAL;
 952                 }
 953
 954                 if (DRM_COPY_FROM_USER(&h_pending, ref_age_base + buf_idx + 1, sizeof(u32))) {
 955                         return -EINVAL;
 956                 }
 957
 958                 if (h_pending == 0) {
 959                         return -EINVAL;
 960                 }
 961
 962                 h_pending--;
 963
 964                 if (DRM_COPY_TO_USER(ref_age_base + buf_idx + 1, &h_pending, sizeof(u32))) {
 965                         return -EINVAL;
 966                 }
 967
 968                 cmdbuf->buf += sizeof(buf_idx);
 969                 cmdbuf->bufsz -= sizeof(buf_idx);
 970         }
 971
 972         BEGIN_RING(2);
 973         OUT_RING( CP_PACKET0( RADEON_SCRATCH_REG0 + header.scratch.reg * 4, 0 ) );
 974         OUT_RING( dev_priv->scratch_ages[header.scratch.reg] );
 975         ADVANCE_RING();
 976
 977         return 0;
 978 }
 979
 980 /**
 981  * Uploads user-supplied vertex program instructions or parameters onto
 982  * the graphics card.
 983  * Called by r300_do_cp_cmdbuf.
 984  */
 985 static __inline__ int r300_emit_r500fp(drm_radeon_private_t *dev_priv,
 986                                        drm_radeon_kcmd_buffer_t *cmdbuf,
 987                                        drm_r300_cmd_header_t header)
 988 {
 989         int sz;
 990         int addr;
 991         int type;
 992         int clamp;
 993         int stride;
 994         RING_LOCALS;
 995
 996         sz = header.r500fp.count;
 997         /* address is 9 bits 0 - 8, bit 1 of flags is part of address */
 998         addr = ((header.r500fp.adrhi_flags & 1) << 8) | header.r500fp.adrlo;
 999
1000         type = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_TYPE);
1001         clamp = !!(header.r500fp.adrhi_flags & R500FP_CONSTANT_CLAMP);
1002
1003         addr |= (type << 16);
1004         addr |= (clamp << 17);
1005
1006         stride = type ? 4 : 6;
1007
1008         DRM_DEBUG("r500fp %d %d type: %d\n", sz, addr, type);
1009         if (!sz)
1010                 return 0;
1011         if (sz * stride * 4 > cmdbuf->bufsz)
1012                 return -EINVAL;
1013
1014         BEGIN_RING(3 + sz * stride);
1015         OUT_RING_REG(R500_GA_US_VECTOR_INDEX, addr);
1016         OUT_RING(CP_PACKET0_TABLE(R500_GA_US_VECTOR_DATA, sz * stride - 1));
1017         OUT_RING_TABLE((int *)cmdbuf->buf, sz * stride);
1018
1019         ADVANCE_RING();
1020
1021         cmdbuf->buf += sz * stride * 4;
1022         cmdbuf->bufsz -= sz * stride * 4;
1023
1024         return 0;
1025 }
1026
1027
1028 /**
1029  * Parses and validates a user-supplied command buffer and emits appropriate
1030  * commands on the DMA ring buffer.
1031  * Called by the ioctl handler function radeon_cp_cmdbuf.
1032  */
1033 int r300_do_cp_cmdbuf(struct drm_device *dev,
1034                       struct drm_file *file_priv,
1035                       drm_radeon_kcmd_buffer_t *cmdbuf)
1036 {
1037         drm_radeon_private_t *dev_priv = dev->dev_private;
1038         struct drm_device_dma *dma = dev->dma;
1039         struct drm_buf *buf = NULL;
1040         int emit_dispatch_age = 0;
1041         int ret = 0;
1042
1043         DRM_DEBUG("\n");
1044
1045         /* pacify */
1046         r300_pacify(dev_priv);
1047
1048         if (cmdbuf->nbox <= R300_SIMULTANEOUS_CLIPRECTS) {
1049                 ret = r300_emit_cliprects(dev_priv, cmdbuf, 0);
1050                 if (ret)
1051                         goto cleanup;
1052         }
1053
1054         while (cmdbuf->bufsz >= sizeof(drm_r300_cmd_header_t)) {
1055                 int idx;
1056                 drm_r300_cmd_header_t header;
1057
1058                 header.u = *(unsigned int *)cmdbuf->buf;
1059
1060                 cmdbuf->buf += sizeof(header);
1061                 cmdbuf->bufsz -= sizeof(header);
1062
1063                 switch (header.header.cmd_type) {
1064                 case R300_CMD_PACKET0:
1065                         ret = r300_emit_packet0(dev_priv, cmdbuf, header);
1066                         if (ret) {
1067                                 DRM_ERROR("r300_emit_packet0 failed\n");
1068                                 goto cleanup;
1069                         }
1070                         break;
1071
1072                 case R300_CMD_VPU:
1073                         DRM_DEBUG("R300_CMD_VPU\n");
1074                         ret = r300_emit_vpu(dev_priv, cmdbuf, header);
1075                         if (ret) {
1076                                 DRM_ERROR("r300_emit_vpu failed\n");
1077                                 goto cleanup;
1078                         }
1079                         break;
1080
1081                 case R300_CMD_PACKET3:
1082                         DRM_DEBUG("R300_CMD_PACKET3\n");
1083                         ret = r300_emit_packet3(dev_priv, cmdbuf, header);
1084                         if (ret) {
1085                                 DRM_ERROR("r300_emit_packet3 failed\n");
1086                                 goto cleanup;
1087                         }
1088                         break;
1089
1090                 case R300_CMD_END3D:
1091                         DRM_DEBUG("R300_CMD_END3D\n");
1092                         /* TODO:
1093                            Ideally userspace driver should not need to issue this call,
1094                            i.e. the drm driver should issue it automatically and prevent
1095                            lockups.
1096
1097                            In practice, we do not understand why this call is needed and what
1098                            it does (except for some vague guesses that it has to do with cache
1099                            coherence) and so the user space driver does it.
1100
1101                            Once we are sure which uses prevent lockups the code could be moved
1102                            into the kernel and the userspace driver will not
1103                            need to use this command.
1104
1105                            Note that issuing this command does not hurt anything
1106                            except, possibly, performance */
1107                         r300_pacify(dev_priv);
1108                         break;
1109
1110                 case R300_CMD_CP_DELAY:
1111                         /* simple enough, we can do it here */
1112                         DRM_DEBUG("R300_CMD_CP_DELAY\n");
1113                         {
1114                                 int i;
1115                                 RING_LOCALS;
1116
1117                                 BEGIN_RING(header.delay.count);
1118                                 for (i = 0; i < header.delay.count; i++)
1119                                         OUT_RING(RADEON_CP_PACKET2);
1120                                 ADVANCE_RING();
1121                         }
1122                         break;
1123
1124                 case R300_CMD_DMA_DISCARD:
1125                         DRM_DEBUG("RADEON_CMD_DMA_DISCARD\n");
1126                         idx = header.dma.buf_idx;
1127                         if (idx < 0 || idx >= dma->buf_count) {
1128                                 DRM_ERROR("buffer index %d (of %d max)\n",
1129                                           idx, dma->buf_count - 1);
1130                                 ret = -EINVAL;
1131                                 goto cleanup;
1132                         }
1133
1134                         buf = dma->buflist[idx];
1135                         if (buf->file_priv != file_priv || buf->pending) {
1136                                 DRM_ERROR("bad buffer %p %p %d\n",
1137                                           buf->file_priv, file_priv,
1138                                           buf->pending);
1139                                 ret = -EINVAL;
1140                                 goto cleanup;
1141                         }
1142
1143                         emit_dispatch_age = 1;
1144                         r300_discard_buffer(dev, buf);
1145                         break;
1146
1147                 case R300_CMD_WAIT:
1148                         DRM_DEBUG("R300_CMD_WAIT\n");
1149                         r300_cmd_wait(dev_priv, header);
1150                         break;
1151
1152                 case R300_CMD_SCRATCH:
1153                         DRM_DEBUG("R300_CMD_SCRATCH\n");
1154                         ret = r300_scratch(dev_priv, cmdbuf, header);
1155                         if (ret) {
1156                                 DRM_ERROR("r300_scratch failed\n");
1157                                 goto cleanup;
1158                         }
1159                         break;
1160
1161                 case R300_CMD_R500FP:
1162                         if ((dev_priv->flags & RADEON_FAMILY_MASK) < CHIP_RV515) {
1163                                 DRM_ERROR("Calling r500 command on r300 card\n");
1164                                 ret = -EINVAL;
1165                                 goto cleanup;
1166                         }
1167                         DRM_DEBUG("R300_CMD_R500FP\n");
1168                         ret = r300_emit_r500fp(dev_priv, cmdbuf, header);
1169                         if (ret) {
1170                                 DRM_ERROR("r300_emit_r500fp failed\n");
1171                                 goto cleanup;
1172                         }
1173                         break;
1174                 default:
1175                         DRM_ERROR("bad cmd_type %i at %p\n",
1176                                   header.header.cmd_type,
1177                                   cmdbuf->buf - sizeof(header));
1178                         ret = -EINVAL;
1179                         goto cleanup;
1180                 }
1181         }
1182
1183         DRM_DEBUG("END\n");
1184
1185       cleanup:
1186         r300_pacify(dev_priv);
1187
1188         /* We emit the vertex buffer age here, outside the pacifier "brackets"
1189          * for two reasons:
1190          *  (1) This may coalesce multiple age emissions into a single one and
1191          *  (2) more importantly, some chips lock up hard when scratch registers
1192          *      are written inside the pacifier bracket.
1193          */
1194         if (emit_dispatch_age) {
1195                 RING_LOCALS;
1196
1197                 /* Emit the vertex buffer age */
1198                 BEGIN_RING(2);
1199                 RADEON_DISPATCH_AGE(dev_priv->sarea_priv->last_dispatch);
1200                 ADVANCE_RING();
1201         }
1202
1203         COMMIT_RING();
1204
1205         return ret;
1206 }