sys/dev/disk/aic7xxx/aic7xxx.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2002 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#134 $
  41  *
  42  * $FreeBSD: src/sys/dev/aic7xxx/aic7xxx.c,v 1.41.2.27 2003/06/10 03:26:08 gibbs Exp $
  43  * $DragonFly: src/sys/dev/disk/aic7xxx/aic7xxx.c,v 1.9 2006/09/05 03:48:09 dillon Exp $
  44  */
  45
  46 #include "aic7xxx_osm.h"
  47 #include "aic7xxx_inline.h"
  48 #include "aicasm/aicasm_insformat.h"
  49
  50 /****************************** Softc Data ************************************/
  51 struct ahc_softc_tailq ahc_tailq = TAILQ_HEAD_INITIALIZER(ahc_tailq);
  52
  53 /***************************** Lookup Tables **********************************/
  54 char *ahc_chip_names[] =
  55 {
  56         "NONE",
  57         "aic7770",
  58         "aic7850",
  59         "aic7855",
  60         "aic7859",
  61         "aic7860",
  62         "aic7870",
  63         "aic7880",
  64         "aic7895",
  65         "aic7895C",
  66         "aic7890/91",
  67         "aic7896/97",
  68         "aic7892",
  69         "aic7899"
  70 };
  71 static const u_int num_chip_names = NUM_ELEMENTS(ahc_chip_names);
  72
  73 /*
  74  * Hardware error codes.
  75  */
  76 struct ahc_hard_error_entry {
  77         uint8_t errno;
  78         char *errmesg;
  79 };
  80
  81 static struct ahc_hard_error_entry ahc_hard_errors[] = {
  82         { ILLHADDR,     "Illegal Host Access" },
  83         { ILLSADDR,     "Illegal Sequencer Address referrenced" },
  84         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  85         { SQPARERR,     "Sequencer Parity Error" },
  86         { DPARERR,      "Data-path Parity Error" },
  87         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  88         { PCIERRSTAT,   "PCI Error detected" },
  89         { CIOPARERR,    "CIOBUS Parity Error" },
  90 };
  91 static const u_int num_errors = NUM_ELEMENTS(ahc_hard_errors);
  92
  93 static struct ahc_phase_table_entry ahc_phase_table[] =
  94 {
  95         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  96         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  97         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  98         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  99         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
 100         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
 101         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
 102         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
 103         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
 104         { 0,            MSG_NOOP,               "in unknown phase"      }
 105 };
 106
 107 /*
 108  * In most cases we only wish to itterate over real phases, so
 109  * exclude the last element from the count.
 110  */
 111 static const u_int num_phases = NUM_ELEMENTS(ahc_phase_table) - 1;
 112
 113 /*
 114  * Valid SCSIRATE values.  (p. 3-17)
 115  * Provides a mapping of tranfer periods in ns to the proper value to
 116  * stick in the scsixfer reg.
 117  */
 118 static struct ahc_syncrate ahc_syncrates[] =
 119 {
 120       /* ultra2    fast/ultra  period     rate */
 121         { 0x42,      0x000,      9,      "80.0" },
 122         { 0x03,      0x000,     10,      "40.0" },
 123         { 0x04,      0x000,     11,      "33.0" },
 124         { 0x05,      0x100,     12,      "20.0" },
 125         { 0x06,      0x110,     15,      "16.0" },
 126         { 0x07,      0x120,     18,      "13.4" },
 127         { 0x08,      0x000,     25,      "10.0" },
 128         { 0x19,      0x010,     31,      "8.0"  },
 129         { 0x1a,      0x020,     37,      "6.67" },
 130         { 0x1b,      0x030,     43,      "5.7"  },
 131         { 0x1c,      0x040,     50,      "5.0"  },
 132         { 0x00,      0x050,     56,      "4.4"  },
 133         { 0x00,      0x060,     62,      "4.0"  },
 134         { 0x00,      0x070,     68,      "3.6"  },
 135         { 0x00,      0x000,      0,      NULL   }
 136 };
 137
 138 /* Our Sequencer Program */
 139 #include "aic7xxx_seq.h"
 140
 141 /**************************** Function Declarations ***************************/
 142 static void             ahc_force_renegotiation(struct ahc_softc *ahc,
 143                                                 struct ahc_devinfo *devinfo);
 144 static struct ahc_tmode_tstate*
 145                         ahc_alloc_tstate(struct ahc_softc *ahc,
 146                                          u_int scsi_id, char channel);
 147 #ifdef AHC_TARGET_MODE
 148 static void             ahc_free_tstate(struct ahc_softc *ahc,
 149                                         u_int scsi_id, char channel, int force);
 150 #endif
 151 static struct ahc_syncrate*
 152                         ahc_devlimited_syncrate(struct ahc_softc *ahc,
 153                                                 struct ahc_initiator_tinfo *,
 154                                                 u_int *period,
 155                                                 u_int *ppr_options,
 156                                                 role_t role);
 157 static void             ahc_update_pending_scbs(struct ahc_softc *ahc);
 158 static void             ahc_fetch_devinfo(struct ahc_softc *ahc,
 159                                           struct ahc_devinfo *devinfo);
 160 static void             ahc_scb_devinfo(struct ahc_softc *ahc,
 161                                         struct ahc_devinfo *devinfo,
 162                                         struct scb *scb);
 163 static void             ahc_assert_atn(struct ahc_softc *ahc);
 164 static void             ahc_setup_initiator_msgout(struct ahc_softc *ahc,
 165                                                    struct ahc_devinfo *devinfo,
 166                                                    struct scb *scb);
 167 static void             ahc_build_transfer_msg(struct ahc_softc *ahc,
 168                                                struct ahc_devinfo *devinfo);
 169 static void             ahc_construct_sdtr(struct ahc_softc *ahc,
 170                                            struct ahc_devinfo *devinfo,
 171                                            u_int period, u_int offset);
 172 static void             ahc_construct_wdtr(struct ahc_softc *ahc,
 173                                            struct ahc_devinfo *devinfo,
 174                                            u_int bus_width);
 175 static void             ahc_construct_ppr(struct ahc_softc *ahc,
 176                                           struct ahc_devinfo *devinfo,
 177                                           u_int period, u_int offset,
 178                                           u_int bus_width, u_int ppr_options);
 179 static void             ahc_clear_msg_state(struct ahc_softc *ahc);
 180 static void             ahc_handle_proto_violation(struct ahc_softc *ahc);
 181 static void             ahc_handle_message_phase(struct ahc_softc *ahc);
 182 typedef enum {
 183         AHCMSG_1B,
 184         AHCMSG_2B,
 185         AHCMSG_EXT
 186 } ahc_msgtype;
 187 static int              ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
 188                                      u_int msgval, int full);
 189 static int              ahc_parse_msg(struct ahc_softc *ahc,
 190                                       struct ahc_devinfo *devinfo);
 191 static int              ahc_handle_msg_reject(struct ahc_softc *ahc,
 192                                               struct ahc_devinfo *devinfo);
 193 static void             ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
 194                                                 struct ahc_devinfo *devinfo);
 195 static void             ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
 196 static void             ahc_handle_devreset(struct ahc_softc *ahc,
 197                                             struct ahc_devinfo *devinfo,
 198                                             cam_status status, char *message,
 199                                             int verbose_level);
 200 #ifdef AHC_TARGET_MODE
 201 static void             ahc_setup_target_msgin(struct ahc_softc *ahc,
 202                                                struct ahc_devinfo *devinfo,
 203                                                struct scb *scb);
 204 #endif
 205
 206 static bus_dmamap_callback_t    ahc_dmamap_cb;
 207 static void                     ahc_build_free_scb_list(struct ahc_softc *ahc);
 208 static int                      ahc_init_scbdata(struct ahc_softc *ahc);
 209 static void                     ahc_fini_scbdata(struct ahc_softc *ahc);
 210 static void             ahc_qinfifo_requeue(struct ahc_softc *ahc,
 211                                             struct scb *prev_scb,
 212                                             struct scb *scb);
 213 static int              ahc_qinfifo_count(struct ahc_softc *ahc);
 214 static u_int            ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
 215                                                    u_int prev, u_int scbptr);
 216 static void             ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
 217 static u_int            ahc_rem_wscb(struct ahc_softc *ahc,
 218                                      u_int scbpos, u_int prev);
 219 static void             ahc_reset_current_bus(struct ahc_softc *ahc);
 220 #ifdef AHC_DUMP_SEQ
 221 static void             ahc_dumpseq(struct ahc_softc *ahc);
 222 #endif
 223 static int              ahc_loadseq(struct ahc_softc *ahc);
 224 static int              ahc_check_patch(struct ahc_softc *ahc,
 225                                         struct patch **start_patch,
 226                                         u_int start_instr, u_int *skip_addr);
 227 static void             ahc_download_instr(struct ahc_softc *ahc,
 228                                            u_int instrptr, uint8_t *dconsts);
 229 #ifdef AHC_TARGET_MODE
 230 static void             ahc_queue_lstate_event(struct ahc_softc *ahc,
 231                                                struct ahc_tmode_lstate *lstate,
 232                                                u_int initiator_id,
 233                                                u_int event_type,
 234                                                u_int event_arg);
 235 static void             ahc_update_scsiid(struct ahc_softc *ahc,
 236                                           u_int targid_mask);
 237 static int              ahc_handle_target_cmd(struct ahc_softc *ahc,
 238                                               struct target_cmd *cmd);
 239 #endif
 240 /************************* Sequencer Execution Control ************************/
 241 /*
 242  * Restart the sequencer program from address zero
 243  */
 244 void
 245 ahc_restart(struct ahc_softc *ahc)
 246 {
 247
 248         ahc_pause(ahc);
 249
 250         /* No more pending messages. */
 251         ahc_clear_msg_state(ahc);
 252
 253         ahc_outb(ahc, SCSISIGO, 0);             /* De-assert BSY */
 254         ahc_outb(ahc, MSG_OUT, MSG_NOOP);       /* No message to send */
 255         ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 256         ahc_outb(ahc, LASTPHASE, P_BUSFREE);
 257         ahc_outb(ahc, SAVED_SCSIID, 0xFF);
 258         ahc_outb(ahc, SAVED_LUN, 0xFF);
 259
 260         /*
 261          * Ensure that the sequencer's idea of TQINPOS
 262          * matches our own.  The sequencer increments TQINPOS
 263          * only after it sees a DMA complete and a reset could
 264          * occur before the increment leaving the kernel to believe
 265          * the command arrived but the sequencer to not.
 266          */
 267         ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
 268
 269         /* Always allow reselection */
 270         ahc_outb(ahc, SCSISEQ,
 271                  ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 272         if ((ahc->features & AHC_CMD_CHAN) != 0) {
 273                 /* Ensure that no DMA operations are in progress */
 274                 ahc_outb(ahc, CCSCBCNT, 0);
 275                 ahc_outb(ahc, CCSGCTL, 0);
 276                 ahc_outb(ahc, CCSCBCTL, 0);
 277         }
 278         /*
 279          * If we were in the process of DMA'ing SCB data into
 280          * an SCB, replace that SCB on the free list.  This prevents
 281          * an SCB leak.
 282          */
 283         if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
 284                 ahc_add_curscb_to_free_list(ahc);
 285                 ahc_outb(ahc, SEQ_FLAGS2,
 286                          ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
 287         }
 288         ahc_outb(ahc, MWI_RESIDUAL, 0);
 289         ahc_outb(ahc, SEQCTL, ahc->seqctl);
 290         ahc_outb(ahc, SEQADDR0, 0);
 291         ahc_outb(ahc, SEQADDR1, 0);
 292         ahc_unpause(ahc);
 293 }
 294
 295 /************************* Input/Output Queues ********************************/
 296 void
 297 ahc_run_qoutfifo(struct ahc_softc *ahc)
 298 {
 299         struct scb *scb;
 300         u_int  scb_index;
 301
 302         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
 303         while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
 304
 305                 scb_index = ahc->qoutfifo[ahc->qoutfifonext];
 306                 if ((ahc->qoutfifonext & 0x03) == 0x03) {
 307                         u_int modnext;
 308
 309                         /*
 310                          * Clear 32bits of QOUTFIFO at a time
 311                          * so that we don't clobber an incoming
 312                          * byte DMA to the array on architectures
 313                          * that only support 32bit load and store
 314                          * operations.
 315                          */
 316                         modnext = ahc->qoutfifonext & ~0x3;
 317                         *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
 318                         ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 319                                         ahc->shared_data_dmamap,
 320                                         /*offset*/modnext, /*len*/4,
 321                                         BUS_DMASYNC_PREREAD);
 322                 }
 323                 ahc->qoutfifonext++;
 324
 325                 scb = ahc_lookup_scb(ahc, scb_index);
 326                 if (scb == NULL) {
 327                         printf("%s: WARNING no command for scb %d "
 328                                "(cmdcmplt)\nQOUTPOS = %d\n",
 329                                ahc_name(ahc), scb_index,
 330                                (ahc->qoutfifonext - 1) & 0xFF);
 331                         continue;
 332                 }
 333
 334                 /*
 335                  * Save off the residual
 336                  * if there is one.
 337                  */
 338                 ahc_update_residual(ahc, scb);
 339                 ahc_done(ahc, scb);
 340         }
 341 }
 342
 343 void
 344 ahc_run_untagged_queues(struct ahc_softc *ahc)
 345 {
 346         int i;
 347
 348         for (i = 0; i < 16; i++)
 349                 ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
 350 }
 351
 352 void
 353 ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
 354 {
 355         struct scb *scb;
 356
 357         if (ahc->untagged_queue_lock != 0)
 358                 return;
 359
 360         if ((scb = TAILQ_FIRST(queue)) != NULL
 361          && (scb->flags & SCB_ACTIVE) == 0) {
 362                 scb->flags |= SCB_ACTIVE;
 363                 ahc_queue_scb(ahc, scb);
 364         }
 365 }
 366
 367 /************************* Interrupt Handling *********************************/
 368 void
 369 ahc_handle_brkadrint(struct ahc_softc *ahc)
 370 {
 371         /*
 372          * We upset the sequencer :-(
 373          * Lookup the error message
 374          */
 375         int i;
 376         int error;
 377
 378         error = ahc_inb(ahc, ERROR);
 379         for (i = 0; error != 1 && i < num_errors; i++)
 380                 error >>= 1;
 381         printf("%s: brkadrint, %s at seqaddr = 0x%x\n",
 382                ahc_name(ahc), ahc_hard_errors[i].errmesg,
 383                ahc_inb(ahc, SEQADDR0) |
 384                (ahc_inb(ahc, SEQADDR1) << 8));
 385
 386         ahc_dump_card_state(ahc);
 387
 388         /* Tell everyone that this HBA is no longer available */
 389         ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 390                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 391                        CAM_NO_HBA);
 392
 393         /* Disable all interrupt sources by resetting the controller */
 394         ahc_shutdown(ahc);
 395 }
 396
 397 void
 398 ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
 399 {
 400         struct scb *scb;
 401         struct ahc_devinfo devinfo;
 402
 403         ahc_fetch_devinfo(ahc, &devinfo);
 404
 405         /*
 406          * Clear the upper byte that holds SEQINT status
 407          * codes and clear the SEQINT bit. We will unpause
 408          * the sequencer, if appropriate, after servicing
 409          * the request.
 410          */
 411         ahc_outb(ahc, CLRINT, CLRSEQINT);
 412         switch (intstat & SEQINT_MASK) {
 413         case BAD_STATUS:
 414         {
 415                 u_int  scb_index;
 416                 struct hardware_scb *hscb;
 417
 418                 /*
 419                  * Set the default return value to 0 (don't
 420                  * send sense).  The sense code will change
 421                  * this if needed.
 422                  */
 423                 ahc_outb(ahc, RETURN_1, 0);
 424
 425                 /*
 426                  * The sequencer will notify us when a command
 427                  * has an error that would be of interest to
 428                  * the kernel.  This allows us to leave the sequencer
 429                  * running in the common case of command completes
 430                  * without error.  The sequencer will already have
 431                  * dma'd the SCB back up to us, so we can reference
 432                  * the in kernel copy directly.
 433                  */
 434                 scb_index = ahc_inb(ahc, SCB_TAG);
 435                 scb = ahc_lookup_scb(ahc, scb_index);
 436                 if (scb == NULL) {
 437                         ahc_print_devinfo(ahc, &devinfo);
 438                         printf("ahc_intr - referenced scb "
 439                                "not valid during seqint 0x%x scb(%d)\n",
 440                                intstat, scb_index);
 441                         ahc_dump_card_state(ahc);
 442                         panic("for safety");
 443                         goto unpause;
 444                 }
 445
 446                 hscb = scb->hscb;
 447
 448                 /* Don't want to clobber the original sense code */
 449                 if ((scb->flags & SCB_SENSE) != 0) {
 450                         /*
 451                          * Clear the SCB_SENSE Flag and have
 452                          * the sequencer do a normal command
 453                          * complete.
 454                          */
 455                         scb->flags &= ~SCB_SENSE;
 456                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
 457                         break;
 458                 }
 459                 ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
 460                 /* Freeze the queue until the client sees the error. */
 461                 ahc_freeze_devq(ahc, scb);
 462                 ahc_freeze_scb(scb);
 463                 ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
 464                 switch (hscb->shared_data.status.scsi_status) {
 465                 case SCSI_STATUS_OK:
 466                         printf("%s: Interrupted for staus of 0???\n",
 467                                ahc_name(ahc));
 468                         break;
 469                 case SCSI_STATUS_CMD_TERMINATED:
 470                 case SCSI_STATUS_CHECK_COND:
 471                 {
 472                         struct ahc_dma_seg *sg;
 473                         struct scsi_sense *sc;
 474                         struct ahc_initiator_tinfo *targ_info;
 475                         struct ahc_tmode_tstate *tstate;
 476                         struct ahc_transinfo *tinfo;
 477 #ifdef AHC_DEBUG
 478                         if (ahc_debug & AHC_SHOW_SENSE) {
 479                                 ahc_print_path(ahc, scb);
 480                                 printf("SCB %d: requests Check Status\n",
 481                                        scb->hscb->tag);
 482                         }
 483 #endif
 484
 485                         if (ahc_perform_autosense(scb) == 0)
 486                                 break;
 487
 488                         targ_info = ahc_fetch_transinfo(ahc,
 489                                                         devinfo.channel,
 490                                                         devinfo.our_scsiid,
 491                                                         devinfo.target,
 492                                                         &tstate);
 493                         tinfo = &targ_info->curr;
 494                         sg = scb->sg_list;
 495                         sc = (struct scsi_sense *)(&hscb->shared_data.cdb);
 496                         /*
 497                          * Save off the residual if there is one.
 498                          */
 499                         ahc_update_residual(ahc, scb);
 500 #ifdef AHC_DEBUG
 501                         if (ahc_debug & AHC_SHOW_SENSE) {
 502                                 ahc_print_path(ahc, scb);
 503                                 printf("Sending Sense\n");
 504                         }
 505 #endif
 506                         sg->addr = ahc_get_sense_bufaddr(ahc, scb);
 507                         sg->len = ahc_get_sense_bufsize(ahc, scb);
 508                         sg->len |= AHC_DMA_LAST_SEG;
 509
 510                         /* Fixup byte order */
 511                         sg->addr = ahc_htole32(sg->addr);
 512                         sg->len = ahc_htole32(sg->len);
 513
 514                         sc->opcode = REQUEST_SENSE;
 515                         sc->byte2 = 0;
 516                         if (tinfo->protocol_version <= SCSI_REV_2
 517                          && SCB_GET_LUN(scb) < 8)
 518                                 sc->byte2 = SCB_GET_LUN(scb) << 5;
 519                         sc->unused[0] = 0;
 520                         sc->unused[1] = 0;
 521                         sc->length = sg->len;
 522                         sc->control = 0;
 523
 524                         /*
 525                          * We can't allow the target to disconnect.
 526                          * This will be an untagged transaction and
 527                          * having the target disconnect will make this
 528                          * transaction indestinguishable from outstanding
 529                          * tagged transactions.
 530                          */
 531                         hscb->control = 0;
 532
 533                         /*
 534                          * This request sense could be because the
 535                          * the device lost power or in some other
 536                          * way has lost our transfer negotiations.
 537                          * Renegotiate if appropriate.  Unit attention
 538                          * errors will be reported before any data
 539                          * phases occur.
 540                          */
 541                         if (ahc_get_residual(scb)
 542                          == ahc_get_transfer_length(scb)) {
 543                                 ahc_update_neg_request(ahc, &devinfo,
 544                                                        tstate, targ_info,
 545                                                        AHC_NEG_IF_NON_ASYNC);
 546                         }
 547                         if (tstate->auto_negotiate & devinfo.target_mask) {
 548                                 hscb->control |= MK_MESSAGE;
 549                                 scb->flags &= ~SCB_NEGOTIATE;
 550                                 scb->flags |= SCB_AUTO_NEGOTIATE;
 551                         }
 552                         hscb->cdb_len = sizeof(*sc);
 553                         hscb->dataptr = sg->addr;
 554                         hscb->datacnt = sg->len;
 555                         hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
 556                         hscb->sgptr = ahc_htole32(hscb->sgptr);
 557                         scb->sg_count = 1;
 558                         scb->flags |= SCB_SENSE;
 559                         ahc_qinfifo_requeue_tail(ahc, scb);
 560                         ahc_outb(ahc, RETURN_1, SEND_SENSE);
 561                         /*
 562                          * Ensure we have enough time to actually
 563                          * retrieve the sense.
 564                          */
 565                         ahc_scb_timer_reset(scb, 5 * 1000000);
 566                         break;
 567                 }
 568                 default:
 569                         break;
 570                 }
 571                 break;
 572         }
 573         case NO_MATCH:
 574         {
 575                 /* Ensure we don't leave the selection hardware on */
 576                 ahc_outb(ahc, SCSISEQ,
 577                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
 578
 579                 printf("%s:%c:%d: no active SCB for reconnecting "
 580                        "target - issuing BUS DEVICE RESET\n",
 581                        ahc_name(ahc), devinfo.channel, devinfo.target);
 582                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
 583                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
 584                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
 585                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
 586                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
 587                        "SINDEX == 0x%x\n",
 588                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
 589                        ahc_index_busy_tcl(ahc,
 590                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
 591                                       ahc_inb(ahc, SAVED_LUN))),
 592                        ahc_inb(ahc, SINDEX));
 593                 printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
 594                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
 595                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
 596                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
 597                        ahc_inb(ahc, SCB_CONTROL));
 598                 printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
 599                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
 600                 printf("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
 601                 printf("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
 602                 ahc_dump_card_state(ahc);
 603                 ahc->msgout_buf[0] = MSG_BUS_DEV_RESET;
 604                 ahc->msgout_len = 1;
 605                 ahc->msgout_index = 0;
 606                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 607                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
 608                 ahc_assert_atn(ahc);
 609                 break;
 610         }
 611         case SEND_REJECT:
 612         {
 613                 u_int rejbyte = ahc_inb(ahc, ACCUM);
 614                 printf("%s:%c:%d: Warning - unknown message received from "
 615                        "target (0x%x).  Rejecting\n",
 616                        ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
 617                 break;
 618         }
 619         case PROTO_VIOLATION:
 620         {
 621                 ahc_handle_proto_violation(ahc);
 622                 break;
 623         }
 624         case IGN_WIDE_RES:
 625                 ahc_handle_ign_wide_residue(ahc, &devinfo);
 626                 break;
 627         case PDATA_REINIT:
 628                 ahc_reinitialize_dataptrs(ahc);
 629                 break;
 630         case BAD_PHASE:
 631         {
 632                 u_int lastphase;
 633
 634                 lastphase = ahc_inb(ahc, LASTPHASE);
 635                 printf("%s:%c:%d: unknown scsi bus phase %x, "
 636                        "lastphase = 0x%x.  Attempting to continue\n",
 637                        ahc_name(ahc), devinfo.channel, devinfo.target,
 638                        lastphase, ahc_inb(ahc, SCSISIGI));
 639                 break;
 640         }
 641         case MISSED_BUSFREE:
 642         {
 643                 u_int lastphase;
 644
 645                 lastphase = ahc_inb(ahc, LASTPHASE);
 646                 printf("%s:%c:%d: Missed busfree. "
 647                        "Lastphase = 0x%x, Curphase = 0x%x\n",
 648                        ahc_name(ahc), devinfo.channel, devinfo.target,
 649                        lastphase, ahc_inb(ahc, SCSISIGI));
 650                 ahc_restart(ahc);
 651                 return;
 652         }
 653         case HOST_MSG_LOOP:
 654         {
 655                 /*
 656                  * The sequencer has encountered a message phase
 657                  * that requires host assistance for completion.
 658                  * While handling the message phase(s), we will be
 659                  * notified by the sequencer after each byte is
 660                  * transfered so we can track bus phase changes.
 661                  *
 662                  * If this is the first time we've seen a HOST_MSG_LOOP
 663                  * interrupt, initialize the state of the host message
 664                  * loop.
 665                  */
 666                 if (ahc->msg_type == MSG_TYPE_NONE) {
 667                         struct scb *scb;
 668                         u_int scb_index;
 669                         u_int bus_phase;
 670
 671                         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
 672                         if (bus_phase != P_MESGIN
 673                          && bus_phase != P_MESGOUT) {
 674                                 printf("ahc_intr: HOST_MSG_LOOP bad "
 675                                        "phase 0x%x\n",
 676                                       bus_phase);
 677                                 /*
 678                                  * Probably transitioned to bus free before
 679                                  * we got here.  Just punt the message.
 680                                  */
 681                                 ahc_clear_intstat(ahc);
 682                                 ahc_restart(ahc);
 683                                 return;
 684                         }
 685
 686                         scb_index = ahc_inb(ahc, SCB_TAG);
 687                         scb = ahc_lookup_scb(ahc, scb_index);
 688                         if (devinfo.role == ROLE_INITIATOR) {
 689                                 if (scb == NULL)
 690                                         panic("HOST_MSG_LOOP with "
 691                                               "invalid SCB %x\n", scb_index);
 692
 693                                 if (bus_phase == P_MESGOUT)
 694                                         ahc_setup_initiator_msgout(ahc,
 695                                                                    &devinfo,
 696                                                                    scb);
 697                                 else {
 698                                         ahc->msg_type =
 699                                             MSG_TYPE_INITIATOR_MSGIN;
 700                                         ahc->msgin_index = 0;
 701                                 }
 702                         }
 703 #ifdef AHC_TARGET_MODE
 704                         else {
 705                                 if (bus_phase == P_MESGOUT) {
 706                                         ahc->msg_type =
 707                                             MSG_TYPE_TARGET_MSGOUT;
 708                                         ahc->msgin_index = 0;
 709                                 }
 710                                 else
 711                                         ahc_setup_target_msgin(ahc,
 712                                                                &devinfo,
 713                                                                scb);
 714                         }
 715 #endif
 716                 }
 717
 718                 ahc_handle_message_phase(ahc);
 719                 break;
 720         }
 721         case PERR_DETECTED:
 722         {
 723                 /*
 724                  * If we've cleared the parity error interrupt
 725                  * but the sequencer still believes that SCSIPERR
 726                  * is true, it must be that the parity error is
 727                  * for the currently presented byte on the bus,
 728                  * and we are not in a phase (data-in) where we will
 729                  * eventually ack this byte.  Ack the byte and
 730                  * throw it away in the hope that the target will
 731                  * take us to message out to deliver the appropriate
 732                  * error message.
 733                  */
 734                 if ((intstat & SCSIINT) == 0
 735                  && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
 736
 737                         if ((ahc->features & AHC_DT) == 0) {
 738                                 u_int curphase;
 739
 740                                 /*
 741                                  * The hardware will only let you ack bytes
 742                                  * if the expected phase in SCSISIGO matches
 743                                  * the current phase.  Make sure this is
 744                                  * currently the case.
 745                                  */
 746                                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
 747                                 ahc_outb(ahc, LASTPHASE, curphase);
 748                                 ahc_outb(ahc, SCSISIGO, curphase);
 749                         }
 750                         if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
 751                                 int wait;
 752
 753                                 /*
 754                                  * In a data phase.  Faster to bitbucket
 755                                  * the data than to individually ack each
 756                                  * byte.  This is also the only strategy
 757                                  * that will work with AUTOACK enabled.
 758                                  */
 759                                 ahc_outb(ahc, SXFRCTL1,
 760                                          ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
 761                                 wait = 5000;
 762                                 while (--wait != 0) {
 763                                         if ((ahc_inb(ahc, SCSISIGI)
 764                                           & (CDI|MSGI)) != 0)
 765                                                 break;
 766                                         ahc_delay(100);
 767                                 }
 768                                 ahc_outb(ahc, SXFRCTL1,
 769                                          ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 770                                 if (wait == 0) {
 771                                         struct  scb *scb;
 772                                         u_int   scb_index;
 773
 774                                         ahc_print_devinfo(ahc, &devinfo);
 775                                         printf("Unable to clear parity error.  "
 776                                                "Resetting bus.\n");
 777                                         scb_index = ahc_inb(ahc, SCB_TAG);
 778                                         scb = ahc_lookup_scb(ahc, scb_index);
 779                                         if (scb != NULL)
 780                                                 ahc_set_transaction_status(scb,
 781                                                     CAM_UNCOR_PARITY);
 782                                         ahc_reset_channel(ahc, devinfo.channel,
 783                                                           /*init reset*/TRUE);
 784                                 }
 785                         } else {
 786                                 ahc_inb(ahc, SCSIDATL);
 787                         }
 788                 }
 789                 break;
 790         }
 791         case DATA_OVERRUN:
 792         {
 793                 /*
 794                  * When the sequencer detects an overrun, it
 795                  * places the controller in "BITBUCKET" mode
 796                  * and allows the target to complete its transfer.
 797                  * Unfortunately, none of the counters get updated
 798                  * when the controller is in this mode, so we have
 799                  * no way of knowing how large the overrun was.
 800                  */
 801                 u_int scbindex = ahc_inb(ahc, SCB_TAG);
 802                 u_int lastphase = ahc_inb(ahc, LASTPHASE);
 803                 u_int i;
 804
 805                 scb = ahc_lookup_scb(ahc, scbindex);
 806                 for (i = 0; i < num_phases; i++) {
 807                         if (lastphase == ahc_phase_table[i].phase)
 808                                 break;
 809                 }
 810                 ahc_print_path(ahc, scb);
 811                 printf("data overrun detected %s."
 812                        "  Tag == 0x%x.\n",
 813                        ahc_phase_table[i].phasemsg,
 814                        scb->hscb->tag);
 815                 ahc_print_path(ahc, scb);
 816                 printf("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
 817                        ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
 818                        ahc_get_transfer_length(scb), scb->sg_count);
 819                 if (scb->sg_count > 0) {
 820                         for (i = 0; i < scb->sg_count; i++) {
 821
 822                                 printf("sg[%d] - Addr 0x%x%x : Length %d\n",
 823                                        i,
 824                                        (ahc_le32toh(scb->sg_list[i].len) >> 24
 825                                         & SG_HIGH_ADDR_BITS),
 826                                        ahc_le32toh(scb->sg_list[i].addr),
 827                                        ahc_le32toh(scb->sg_list[i].len)
 828                                        & AHC_SG_LEN_MASK);
 829                         }
 830                 }
 831                 /*
 832                  * Set this and it will take effect when the
 833                  * target does a command complete.
 834                  */
 835                 ahc_freeze_devq(ahc, scb);
 836                 if ((scb->flags & SCB_SENSE) == 0) {
 837                         ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
 838                 } else {
 839                         scb->flags &= ~SCB_SENSE;
 840                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
 841                 }
 842                 ahc_freeze_scb(scb);
 843
 844                 if ((ahc->features & AHC_ULTRA2) != 0) {
 845                         /*
 846                          * Clear the channel in case we return
 847                          * to data phase later.
 848                          */
 849                         ahc_outb(ahc, SXFRCTL0,
 850                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
 851                         ahc_outb(ahc, SXFRCTL0,
 852                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
 853                 }
 854                 if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
 855                         u_int dscommand1;
 856
 857                         /* Ensure HHADDR is 0 for future DMA operations. */
 858                         dscommand1 = ahc_inb(ahc, DSCOMMAND1);
 859                         ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
 860                         ahc_outb(ahc, HADDR, 0);
 861                         ahc_outb(ahc, DSCOMMAND1, dscommand1);
 862                 }
 863                 break;
 864         }
 865         case MKMSG_FAILED:
 866         {
 867                 u_int scbindex;
 868
 869                 printf("%s:%c:%d:%d: Attempt to issue message failed\n",
 870                        ahc_name(ahc), devinfo.channel, devinfo.target,
 871                        devinfo.lun);
 872                 scbindex = ahc_inb(ahc, SCB_TAG);
 873                 scb = ahc_lookup_scb(ahc, scbindex);
 874                 if (scb != NULL
 875                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
 876                         /*
 877                          * Ensure that we didn't put a second instance of this
 878                          * SCB into the QINFIFO.
 879                          */
 880                         ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
 881                                            SCB_GET_CHANNEL(ahc, scb),
 882                                            SCB_GET_LUN(scb), scb->hscb->tag,
 883                                            ROLE_INITIATOR, /*status*/0,
 884                                            SEARCH_REMOVE);
 885                 break;
 886         }
 887         case NO_FREE_SCB:
 888         {
 889                 printf("%s: No free or disconnected SCBs\n", ahc_name(ahc));
 890                 ahc_dump_card_state(ahc);
 891                 panic("for safety");
 892                 break;
 893         }
 894         case SCB_MISMATCH:
 895         {
 896                 u_int scbptr;
 897
 898                 scbptr = ahc_inb(ahc, SCBPTR);
 899                 printf("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
 900                        scbptr, ahc_inb(ahc, ARG_1),
 901                        ahc->scb_data->hscbs[scbptr].tag);
 902                 ahc_dump_card_state(ahc);
 903                 panic("for saftey");
 904                 break;
 905         }
 906         case OUT_OF_RANGE:
 907         {
 908                 printf("%s: BTT calculation out of range\n", ahc_name(ahc));
 909                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
 910                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
 911                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
 912                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
 913                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
 914                        "SINDEX == 0x%x\n, A == 0x%x\n",
 915                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
 916                        ahc_index_busy_tcl(ahc,
 917                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
 918                                       ahc_inb(ahc, SAVED_LUN))),
 919                        ahc_inb(ahc, SINDEX),
 920                        ahc_inb(ahc, ACCUM));
 921                 printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
 922                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
 923                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
 924                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
 925                        ahc_inb(ahc, SCB_CONTROL));
 926                 printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
 927                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
 928                 ahc_dump_card_state(ahc);
 929                 panic("for safety");
 930                 break;
 931         }
 932         default:
 933                 printf("ahc_intr: seqint, "
 934                        "intstat == 0x%x, scsisigi = 0x%x\n",
 935                        intstat, ahc_inb(ahc, SCSISIGI));
 936                 break;
 937         }
 938 unpause:
 939         /*
 940          *  The sequencer is paused immediately on
 941          *  a SEQINT, so we should restart it when
 942          *  we're done.
 943          */
 944         ahc_unpause(ahc);
 945 }
 946
 947 void
 948 ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
 949 {
 950         u_int   scb_index;
 951         u_int   status0;
 952         u_int   status;
 953         struct  scb *scb;
 954         char    cur_channel;
 955         char    intr_channel;
 956
 957         if ((ahc->features & AHC_TWIN) != 0
 958          && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
 959                 cur_channel = 'B';
 960         else
 961                 cur_channel = 'A';
 962         intr_channel = cur_channel;
 963
 964         if ((ahc->features & AHC_ULTRA2) != 0)
 965                 status0 = ahc_inb(ahc, SSTAT0) & IOERR;
 966         else
 967                 status0 = 0;
 968         status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
 969         if (status == 0 && status0 == 0) {
 970                 if ((ahc->features & AHC_TWIN) != 0) {
 971                         /* Try the other channel */
 972                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
 973                         status = ahc_inb(ahc, SSTAT1)
 974                                & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
 975                         intr_channel = (cur_channel == 'A') ? 'B' : 'A';
 976                 }
 977                 if (status == 0) {
 978                         printf("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
 979                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
 980                         ahc_unpause(ahc);
 981                         return;
 982                 }
 983         }
 984
 985         /* Make sure the sequencer is in a safe location. */
 986         ahc_clear_critical_section(ahc);
 987
 988         scb_index = ahc_inb(ahc, SCB_TAG);
 989         scb = ahc_lookup_scb(ahc, scb_index);
 990         if (scb != NULL
 991          && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
 992                 scb = NULL;
 993
 994         if ((ahc->features & AHC_ULTRA2) != 0
 995          && (status0 & IOERR) != 0) {
 996                 int now_lvd;
 997
 998                 now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
 999                 printf("%s: Transceiver State Has Changed to %s mode\n",
1000                        ahc_name(ahc), now_lvd ? "LVD" : "SE");
1001                 ahc_outb(ahc, CLRSINT0, CLRIOERR);
1002                 /*
1003                  * When transitioning to SE mode, the reset line
1004                  * glitches, triggering an arbitration bug in some
1005                  * Ultra2 controllers.  This bug is cleared when we
1006                  * assert the reset line.  Since a reset glitch has
1007                  * already occurred with this transition and a
1008                  * transceiver state change is handled just like
1009                  * a bus reset anyway, asserting the reset line
1010                  * ourselves is safe.
1011                  */
1012                 ahc_reset_channel(ahc, intr_channel,
1013                                  /*Initiate Reset*/now_lvd == 0);
1014         } else if ((status & SCSIRSTI) != 0) {
1015                 printf("%s: Someone reset channel %c\n",
1016                         ahc_name(ahc), intr_channel);
1017                 if (intr_channel != cur_channel)
1018                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1019                 ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1020         } else if ((status & SCSIPERR) != 0) {
1021                 /*
1022                  * Determine the bus phase and queue an appropriate message.
1023                  * SCSIPERR is latched true as soon as a parity error
1024                  * occurs.  If the sequencer acked the transfer that
1025                  * caused the parity error and the currently presented
1026                  * transfer on the bus has correct parity, SCSIPERR will
1027                  * be cleared by CLRSCSIPERR.  Use this to determine if
1028                  * we should look at the last phase the sequencer recorded,
1029                  * or the current phase presented on the bus.
1030                  */
1031                 struct  ahc_devinfo devinfo;
1032                 u_int   mesg_out;
1033                 u_int   curphase;
1034                 u_int   errorphase;
1035                 u_int   lastphase;
1036                 u_int   scsirate;
1037                 u_int   i;
1038                 u_int   sstat2;
1039                 int     silent;
1040
1041                 lastphase = ahc_inb(ahc, LASTPHASE);
1042                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1043                 sstat2 = ahc_inb(ahc, SSTAT2);
1044                 ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1045                 /*
1046                  * For all phases save DATA, the sequencer won't
1047                  * automatically ack a byte that has a parity error
1048                  * in it.  So the only way that the current phase
1049                  * could be 'data-in' is if the parity error is for
1050                  * an already acked byte in the data phase.  During
1051                  * synchronous data-in transfers, we may actually
1052                  * ack bytes before latching the current phase in
1053                  * LASTPHASE, leading to the discrepancy between
1054                  * curphase and lastphase.
1055                  */
1056                 if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1057                  || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1058                         errorphase = curphase;
1059                 else
1060                         errorphase = lastphase;
1061
1062                 for (i = 0; i < num_phases; i++) {
1063                         if (errorphase == ahc_phase_table[i].phase)
1064                                 break;
1065                 }
1066                 mesg_out = ahc_phase_table[i].mesg_out;
1067                 silent = FALSE;
1068                 if (scb != NULL) {
1069                         if (SCB_IS_SILENT(scb))
1070                                 silent = TRUE;
1071                         else
1072                                 ahc_print_path(ahc, scb);
1073                         scb->flags |= SCB_TRANSMISSION_ERROR;
1074                 } else
1075                         printf("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1076                                SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1077                 scsirate = ahc_inb(ahc, SCSIRATE);
1078                 if (silent == FALSE) {
1079                         printf("parity error detected %s. "
1080                                "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1081                                ahc_phase_table[i].phasemsg,
1082                                ahc_inw(ahc, SEQADDR0),
1083                                scsirate);
1084                         if ((ahc->features & AHC_DT) != 0) {
1085                                 if ((sstat2 & CRCVALERR) != 0)
1086                                         printf("\tCRC Value Mismatch\n");
1087                                 if ((sstat2 & CRCENDERR) != 0)
1088                                         printf("\tNo terminal CRC packet "
1089                                                "recevied\n");
1090                                 if ((sstat2 & CRCREQERR) != 0)
1091                                         printf("\tIllegal CRC packet "
1092                                                "request\n");
1093                                 if ((sstat2 & DUAL_EDGE_ERR) != 0)
1094                                         printf("\tUnexpected %sDT Data Phase\n",
1095                                                (scsirate & SINGLE_EDGE)
1096                                              ? "" : "non-");
1097                         }
1098                 }
1099
1100                 if ((ahc->features & AHC_DT) != 0
1101                  && (sstat2 & DUAL_EDGE_ERR) != 0) {
1102                         /*
1103                          * This error applies regardless of
1104                          * data direction, so ignore the value
1105                          * in the phase table.
1106                          */
1107                         mesg_out = MSG_INITIATOR_DET_ERR;
1108                 }
1109
1110                 /*
1111                  * We've set the hardware to assert ATN if we
1112                  * get a parity error on "in" phases, so all we
1113                  * need to do is stuff the message buffer with
1114                  * the appropriate message.  "In" phases have set
1115                  * mesg_out to something other than MSG_NOP.
1116                  */
1117                 if (mesg_out != MSG_NOOP) {
1118                         if (ahc->msg_type != MSG_TYPE_NONE)
1119                                 ahc->send_msg_perror = TRUE;
1120                         else
1121                                 ahc_outb(ahc, MSG_OUT, mesg_out);
1122                 }
1123                 /*
1124                  * Force a renegotiation with this target just in
1125                  * case we are out of sync for some external reason
1126                  * unknown (or unreported) by the target.
1127                  */
1128                 ahc_fetch_devinfo(ahc, &devinfo);
1129                 ahc_force_renegotiation(ahc, &devinfo);
1130
1131                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1132                 ahc_unpause(ahc);
1133         } else if ((status & SELTO) != 0) {
1134                 u_int   scbptr;
1135
1136                 /* Stop the selection */
1137                 ahc_outb(ahc, SCSISEQ, 0);
1138
1139                 /* No more pending messages */
1140                 ahc_clear_msg_state(ahc);
1141
1142                 /* Clear interrupt state */
1143                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1144                 ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1145
1146                 /*
1147                  * Although the driver does not care about the
1148                  * 'Selection in Progress' status bit, the busy
1149                  * LED does.  SELINGO is only cleared by a successful
1150                  * selection, so we must manually clear it to insure
1151                  * the LED turns off just in case no future successful
1152                  * selections occur (e.g. no devices on the bus).
1153                  */
1154                 ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1155
1156                 scbptr = ahc_inb(ahc, WAITING_SCBH);
1157                 ahc_outb(ahc, SCBPTR, scbptr);
1158                 scb_index = ahc_inb(ahc, SCB_TAG);
1159
1160                 scb = ahc_lookup_scb(ahc, scb_index);
1161                 if (scb == NULL) {
1162                         printf("%s: ahc_intr - referenced scb not "
1163                                "valid during SELTO scb(%d, %d)\n",
1164                                ahc_name(ahc), scbptr, scb_index);
1165                         ahc_dump_card_state(ahc);
1166                 } else {
1167                         struct ahc_devinfo devinfo;
1168 #ifdef AHC_DEBUG
1169                         if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1170                                 ahc_print_path(ahc, scb);
1171                                 printf("Saw Selection Timeout for SCB 0x%x\n",
1172                                        scb_index);
1173                         }
1174 #endif
1175                         /*
1176                          * Force a renegotiation with this target just in
1177                          * case the cable was pulled and will later be
1178                          * re-attached.  The target may forget its negotiation
1179                          * settings with us should it attempt to reselect
1180                          * during the interruption.  The target will not issue
1181                          * a unit attention in this case, so we must always
1182                          * renegotiate.
1183                          */
1184                         ahc_scb_devinfo(ahc, &devinfo, scb);
1185                         ahc_force_renegotiation(ahc, &devinfo);
1186                         ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1187                         ahc_freeze_devq(ahc, scb);
1188                 }
1189                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1190                 ahc_restart(ahc);
1191         } else if ((status & BUSFREE) != 0
1192                 && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1193                 struct  ahc_devinfo devinfo;
1194                 u_int   lastphase;
1195                 u_int   saved_scsiid;
1196                 u_int   saved_lun;
1197                 u_int   target;
1198                 u_int   initiator_role_id;
1199                 char    channel;
1200                 int     printerror;
1201
1202                 /*
1203                  * Clear our selection hardware as soon as possible.
1204                  * We may have an entry in the waiting Q for this target,
1205                  * that is affected by this busfree and we don't want to
1206                  * go about selecting the target while we handle the event.
1207                  */
1208                 ahc_outb(ahc, SCSISEQ,
1209                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1210
1211                 /*
1212                  * Disable busfree interrupts and clear the busfree
1213                  * interrupt status.  We do this here so that several
1214                  * bus transactions occur prior to clearing the SCSIINT
1215                  * latch.  It can take a bit for the clearing to take effect.
1216                  */
1217                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1218                 ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1219
1220                 /*
1221                  * Look at what phase we were last in.
1222                  * If its message out, chances are pretty good
1223                  * that the busfree was in response to one of
1224                  * our abort requests.
1225                  */
1226                 lastphase = ahc_inb(ahc, LASTPHASE);
1227                 saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1228                 saved_lun = ahc_inb(ahc, SAVED_LUN);
1229                 target = SCSIID_TARGET(ahc, saved_scsiid);
1230                 initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1231                 channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1232                 ahc_compile_devinfo(&devinfo, initiator_role_id,
1233                                     target, saved_lun, channel, ROLE_INITIATOR);
1234                 printerror = 1;
1235
1236                 if (lastphase == P_MESGOUT) {
1237                         u_int tag;
1238
1239                         tag = SCB_LIST_NULL;
1240                         if (ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT_TAG, TRUE)
1241                          || ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT, TRUE)) {
1242                                 if (ahc->msgout_buf[ahc->msgout_index - 1]
1243                                  == MSG_ABORT_TAG)
1244                                         tag = scb->hscb->tag;
1245                                 ahc_print_path(ahc, scb);
1246                                 printf("SCB %d - Abort%s Completed.\n",
1247                                        scb->hscb->tag, tag == SCB_LIST_NULL ?
1248                                        "" : " Tag");
1249                                 ahc_abort_scbs(ahc, target, channel,
1250                                                saved_lun, tag,
1251                                                ROLE_INITIATOR,
1252                                                CAM_REQ_ABORTED);
1253                                 printerror = 0;
1254                         } else if (ahc_sent_msg(ahc, AHCMSG_1B,
1255                                                 MSG_BUS_DEV_RESET, TRUE)) {
1256 #if defined(__DragonFly__) || defined(__FreeBSD__)
1257                                 /*
1258                                  * Don't mark the user's request for this BDR
1259                                  * as completing with CAM_BDR_SENT.  CAM3
1260                                  * specifies CAM_REQ_CMP.
1261                                  */
1262                                 if (scb != NULL
1263                                  && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
1264                                  && ahc_match_scb(ahc, scb, target, channel,
1265                                                   CAM_LUN_WILDCARD,
1266                                                   SCB_LIST_NULL,
1267                                                   ROLE_INITIATOR)) {
1268                                         ahc_set_transaction_status(scb, CAM_REQ_CMP);
1269                                 }
1270 #endif
1271                                 ahc_compile_devinfo(&devinfo,
1272                                                     initiator_role_id,
1273                                                     target,
1274                                                     CAM_LUN_WILDCARD,
1275                                                     channel,
1276                                                     ROLE_INITIATOR);
1277                                 ahc_handle_devreset(ahc, &devinfo,
1278                                                     CAM_BDR_SENT,
1279                                                     "Bus Device Reset",
1280                                                     /*verbose_level*/0);
1281                                 printerror = 0;
1282                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1283                                                 MSG_EXT_PPR, FALSE)) {
1284                                 struct ahc_initiator_tinfo *tinfo;
1285                                 struct ahc_tmode_tstate *tstate;
1286
1287                                 /*
1288                                  * PPR Rejected.  Try non-ppr negotiation
1289                                  * and retry command.
1290                                  */
1291                                 tinfo = ahc_fetch_transinfo(ahc,
1292                                                             devinfo.channel,
1293                                                             devinfo.our_scsiid,
1294                                                             devinfo.target,
1295                                                             &tstate);
1296                                 tinfo->curr.transport_version = 2;
1297                                 tinfo->goal.transport_version = 2;
1298                                 tinfo->goal.ppr_options = 0;
1299                                 ahc_qinfifo_requeue_tail(ahc, scb);
1300                                 printerror = 0;
1301                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1302                                                 MSG_EXT_WDTR, FALSE)) {
1303                                 /*
1304                                  * Negotiation Rejected.  Go-narrow and
1305                                  * retry command.
1306                                  */
1307                                 ahc_set_width(ahc, &devinfo,
1308                                               MSG_EXT_WDTR_BUS_8_BIT,
1309                                               AHC_TRANS_CUR|AHC_TRANS_GOAL,
1310                                               /*paused*/TRUE);
1311                                 ahc_qinfifo_requeue_tail(ahc, scb);
1312                                 printerror = 0;
1313                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1314                                                 MSG_EXT_SDTR, FALSE)) {
1315                                 /*
1316                                  * Negotiation Rejected.  Go-async and
1317                                  * retry command.
1318                                  */
1319                                 ahc_set_syncrate(ahc, &devinfo,
1320                                                 /*syncrate*/NULL,
1321                                                 /*period*/0, /*offset*/0,
1322                                                 /*ppr_options*/0,
1323                                                 AHC_TRANS_CUR|AHC_TRANS_GOAL,
1324                                                 /*paused*/TRUE);
1325                                 ahc_qinfifo_requeue_tail(ahc, scb);
1326                                 printerror = 0;
1327                         }
1328                 }
1329                 if (printerror != 0) {
1330                         u_int i;
1331
1332                         if (scb != NULL) {
1333                                 u_int tag;
1334
1335                                 if ((scb->hscb->control & TAG_ENB) != 0)
1336                                         tag = scb->hscb->tag;
1337                                 else
1338                                         tag = SCB_LIST_NULL;
1339                                 ahc_print_path(ahc, scb);
1340                                 ahc_abort_scbs(ahc, target, channel,
1341                                                SCB_GET_LUN(scb), tag,
1342                                                ROLE_INITIATOR,
1343                                                CAM_UNEXP_BUSFREE);
1344                         } else {
1345                                 /*
1346                                  * We had not fully identified this connection,
1347                                  * so we cannot abort anything.
1348                                  */
1349                                 printf("%s: ", ahc_name(ahc));
1350                         }
1351                         for (i = 0; i < num_phases; i++) {
1352                                 if (lastphase == ahc_phase_table[i].phase)
1353                                         break;
1354                         }
1355                         if (lastphase != P_BUSFREE) {
1356                                 /*
1357                                  * Renegotiate with this device at the
1358                                  * next oportunity just in case this busfree
1359                                  * is due to a negotiation mismatch with the
1360                                  * device.
1361                                  */
1362                                 ahc_force_renegotiation(ahc, &devinfo);
1363                         }
1364                         printf("Unexpected busfree %s\n"
1365                                "SEQADDR == 0x%x\n",
1366                                ahc_phase_table[i].phasemsg,
1367                                ahc_inb(ahc, SEQADDR0)
1368                                 | (ahc_inb(ahc, SEQADDR1) << 8));
1369                 }
1370                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1371                 ahc_restart(ahc);
1372         } else {
1373                 printf("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1374                        ahc_name(ahc), status);
1375                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1376         }
1377 }
1378
1379 /*
1380  * Force renegotiation to occur the next time we initiate
1381  * a command to the current device.
1382  */
1383 static void
1384 ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1385 {
1386         struct  ahc_initiator_tinfo *targ_info;
1387         struct  ahc_tmode_tstate *tstate;
1388
1389         targ_info = ahc_fetch_transinfo(ahc,
1390                                         devinfo->channel,
1391                                         devinfo->our_scsiid,
1392                                         devinfo->target,
1393                                         &tstate);
1394         ahc_update_neg_request(ahc, devinfo, tstate,
1395                                targ_info, AHC_NEG_IF_NON_ASYNC);
1396 }
1397
1398 #define AHC_MAX_STEPS 2000
1399 void
1400 ahc_clear_critical_section(struct ahc_softc *ahc)
1401 {
1402         int     stepping;
1403         int     steps;
1404         u_int   simode0;
1405         u_int   simode1;
1406
1407         if (ahc->num_critical_sections == 0)
1408                 return;
1409
1410         stepping = FALSE;
1411         steps = 0;
1412         simode0 = 0;
1413         simode1 = 0;
1414         for (;;) {
1415                 struct  cs *cs;
1416                 u_int   seqaddr;
1417                 u_int   i;
1418
1419                 seqaddr = ahc_inb(ahc, SEQADDR0)
1420                         | (ahc_inb(ahc, SEQADDR1) << 8);
1421
1422                 /*
1423                  * Seqaddr represents the next instruction to execute,
1424                  * so we are really executing the instruction just
1425                  * before it.
1426                  */
1427                 if (seqaddr != 0)
1428                         seqaddr -= 1;
1429                 cs = ahc->critical_sections;
1430                 for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
1431
1432                         if (cs->begin < seqaddr && cs->end >= seqaddr)
1433                                 break;
1434                 }
1435
1436                 if (i == ahc->num_critical_sections)
1437                         break;
1438
1439                 if (steps > AHC_MAX_STEPS) {
1440                         printf("%s: Infinite loop in critical section\n",
1441                                ahc_name(ahc));
1442                         ahc_dump_card_state(ahc);
1443                         panic("critical section loop");
1444                 }
1445
1446                 steps++;
1447                 if (stepping == FALSE) {
1448
1449                         /*
1450                          * Disable all interrupt sources so that the
1451                          * sequencer will not be stuck by a pausing
1452                          * interrupt condition while we attempt to
1453                          * leave a critical section.
1454                          */
1455                         simode0 = ahc_inb(ahc, SIMODE0);
1456                         ahc_outb(ahc, SIMODE0, 0);
1457                         simode1 = ahc_inb(ahc, SIMODE1);
1458                         if ((ahc->features & AHC_DT) != 0)
1459                                 /*
1460                                  * On DT class controllers, we
1461                                  * use the enhanced busfree logic.
1462                                  * Unfortunately we cannot re-enable
1463                                  * busfree detection within the
1464                                  * current connection, so we must
1465                                  * leave it on while single stepping.
1466                                  */
1467                                 ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
1468                         else
1469                                 ahc_outb(ahc, SIMODE1, 0);
1470                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1471                         ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
1472                         stepping = TRUE;
1473                 }
1474                 if ((ahc->features & AHC_DT) != 0) {
1475                         ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
1476                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1477                 }
1478                 ahc_outb(ahc, HCNTRL, ahc->unpause);
1479                 while (!ahc_is_paused(ahc))
1480                         ahc_delay(200);
1481         }
1482         if (stepping) {
1483                 ahc_outb(ahc, SIMODE0, simode0);
1484                 ahc_outb(ahc, SIMODE1, simode1);
1485                 ahc_outb(ahc, SEQCTL, ahc->seqctl);
1486         }
1487 }
1488
1489 /*
1490  * Clear any pending interrupt status.
1491  */
1492 void
1493 ahc_clear_intstat(struct ahc_softc *ahc)
1494 {
1495         /* Clear any interrupt conditions this may have caused */
1496         ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
1497                                 |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
1498                                 CLRREQINIT);
1499         ahc_flush_device_writes(ahc);
1500         ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
1501         ahc_flush_device_writes(ahc);
1502         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1503         ahc_flush_device_writes(ahc);
1504 }
1505
1506 /**************************** Debugging Routines ******************************/
1507 #ifdef AHC_DEBUG
1508 uint32_t ahc_debug = AHC_DEBUG_OPTS;
1509 #endif
1510
1511 void
1512 ahc_print_scb(struct scb *scb)
1513 {
1514         int i;
1515
1516         struct hardware_scb *hscb = scb->hscb;
1517
1518         printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
1519                (void *)scb,
1520                hscb->control,
1521                hscb->scsiid,
1522                hscb->lun,
1523                hscb->cdb_len);
1524         printf("Shared Data: ");
1525         for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
1526                 printf("%#02x", hscb->shared_data.cdb[i]);
1527         printf("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
1528                 ahc_le32toh(hscb->dataptr),
1529                 ahc_le32toh(hscb->datacnt),
1530                 ahc_le32toh(hscb->sgptr),
1531                 hscb->tag);
1532         if (scb->sg_count > 0) {
1533                 for (i = 0; i < scb->sg_count; i++) {
1534                         printf("sg[%d] - Addr 0x%x%x : Length %d\n",
1535                                i,
1536                                (ahc_le32toh(scb->sg_list[i].len) >> 24
1537                                 & SG_HIGH_ADDR_BITS),
1538                                ahc_le32toh(scb->sg_list[i].addr),
1539                                ahc_le32toh(scb->sg_list[i].len));
1540                 }
1541         }
1542 }
1543
1544 /************************* Transfer Negotiation *******************************/
1545 /*
1546  * Allocate per target mode instance (ID we respond to as a target)
1547  * transfer negotiation data structures.
1548  */
1549 static struct ahc_tmode_tstate *
1550 ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
1551 {
1552         struct ahc_tmode_tstate *master_tstate;
1553         struct ahc_tmode_tstate *tstate;
1554         int i;
1555
1556         master_tstate = ahc->enabled_targets[ahc->our_id];
1557         if (channel == 'B') {
1558                 scsi_id += 8;
1559                 master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
1560         }
1561         if (ahc->enabled_targets[scsi_id] != NULL
1562          && ahc->enabled_targets[scsi_id] != master_tstate)
1563                 panic("%s: ahc_alloc_tstate - Target already allocated",
1564                       ahc_name(ahc));
1565         tstate = kmalloc(sizeof(*tstate), M_DEVBUF, M_INTWAIT);
1566
1567         /*
1568          * If we have allocated a master tstate, copy user settings from
1569          * the master tstate (taken from SRAM or the EEPROM) for this
1570          * channel, but reset our current and goal settings to async/narrow
1571          * until an initiator talks to us.
1572          */
1573         if (master_tstate != NULL) {
1574                 memcpy(tstate, master_tstate, sizeof(*tstate));
1575                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
1576                 tstate->ultraenb = 0;
1577                 for (i = 0; i < AHC_NUM_TARGETS; i++) {
1578                         memset(&tstate->transinfo[i].curr, 0,
1579                               sizeof(tstate->transinfo[i].curr));
1580                         memset(&tstate->transinfo[i].goal, 0,
1581                               sizeof(tstate->transinfo[i].goal));
1582                 }
1583         } else
1584                 memset(tstate, 0, sizeof(*tstate));
1585         ahc->enabled_targets[scsi_id] = tstate;
1586         return (tstate);
1587 }
1588
1589 #ifdef AHC_TARGET_MODE
1590 /*
1591  * Free per target mode instance (ID we respond to as a target)
1592  * transfer negotiation data structures.
1593  */
1594 static void
1595 ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
1596 {
1597         struct ahc_tmode_tstate *tstate;
1598
1599         /*
1600          * Don't clean up our "master" tstate.
1601          * It has our default user settings.
1602          */
1603         if (((channel == 'B' && scsi_id == ahc->our_id_b)
1604           || (channel == 'A' && scsi_id == ahc->our_id))
1605          && force == FALSE)
1606                 return;
1607
1608         if (channel == 'B')
1609                 scsi_id += 8;
1610         tstate = ahc->enabled_targets[scsi_id];
1611         if (tstate != NULL)
1612                 kfree(tstate, M_DEVBUF);
1613         ahc->enabled_targets[scsi_id] = NULL;
1614 }
1615 #endif
1616
1617 /*
1618  * Called when we have an active connection to a target on the bus,
1619  * this function finds the nearest syncrate to the input period limited
1620  * by the capabilities of the bus connectivity of and sync settings for
1621  * the target.
1622  */
1623 struct ahc_syncrate *
1624 ahc_devlimited_syncrate(struct ahc_softc *ahc,
1625                         struct ahc_initiator_tinfo *tinfo,
1626                         u_int *period, u_int *ppr_options, role_t role)
1627 {
1628         struct  ahc_transinfo *transinfo;
1629         u_int   maxsync;
1630
1631         if ((ahc->features & AHC_ULTRA2) != 0) {
1632                 if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
1633                  && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
1634                         maxsync = AHC_SYNCRATE_DT;
1635                 } else {
1636                         maxsync = AHC_SYNCRATE_ULTRA;
1637                         /* Can't do DT on an SE bus */
1638                         *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1639                 }
1640         } else if ((ahc->features & AHC_ULTRA) != 0) {
1641                 maxsync = AHC_SYNCRATE_ULTRA;
1642         } else {
1643                 maxsync = AHC_SYNCRATE_FAST;
1644         }
1645         /*
1646          * Never allow a value higher than our current goal
1647          * period otherwise we may allow a target initiated
1648          * negotiation to go above the limit as set by the
1649          * user.  In the case of an initiator initiated
1650          * sync negotiation, we limit based on the user
1651          * setting.  This allows the system to still accept
1652          * incoming negotiations even if target initiated
1653          * negotiation is not performed.
1654          */
1655         if (role == ROLE_TARGET)
1656                 transinfo = &tinfo->user;
1657         else
1658                 transinfo = &tinfo->goal;
1659         *ppr_options &= transinfo->ppr_options;
1660         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
1661                 maxsync = MAX(maxsync, AHC_SYNCRATE_ULTRA2);
1662                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1663         }
1664         if (transinfo->period == 0) {
1665                 *period = 0;
1666                 *ppr_options = 0;
1667                 return (NULL);
1668         }
1669         *period = MAX(*period, transinfo->period);
1670         return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
1671 }
1672
1673 /*
1674  * Look up the valid period to SCSIRATE conversion in our table.
1675  * Return the period and offset that should be sent to the target
1676  * if this was the beginning of an SDTR.
1677  */
1678 struct ahc_syncrate *
1679 ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
1680                   u_int *ppr_options, u_int maxsync)
1681 {
1682         struct ahc_syncrate *syncrate;
1683
1684         if ((ahc->features & AHC_DT) == 0)
1685                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1686
1687         /* Skip all DT only entries if DT is not available */
1688         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
1689          && maxsync < AHC_SYNCRATE_ULTRA2)
1690                 maxsync = AHC_SYNCRATE_ULTRA2;
1691
1692         for (syncrate = &ahc_syncrates[maxsync];
1693              syncrate->rate != NULL;
1694              syncrate++) {
1695
1696                 /*
1697                  * The Ultra2 table doesn't go as low
1698                  * as for the Fast/Ultra cards.
1699                  */
1700                 if ((ahc->features & AHC_ULTRA2) != 0
1701                  && (syncrate->sxfr_u2 == 0))
1702                         break;
1703
1704                 if (*period <= syncrate->period) {
1705                         /*
1706                          * When responding to a target that requests
1707                          * sync, the requested rate may fall between
1708                          * two rates that we can output, but still be
1709                          * a rate that we can receive.  Because of this,
1710                          * we want to respond to the target with
1711                          * the same rate that it sent to us even
1712                          * if the period we use to send data to it
1713                          * is lower.  Only lower the response period
1714                          * if we must.
1715                          */
1716                         if (syncrate == &ahc_syncrates[maxsync])
1717                                 *period = syncrate->period;
1718
1719                         /*
1720                          * At some speeds, we only support
1721                          * ST transfers.
1722                          */
1723                         if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
1724                                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1725                         break;
1726                 }
1727         }
1728
1729         if ((*period == 0)
1730          || (syncrate->rate == NULL)
1731          || ((ahc->features & AHC_ULTRA2) != 0
1732           && (syncrate->sxfr_u2 == 0))) {
1733                 /* Use asynchronous transfers. */
1734                 *period = 0;
1735                 syncrate = NULL;
1736                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1737         }
1738         return (syncrate);
1739 }
1740
1741 /*
1742  * Convert from an entry in our syncrate table to the SCSI equivalent
1743  * sync "period" factor.
1744  */
1745 u_int
1746 ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
1747 {
1748         struct ahc_syncrate *syncrate;
1749
1750         if ((ahc->features & AHC_ULTRA2) != 0)
1751                 scsirate &= SXFR_ULTRA2;
1752         else
1753                 scsirate &= SXFR;
1754
1755         syncrate = &ahc_syncrates[maxsync];
1756         while (syncrate->rate != NULL) {
1757
1758                 if ((ahc->features & AHC_ULTRA2) != 0) {
1759                         if (syncrate->sxfr_u2 == 0)
1760                                 break;
1761                         else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
1762                                 return (syncrate->period);
1763                 } else if (scsirate == (syncrate->sxfr & SXFR)) {
1764                                 return (syncrate->period);
1765                 }
1766                 syncrate++;
1767         }
1768         return (0); /* async */
1769 }
1770
1771 /*
1772  * Truncate the given synchronous offset to a value the
1773  * current adapter type and syncrate are capable of.
1774  */
1775 void
1776 ahc_validate_offset(struct ahc_softc *ahc,
1777                     struct ahc_initiator_tinfo *tinfo,
1778                     struct ahc_syncrate *syncrate,
1779                     u_int *offset, int wide, role_t role)
1780 {
1781         u_int maxoffset;
1782
1783         /* Limit offset to what we can do */
1784         if (syncrate == NULL) {
1785                 maxoffset = 0;
1786         } else if ((ahc->features & AHC_ULTRA2) != 0) {
1787                 maxoffset = MAX_OFFSET_ULTRA2;
1788         } else {
1789                 if (wide)
1790                         maxoffset = MAX_OFFSET_16BIT;
1791                 else
1792                         maxoffset = MAX_OFFSET_8BIT;
1793         }
1794         *offset = MIN(*offset, maxoffset);
1795         if (tinfo != NULL) {
1796                 if (role == ROLE_TARGET)
1797                         *offset = MIN(*offset, tinfo->user.offset);
1798                 else
1799                         *offset = MIN(*offset, tinfo->goal.offset);
1800         }
1801 }
1802
1803 /*
1804  * Truncate the given transfer width parameter to a value the
1805  * current adapter type is capable of.
1806  */
1807 void
1808 ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
1809                    u_int *bus_width, role_t role)
1810 {
1811         switch (*bus_width) {
1812         default:
1813                 if (ahc->features & AHC_WIDE) {
1814                         /* Respond Wide */
1815                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
1816                         break;
1817                 }
1818                 /* FALLTHROUGH */
1819         case MSG_EXT_WDTR_BUS_8_BIT:
1820                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
1821                 break;
1822         }
1823         if (tinfo != NULL) {
1824                 if (role == ROLE_TARGET)
1825                         *bus_width = MIN(tinfo->user.width, *bus_width);
1826                 else
1827                         *bus_width = MIN(tinfo->goal.width, *bus_width);
1828         }
1829 }
1830
1831 /*
1832  * Update the bitmask of targets for which the controller should
1833  * negotiate with at the next convenient oportunity.  This currently
1834  * means the next time we send the initial identify messages for
1835  * a new transaction.
1836  */
1837 int
1838 ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
1839                        struct ahc_tmode_tstate *tstate,
1840                        struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
1841 {
1842         u_int auto_negotiate_orig;
1843
1844         auto_negotiate_orig = tstate->auto_negotiate;
1845         if (neg_type == AHC_NEG_ALWAYS) {
1846                 /*
1847                  * Force our "current" settings to be
1848                  * unknown so that unless a bus reset
1849                  * occurs the need to renegotiate is
1850                  * recorded persistently.
1851                  */
1852                 if ((ahc->features & AHC_WIDE) != 0)
1853                         tinfo->curr.width = AHC_WIDTH_UNKNOWN;
1854                 tinfo->curr.period = AHC_PERIOD_UNKNOWN;
1855                 tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
1856         }
1857         if (tinfo->curr.period != tinfo->goal.period
1858          || tinfo->curr.width != tinfo->goal.width
1859          || tinfo->curr.offset != tinfo->goal.offset
1860          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
1861          || (neg_type == AHC_NEG_IF_NON_ASYNC
1862           && (tinfo->goal.offset != 0
1863            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
1864            || tinfo->goal.ppr_options != 0)))
1865                 tstate->auto_negotiate |= devinfo->target_mask;
1866         else
1867                 tstate->auto_negotiate &= ~devinfo->target_mask;
1868
1869         return (auto_negotiate_orig != tstate->auto_negotiate);
1870 }
1871
1872 /*
1873  * Update the user/goal/curr tables of synchronous negotiation
1874  * parameters as well as, in the case of a current or active update,
1875  * any data structures on the host controller.  In the case of an
1876  * active update, the specified target is currently talking to us on
1877  * the bus, so the transfer parameter update must take effect
1878  * immediately.
1879  */
1880 void
1881 ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
1882                  struct ahc_syncrate *syncrate, u_int period,
1883                  u_int offset, u_int ppr_options, u_int type, int paused)
1884 {
1885         struct  ahc_initiator_tinfo *tinfo;
1886         struct  ahc_tmode_tstate *tstate;
1887         u_int   old_period;
1888         u_int   old_offset;
1889         u_int   old_ppr;
1890         int     active;
1891         int     update_needed;
1892
1893         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
1894         update_needed = 0;
1895
1896         if (syncrate == NULL) {
1897                 period = 0;
1898                 offset = 0;
1899         }
1900
1901         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
1902                                     devinfo->target, &tstate);
1903
1904         if ((type & AHC_TRANS_USER) != 0) {
1905                 tinfo->user.period = period;
1906                 tinfo->user.offset = offset;
1907                 tinfo->user.ppr_options = ppr_options;
1908         }
1909
1910         if ((type & AHC_TRANS_GOAL) != 0) {
1911                 tinfo->goal.period = period;
1912                 tinfo->goal.offset = offset;
1913                 tinfo->goal.ppr_options = ppr_options;
1914         }
1915
1916         old_period = tinfo->curr.period;
1917         old_offset = tinfo->curr.offset;
1918         old_ppr    = tinfo->curr.ppr_options;
1919
1920         if ((type & AHC_TRANS_CUR) != 0
1921          && (old_period != period
1922           || old_offset != offset
1923           || old_ppr != ppr_options)) {
1924                 u_int   scsirate;
1925
1926                 update_needed++;
1927                 scsirate = tinfo->scsirate;
1928                 if ((ahc->features & AHC_ULTRA2) != 0) {
1929
1930                         scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
1931                         if (syncrate != NULL) {
1932                                 scsirate |= syncrate->sxfr_u2;
1933                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
1934                                         scsirate |= ENABLE_CRC;
1935                                 else
1936                                         scsirate |= SINGLE_EDGE;
1937                         }
1938                 } else {
1939
1940                         scsirate &= ~(SXFR|SOFS);
1941                         /*
1942                          * Ensure Ultra mode is set properly for
1943                          * this target.
1944                          */
1945                         tstate->ultraenb &= ~devinfo->target_mask;
1946                         if (syncrate != NULL) {
1947                                 if (syncrate->sxfr & ULTRA_SXFR) {
1948                                         tstate->ultraenb |=
1949                                                 devinfo->target_mask;
1950                                 }
1951                                 scsirate |= syncrate->sxfr & SXFR;
1952                                 scsirate |= offset & SOFS;
1953                         }
1954                         if (active) {
1955                                 u_int sxfrctl0;
1956
1957                                 sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
1958                                 sxfrctl0 &= ~FAST20;
1959                                 if (tstate->ultraenb & devinfo->target_mask)
1960                                         sxfrctl0 |= FAST20;
1961                                 ahc_outb(ahc, SXFRCTL0, sxfrctl0);
1962                         }
1963                 }
1964                 if (active) {
1965                         ahc_outb(ahc, SCSIRATE, scsirate);
1966                         if ((ahc->features & AHC_ULTRA2) != 0)
1967                                 ahc_outb(ahc, SCSIOFFSET, offset);
1968                 }
1969
1970                 tinfo->scsirate = scsirate;
1971                 tinfo->curr.period = period;
1972                 tinfo->curr.offset = offset;
1973                 tinfo->curr.ppr_options = ppr_options;
1974
1975                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
1976                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
1977                 if (bootverbose) {
1978                         if (offset != 0) {
1979                                 printf("%s: target %d synchronous at %sMHz%s, "
1980                                        "offset = 0x%x\n", ahc_name(ahc),
1981                                        devinfo->target, syncrate->rate,
1982                                        (ppr_options & MSG_EXT_PPR_DT_REQ)
1983                                        ? " DT" : "", offset);
1984                         } else {
1985                                 printf("%s: target %d using "
1986                                        "asynchronous transfers\n",
1987                                        ahc_name(ahc), devinfo->target);
1988                         }
1989                 }
1990         }
1991
1992         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
1993                                                 tinfo, AHC_NEG_TO_GOAL);
1994
1995         if (update_needed)
1996                 ahc_update_pending_scbs(ahc);
1997 }
1998
1999 /*
2000  * Update the user/goal/curr tables of wide negotiation
2001  * parameters as well as, in the case of a current or active update,
2002  * any data structures on the host controller.  In the case of an
2003  * active update, the specified target is currently talking to us on
2004  * the bus, so the transfer parameter update must take effect
2005  * immediately.
2006  */
2007 void
2008 ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2009               u_int width, u_int type, int paused)
2010 {
2011         struct  ahc_initiator_tinfo *tinfo;
2012         struct  ahc_tmode_tstate *tstate;
2013         u_int   oldwidth;
2014         int     active;
2015         int     update_needed;
2016
2017         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2018         update_needed = 0;
2019         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2020                                     devinfo->target, &tstate);
2021
2022         if ((type & AHC_TRANS_USER) != 0)
2023                 tinfo->user.width = width;
2024
2025         if ((type & AHC_TRANS_GOAL) != 0)
2026                 tinfo->goal.width = width;
2027
2028         oldwidth = tinfo->curr.width;
2029         if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2030                 u_int   scsirate;
2031
2032                 update_needed++;
2033                 scsirate =  tinfo->scsirate;
2034                 scsirate &= ~WIDEXFER;
2035                 if (width == MSG_EXT_WDTR_BUS_16_BIT)
2036                         scsirate |= WIDEXFER;
2037
2038                 tinfo->scsirate = scsirate;
2039
2040                 if (active)
2041                         ahc_outb(ahc, SCSIRATE, scsirate);
2042
2043                 tinfo->curr.width = width;
2044
2045                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2046                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
2047                 if (bootverbose) {
2048                         printf("%s: target %d using %dbit transfers\n",
2049                                ahc_name(ahc), devinfo->target,
2050                                8 * (0x01 << width));
2051                 }
2052         }
2053
2054         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2055                                                 tinfo, AHC_NEG_TO_GOAL);
2056         if (update_needed)
2057                 ahc_update_pending_scbs(ahc);
2058 }
2059
2060 /*
2061  * Update the current state of tagged queuing for a given target.
2062  */
2063 void
2064 ahc_set_tags(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2065              ahc_queue_alg alg)
2066 {
2067         ahc_platform_set_tags(ahc, devinfo, alg);
2068         ahc_send_async(ahc, devinfo->channel, devinfo->target,
2069                        devinfo->lun, AC_TRANSFER_NEG, &alg);
2070 }
2071
2072 /*
2073  * When the transfer settings for a connection change, update any
2074  * in-transit SCBs to contain the new data so the hardware will
2075  * be set correctly during future (re)selections.
2076  */
2077 static void
2078 ahc_update_pending_scbs(struct ahc_softc *ahc)
2079 {
2080         struct  scb *pending_scb;
2081         int     pending_scb_count;
2082         int     i;
2083         int     paused;
2084         u_int   saved_scbptr;
2085
2086         /*
2087          * Traverse the pending SCB list and ensure that all of the
2088          * SCBs there have the proper settings.
2089          */
2090         pending_scb_count = 0;
2091         LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2092                 struct ahc_devinfo devinfo;
2093                 struct hardware_scb *pending_hscb;
2094                 struct ahc_initiator_tinfo *tinfo;
2095                 struct ahc_tmode_tstate *tstate;
2096
2097                 ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2098                 tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2099                                             devinfo.our_scsiid,
2100                                             devinfo.target, &tstate);
2101                 pending_hscb = pending_scb->hscb;
2102                 pending_hscb->control &= ~ULTRAENB;
2103                 if ((tstate->ultraenb & devinfo.target_mask) != 0)
2104                         pending_hscb->control |= ULTRAENB;
2105                 pending_hscb->scsirate = tinfo->scsirate;
2106                 pending_hscb->scsioffset = tinfo->curr.offset;
2107                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2108                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2109                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2110                         pending_hscb->control &= ~MK_MESSAGE;
2111                 }
2112                 ahc_sync_scb(ahc, pending_scb,
2113                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2114                 pending_scb_count++;
2115         }
2116
2117         if (pending_scb_count == 0)
2118                 return;
2119
2120         if (ahc_is_paused(ahc)) {
2121                 paused = 1;
2122         } else {
2123                 paused = 0;
2124                 ahc_pause(ahc);
2125         }
2126
2127         saved_scbptr = ahc_inb(ahc, SCBPTR);
2128         /* Ensure that the hscbs down on the card match the new information */
2129         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2130                 struct  hardware_scb *pending_hscb;
2131                 u_int   control;
2132                 u_int   scb_tag;
2133
2134                 ahc_outb(ahc, SCBPTR, i);
2135                 scb_tag = ahc_inb(ahc, SCB_TAG);
2136                 pending_scb = ahc_lookup_scb(ahc, scb_tag);
2137                 if (pending_scb == NULL)
2138                         continue;
2139
2140                 pending_hscb = pending_scb->hscb;
2141                 control = ahc_inb(ahc, SCB_CONTROL);
2142                 control &= ~(ULTRAENB|MK_MESSAGE);
2143                 control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2144                 ahc_outb(ahc, SCB_CONTROL, control);
2145                 ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2146                 ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2147         }
2148         ahc_outb(ahc, SCBPTR, saved_scbptr);
2149
2150         if (paused == 0)
2151                 ahc_unpause(ahc);
2152 }
2153
2154 /**************************** Pathing Information *****************************/
2155 static void
2156 ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2157 {
2158         u_int   saved_scsiid;
2159         role_t  role;
2160         int     our_id;
2161
2162         if (ahc_inb(ahc, SSTAT0) & TARGET)
2163                 role = ROLE_TARGET;
2164         else
2165                 role = ROLE_INITIATOR;
2166
2167         if (role == ROLE_TARGET
2168          && (ahc->features & AHC_MULTI_TID) != 0
2169          && (ahc_inb(ahc, SEQ_FLAGS)
2170            & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2171                 /* We were selected, so pull our id from TARGIDIN */
2172                 our_id = ahc_inb(ahc, TARGIDIN) & OID;
2173         } else if ((ahc->features & AHC_ULTRA2) != 0)
2174                 our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2175         else
2176                 our_id = ahc_inb(ahc, SCSIID) & OID;
2177
2178         saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2179         ahc_compile_devinfo(devinfo,
2180                             our_id,
2181                             SCSIID_TARGET(ahc, saved_scsiid),
2182                             ahc_inb(ahc, SAVED_LUN),
2183                             SCSIID_CHANNEL(ahc, saved_scsiid),
2184                             role);
2185 }
2186
2187 struct ahc_phase_table_entry*
2188 ahc_lookup_phase_entry(int phase)
2189 {
2190         struct ahc_phase_table_entry *entry;
2191         struct ahc_phase_table_entry *last_entry;
2192
2193         /*
2194          * num_phases doesn't include the default entry which
2195          * will be returned if the phase doesn't match.
2196          */
2197         last_entry = &ahc_phase_table[num_phases];
2198         for (entry = ahc_phase_table; entry < last_entry; entry++) {
2199                 if (phase == entry->phase)
2200                         break;
2201         }
2202         return (entry);
2203 }
2204
2205 void
2206 ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2207                     u_int lun, char channel, role_t role)
2208 {
2209         devinfo->our_scsiid = our_id;
2210         devinfo->target = target;
2211         devinfo->lun = lun;
2212         devinfo->target_offset = target;
2213         devinfo->channel = channel;
2214         devinfo->role = role;
2215         if (channel == 'B')
2216                 devinfo->target_offset += 8;
2217         devinfo->target_mask = (0x01 << devinfo->target_offset);
2218 }
2219
2220 void
2221 ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2222 {
2223         printf("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2224                devinfo->target, devinfo->lun);
2225 }
2226
2227 static void
2228 ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2229                 struct scb *scb)
2230 {
2231         role_t  role;
2232         int     our_id;
2233
2234         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2235         role = ROLE_INITIATOR;
2236         if ((scb->flags & SCB_TARGET_SCB) != 0)
2237                 role = ROLE_TARGET;
2238         ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2239                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2240 }
2241
2242
2243 /************************ Message Phase Processing ****************************/
2244 static void
2245 ahc_assert_atn(struct ahc_softc *ahc)
2246 {
2247         u_int scsisigo;
2248
2249         scsisigo = ATNO;
2250         if ((ahc->features & AHC_DT) == 0)
2251                 scsisigo |= ahc_inb(ahc, SCSISIGI);
2252         ahc_outb(ahc, SCSISIGO, scsisigo);
2253 }
2254
2255 /*
2256  * When an initiator transaction with the MK_MESSAGE flag either reconnects
2257  * or enters the initial message out phase, we are interrupted.  Fill our
2258  * outgoing message buffer with the appropriate message and beging handing
2259  * the message phase(s) manually.
2260  */
2261 static void
2262 ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2263                            struct scb *scb)
2264 {
2265         /*
2266          * To facilitate adding multiple messages together,
2267          * each routine should increment the index and len
2268          * variables instead of setting them explicitly.
2269          */
2270         ahc->msgout_index = 0;
2271         ahc->msgout_len = 0;
2272
2273         if ((scb->flags & SCB_DEVICE_RESET) == 0
2274          && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2275                 u_int identify_msg;
2276
2277                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2278                 if ((scb->hscb->control & DISCENB) != 0)
2279                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
2280                 ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2281                 ahc->msgout_len++;
2282
2283                 if ((scb->hscb->control & TAG_ENB) != 0) {
2284                         ahc->msgout_buf[ahc->msgout_index++] =
2285                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2286                         ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2287                         ahc->msgout_len += 2;
2288                 }
2289         }
2290
2291         if (scb->flags & SCB_DEVICE_RESET) {
2292                 ahc->msgout_buf[ahc->msgout_index++] = MSG_BUS_DEV_RESET;
2293                 ahc->msgout_len++;
2294                 ahc_print_path(ahc, scb);
2295                 printf("Bus Device Reset Message Sent\n");
2296                 /*
2297                  * Clear our selection hardware in advance of
2298                  * the busfree.  We may have an entry in the waiting
2299                  * Q for this target, and we don't want to go about
2300                  * selecting while we handle the busfree and blow it
2301                  * away.
2302                  */
2303                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2304         } else if ((scb->flags & SCB_ABORT) != 0) {
2305                 if ((scb->hscb->control & TAG_ENB) != 0)
2306                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT_TAG;
2307                 else
2308                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT;
2309                 ahc->msgout_len++;
2310                 ahc_print_path(ahc, scb);
2311                 printf("Abort%s Message Sent\n",
2312                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2313                 /*
2314                  * Clear our selection hardware in advance of
2315                  * the busfree.  We may have an entry in the waiting
2316                  * Q for this target, and we don't want to go about
2317                  * selecting while we handle the busfree and blow it
2318                  * away.
2319                  */
2320                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2321         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2322                 ahc_build_transfer_msg(ahc, devinfo);
2323         } else {
2324                 printf("ahc_intr: AWAITING_MSG for an SCB that "
2325                        "does not have a waiting message\n");
2326                 printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2327                        devinfo->target_mask);
2328                 panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2329                       "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2330                       ahc_inb(ahc, MSG_OUT), scb->flags);
2331         }
2332
2333         /*
2334          * Clear the MK_MESSAGE flag from the SCB so we aren't
2335          * asked to send this message again.
2336          */
2337         ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2338         scb->hscb->control &= ~MK_MESSAGE;
2339         ahc->msgout_index = 0;
2340         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2341 }
2342
2343 /*
2344  * Build an appropriate transfer negotiation message for the
2345  * currently active target.
2346  */
2347 static void
2348 ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2349 {
2350         /*
2351          * We need to initiate transfer negotiations.
2352          * If our current and goal settings are identical,
2353          * we want to renegotiate due to a check condition.
2354          */
2355         struct  ahc_initiator_tinfo *tinfo;
2356         struct  ahc_tmode_tstate *tstate;
2357         struct  ahc_syncrate *rate;
2358         int     dowide;
2359         int     dosync;
2360         int     doppr;
2361         u_int   period;
2362         u_int   ppr_options;
2363         u_int   offset;
2364
2365         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2366                                     devinfo->target, &tstate);
2367         /*
2368          * Filter our period based on the current connection.
2369          * If we can't perform DT transfers on this segment (not in LVD
2370          * mode for instance), then our decision to issue a PPR message
2371          * may change.
2372          */
2373         period = tinfo->goal.period;
2374         offset = tinfo->goal.offset;
2375         ppr_options = tinfo->goal.ppr_options;
2376         /* Target initiated PPR is not allowed in the SCSI spec */
2377         if (devinfo->role == ROLE_TARGET)
2378                 ppr_options = 0;
2379         rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2380                                        &ppr_options, devinfo->role);
2381         dowide = tinfo->curr.width != tinfo->goal.width;
2382         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2383         /*
2384          * Only use PPR if we have options that need it, even if the device
2385          * claims to support it.  There might be an expander in the way
2386          * that doesn't.
2387          */
2388         doppr = ppr_options != 0;
2389
2390         if (!dowide && !dosync && !doppr) {
2391                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
2392                 dosync = tinfo->goal.offset != 0;
2393         }
2394
2395         if (!dowide && !dosync && !doppr) {
2396                 /*
2397                  * Force async with a WDTR message if we have a wide bus,
2398                  * or just issue an SDTR with a 0 offset.
2399                  */
2400                 if ((ahc->features & AHC_WIDE) != 0)
2401                         dowide = 1;
2402                 else
2403                         dosync = 1;
2404
2405                 if (bootverbose) {
2406                         ahc_print_devinfo(ahc, devinfo);
2407                         printf("Ensuring async\n");
2408                 }
2409         }
2410
2411         /* Target initiated PPR is not allowed in the SCSI spec */
2412         if (devinfo->role == ROLE_TARGET)
2413                 doppr = 0;
2414
2415         /*
2416          * Both the PPR message and SDTR message require the
2417          * goal syncrate to be limited to what the target device
2418          * is capable of handling (based on whether an LVD->SE
2419          * expander is on the bus), so combine these two cases.
2420          * Regardless, guarantee that if we are using WDTR and SDTR
2421          * messages that WDTR comes first.
2422          */
2423         if (doppr || (dosync && !dowide)) {
2424
2425                 offset = tinfo->goal.offset;
2426                 ahc_validate_offset(ahc, tinfo, rate, &offset,
2427                                     doppr ? tinfo->goal.width
2428                                           : tinfo->curr.width,
2429                                     devinfo->role);
2430                 if (doppr) {
2431                         ahc_construct_ppr(ahc, devinfo, period, offset,
2432                                           tinfo->goal.width, ppr_options);
2433                 } else {
2434                         ahc_construct_sdtr(ahc, devinfo, period, offset);
2435                 }
2436         } else {
2437                 ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
2438         }
2439 }
2440
2441 /*
2442  * Build a synchronous negotiation message in our message
2443  * buffer based on the input parameters.
2444  */
2445 static void
2446 ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2447                    u_int period, u_int offset)
2448 {
2449         if (offset == 0)
2450                 period = AHC_ASYNC_XFER_PERIOD;
2451         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXTENDED;
2452         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_SDTR_LEN;
2453         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_SDTR;
2454         ahc->msgout_buf[ahc->msgout_index++] = period;
2455         ahc->msgout_buf[ahc->msgout_index++] = offset;
2456         ahc->msgout_len += 5;
2457         if (bootverbose) {
2458                 printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
2459                        ahc_name(ahc), devinfo->channel, devinfo->target,
2460                        devinfo->lun, period, offset);
2461         }
2462 }
2463
2464 /*
2465  * Build a wide negotiation message in our message
2466  * buffer based on the input parameters.
2467  */
2468 static void
2469 ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2470                    u_int bus_width)
2471 {
2472         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXTENDED;
2473         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_WDTR_LEN;
2474         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_WDTR;
2475         ahc->msgout_buf[ahc->msgout_index++] = bus_width;
2476         ahc->msgout_len += 4;
2477         if (bootverbose) {
2478                 printf("(%s:%c:%d:%d): Sending WDTR %x\n",
2479                        ahc_name(ahc), devinfo->channel, devinfo->target,
2480                        devinfo->lun, bus_width);
2481         }
2482 }
2483
2484 /*
2485  * Build a parallel protocol request message in our message
2486  * buffer based on the input parameters.
2487  */
2488 static void
2489 ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2490                   u_int period, u_int offset, u_int bus_width,
2491                   u_int ppr_options)
2492 {
2493         if (offset == 0)
2494                 period = AHC_ASYNC_XFER_PERIOD;
2495         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXTENDED;
2496         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_PPR_LEN;
2497         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_PPR;
2498         ahc->msgout_buf[ahc->msgout_index++] = period;
2499         ahc->msgout_buf[ahc->msgout_index++] = 0;
2500         ahc->msgout_buf[ahc->msgout_index++] = offset;
2501         ahc->msgout_buf[ahc->msgout_index++] = bus_width;
2502         ahc->msgout_buf[ahc->msgout_index++] = ppr_options;
2503         ahc->msgout_len += 8;
2504         if (bootverbose) {
2505                 printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
2506                        "offset %x, ppr_options %x\n", ahc_name(ahc),
2507                        devinfo->channel, devinfo->target, devinfo->lun,
2508                        bus_width, period, offset, ppr_options);
2509         }
2510 }
2511
2512 /*
2513  * Clear any active message state.
2514  */
2515 static void
2516 ahc_clear_msg_state(struct ahc_softc *ahc)
2517 {
2518         ahc->msgout_len = 0;
2519         ahc->msgin_index = 0;
2520         ahc->msg_type = MSG_TYPE_NONE;
2521         if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
2522                 /*
2523                  * The target didn't care to respond to our
2524                  * message request, so clear ATN.
2525                  */
2526                 ahc_outb(ahc, CLRSINT1, CLRATNO);
2527         }
2528         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
2529         ahc_outb(ahc, SEQ_FLAGS2,
2530                  ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
2531 }
2532
2533 static void
2534 ahc_handle_proto_violation(struct ahc_softc *ahc)
2535 {
2536         struct  ahc_devinfo devinfo;
2537         struct  scb *scb;
2538         u_int   scbid;
2539         u_int   seq_flags;
2540         u_int   curphase;
2541         u_int   lastphase;
2542         int     found;
2543
2544         ahc_fetch_devinfo(ahc, &devinfo);
2545         scbid = ahc_inb(ahc, SCB_TAG);
2546         scb = ahc_lookup_scb(ahc, scbid);
2547         seq_flags = ahc_inb(ahc, SEQ_FLAGS);
2548         curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
2549         lastphase = ahc_inb(ahc, LASTPHASE);
2550         if ((seq_flags & NOT_IDENTIFIED) != 0) {
2551
2552                 /*
2553                  * The reconnecting target either did not send an
2554                  * identify message, or did, but we didn't find an SCB
2555                  * to match.
2556                  */
2557                 ahc_print_devinfo(ahc, &devinfo);
2558                 printf("Target did not send an IDENTIFY message. "
2559                        "LASTPHASE = 0x%x.\n", lastphase);
2560                 scb = NULL;
2561         } else if (scb == NULL) {
2562                 /*
2563                  * We don't seem to have an SCB active for this
2564                  * transaction.  Print an error and reset the bus.
2565                  */
2566                 ahc_print_devinfo(ahc, &devinfo);
2567                 printf("No SCB found during protocol violation\n");
2568                 goto proto_violation_reset;
2569         } else {
2570                 ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
2571                 if ((seq_flags & NO_CDB_SENT) != 0) {
2572                         ahc_print_path(ahc, scb);
2573                         printf("No or incomplete CDB sent to device.\n");
2574                 } else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
2575                         /*
2576                          * The target never bothered to provide status to
2577                          * us prior to completing the command.  Since we don't
2578                          * know the disposition of this command, we must attempt
2579                          * to abort it.  Assert ATN and prepare to send an abort
2580                          * message.
2581                          */
2582                         ahc_print_path(ahc, scb);
2583                         printf("Completed command without status.\n");
2584                 } else {
2585                         ahc_print_path(ahc, scb);
2586                         printf("Unknown protocol violation.\n");
2587                         ahc_dump_card_state(ahc);
2588                 }
2589         }
2590         if ((lastphase & ~P_DATAIN_DT) == 0
2591          || lastphase == P_COMMAND) {
2592 proto_violation_reset:
2593                 /*
2594                  * Target either went directly to data/command
2595                  * phase or didn't respond to our ATN.
2596                  * The only safe thing to do is to blow
2597                  * it away with a bus reset.
2598                  */
2599                 found = ahc_reset_channel(ahc, 'A', TRUE);
2600                 printf("%s: Issued Channel %c Bus Reset. "
2601                        "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
2602         } else {
2603                 /*
2604                  * Leave the selection hardware off in case
2605                  * this abort attempt will affect yet to
2606                  * be sent commands.
2607                  */
2608                 ahc_outb(ahc, SCSISEQ,
2609                          ahc_inb(ahc, SCSISEQ) & ~ENSELO);
2610                 ahc_assert_atn(ahc);
2611                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
2612                 if (scb == NULL) {
2613                         ahc_print_devinfo(ahc, &devinfo);
2614                         ahc->msgout_buf[0] = MSG_ABORT_TASK;
2615                         ahc->msgout_len = 1;
2616                         ahc->msgout_index = 0;
2617                         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2618                 } else {
2619                         ahc_print_path(ahc, scb);
2620                         scb->flags |= SCB_ABORT;
2621                 }
2622                 printf("Protocol violation %s.  Attempting to abort.\n",
2623                        ahc_lookup_phase_entry(curphase)->phasemsg);
2624         }
2625 }
2626
2627 /*
2628  * Manual message loop handler.
2629  */
2630 static void
2631 ahc_handle_message_phase(struct ahc_softc *ahc)
2632 {
2633         struct  ahc_devinfo devinfo;
2634         u_int   bus_phase;
2635         int     end_session;
2636
2637         ahc_fetch_devinfo(ahc, &devinfo);
2638         end_session = FALSE;
2639         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
2640
2641 reswitch:
2642         switch (ahc->msg_type) {
2643         case MSG_TYPE_INITIATOR_MSGOUT:
2644         {
2645                 int lastbyte;
2646                 int phasemis;
2647                 int msgdone;
2648
2649                 if (ahc->msgout_len == 0)
2650                         panic("HOST_MSG_LOOP interrupt with no active message");
2651
2652 #ifdef AHC_DEBUG
2653                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2654                         ahc_print_devinfo(ahc, &devinfo);
2655                         printf("INITIATOR_MSG_OUT");
2656                 }
2657 #endif
2658                 phasemis = bus_phase != P_MESGOUT;
2659                 if (phasemis) {
2660 #ifdef AHC_DEBUG
2661                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2662                                 printf(" PHASEMIS %s\n",
2663                                        ahc_lookup_phase_entry(bus_phase)
2664                                                              ->phasemsg);
2665                         }
2666 #endif
2667                         if (bus_phase == P_MESGIN) {
2668                                 /*
2669                                  * Change gears and see if
2670                                  * this messages is of interest to
2671                                  * us or should be passed back to
2672                                  * the sequencer.
2673                                  */
2674                                 ahc_outb(ahc, CLRSINT1, CLRATNO);
2675                                 ahc->send_msg_perror = FALSE;
2676                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
2677                                 ahc->msgin_index = 0;
2678                                 goto reswitch;
2679                         }
2680                         end_session = TRUE;
2681                         break;
2682                 }
2683
2684                 if (ahc->send_msg_perror) {
2685                         ahc_outb(ahc, CLRSINT1, CLRATNO);
2686                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2687 #ifdef AHC_DEBUG
2688                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2689                                 printf(" byte 0x%x\n", ahc->send_msg_perror);
2690 #endif
2691                         ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
2692                         break;
2693                 }
2694
2695                 msgdone = ahc->msgout_index == ahc->msgout_len;
2696                 if (msgdone) {
2697                         /*
2698                          * The target has requested a retry.
2699                          * Re-assert ATN, reset our message index to
2700                          * 0, and try again.
2701                          */
2702                         ahc->msgout_index = 0;
2703                         ahc_assert_atn(ahc);
2704                 }
2705
2706                 lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
2707                 if (lastbyte) {
2708                         /* Last byte is signified by dropping ATN */
2709                         ahc_outb(ahc, CLRSINT1, CLRATNO);
2710                 }
2711
2712                 /*
2713                  * Clear our interrupt status and present
2714                  * the next byte on the bus.
2715                  */
2716                 ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2717 #ifdef AHC_DEBUG
2718                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2719                         printf(" byte 0x%x\n",
2720                                ahc->msgout_buf[ahc->msgout_index]);
2721 #endif
2722                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
2723                 break;
2724         }
2725         case MSG_TYPE_INITIATOR_MSGIN:
2726         {
2727                 int phasemis;
2728                 int message_done;
2729
2730 #ifdef AHC_DEBUG
2731                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2732                         ahc_print_devinfo(ahc, &devinfo);
2733                         printf("INITIATOR_MSG_IN");
2734                 }
2735 #endif
2736                 phasemis = bus_phase != P_MESGIN;
2737                 if (phasemis) {
2738 #ifdef AHC_DEBUG
2739                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2740                                 printf(" PHASEMIS %s\n",
2741                                        ahc_lookup_phase_entry(bus_phase)
2742                                                              ->phasemsg);
2743                         }
2744 #endif
2745                         ahc->msgin_index = 0;
2746                         if (bus_phase == P_MESGOUT
2747                          && (ahc->send_msg_perror == TRUE
2748                           || (ahc->msgout_len != 0
2749                            && ahc->msgout_index == 0))) {
2750                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2751                                 goto reswitch;
2752                         }
2753                         end_session = TRUE;
2754                         break;
2755                 }
2756
2757                 /* Pull the byte in without acking it */
2758                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
2759 #ifdef AHC_DEBUG
2760                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2761                         printf(" byte 0x%x\n",
2762                                ahc->msgin_buf[ahc->msgin_index]);
2763 #endif
2764
2765                 message_done = ahc_parse_msg(ahc, &devinfo);
2766
2767                 if (message_done) {
2768                         /*
2769                          * Clear our incoming message buffer in case there
2770                          * is another message following this one.
2771                          */
2772                         ahc->msgin_index = 0;
2773
2774                         /*
2775                          * If this message illicited a response,
2776                          * assert ATN so the target takes us to the
2777                          * message out phase.
2778                          */
2779                         if (ahc->msgout_len != 0) {
2780 #ifdef AHC_DEBUG
2781                                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2782                                         ahc_print_devinfo(ahc, &devinfo);
2783                                         printf("Asserting ATN for response\n");
2784                                 }
2785 #endif
2786                                 ahc_assert_atn(ahc);
2787                         }
2788                 } else
2789                         ahc->msgin_index++;
2790
2791                 if (message_done == MSGLOOP_TERMINATED) {
2792                         end_session = TRUE;
2793                 } else {
2794                         /* Ack the byte */
2795                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2796                         ahc_inb(ahc, SCSIDATL);
2797                 }
2798                 break;
2799         }
2800         case MSG_TYPE_TARGET_MSGIN:
2801         {
2802                 int msgdone;
2803                 int msgout_request;
2804
2805                 if (ahc->msgout_len == 0)
2806                         panic("Target MSGIN with no active message");
2807
2808                 /*
2809                  * If we interrupted a mesgout session, the initiator
2810                  * will not know this until our first REQ.  So, we
2811                  * only honor mesgout requests after we've sent our
2812                  * first byte.
2813                  */
2814                 if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
2815                  && ahc->msgout_index > 0)
2816                         msgout_request = TRUE;
2817                 else
2818                         msgout_request = FALSE;
2819
2820                 if (msgout_request) {
2821
2822                         /*
2823                          * Change gears and see if
2824                          * this messages is of interest to
2825                          * us or should be passed back to
2826                          * the sequencer.
2827                          */
2828                         ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
2829                         ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
2830                         ahc->msgin_index = 0;
2831                         /* Dummy read to REQ for first byte */
2832                         ahc_inb(ahc, SCSIDATL);
2833                         ahc_outb(ahc, SXFRCTL0,
2834                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2835                         break;
2836                 }
2837
2838                 msgdone = ahc->msgout_index == ahc->msgout_len;
2839                 if (msgdone) {
2840                         ahc_outb(ahc, SXFRCTL0,
2841                                  ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
2842                         end_session = TRUE;
2843                         break;
2844                 }
2845
2846                 /*
2847                  * Present the next byte on the bus.
2848                  */
2849                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2850                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
2851                 break;
2852         }
2853         case MSG_TYPE_TARGET_MSGOUT:
2854         {
2855                 int lastbyte;
2856                 int msgdone;
2857
2858                 /*
2859                  * The initiator signals that this is
2860                  * the last byte by dropping ATN.
2861                  */
2862                 lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
2863
2864                 /*
2865                  * Read the latched byte, but turn off SPIOEN first
2866                  * so that we don't inadvertently cause a REQ for the
2867                  * next byte.
2868                  */
2869                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
2870                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
2871                 msgdone = ahc_parse_msg(ahc, &devinfo);
2872                 if (msgdone == MSGLOOP_TERMINATED) {
2873                         /*
2874                          * The message is *really* done in that it caused
2875                          * us to go to bus free.  The sequencer has already
2876                          * been reset at this point, so pull the ejection
2877                          * handle.
2878                          */
2879                         return;
2880                 }
2881
2882                 ahc->msgin_index++;
2883
2884                 /*
2885                  * XXX Read spec about initiator dropping ATN too soon
2886                  *     and use msgdone to detect it.
2887                  */
2888                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
2889                         ahc->msgin_index = 0;
2890
2891                         /*
2892                          * If this message illicited a response, transition
2893                          * to the Message in phase and send it.
2894                          */
2895                         if (ahc->msgout_len != 0) {
2896                                 ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
2897                                 ahc_outb(ahc, SXFRCTL0,
2898                                          ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2899                                 ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
2900                                 ahc->msgin_index = 0;
2901                                 break;
2902                         }
2903                 }
2904
2905                 if (lastbyte)
2906                         end_session = TRUE;
2907                 else {
2908                         /* Ask for the next byte. */
2909                         ahc_outb(ahc, SXFRCTL0,
2910                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2911                 }
2912
2913                 break;
2914         }
2915         default:
2916                 panic("Unknown REQINIT message type");
2917         }
2918
2919         if (end_session) {
2920                 ahc_clear_msg_state(ahc);
2921                 ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
2922         } else
2923                 ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
2924 }
2925
2926 /*
2927  * See if we sent a particular extended message to the target.
2928  * If "full" is true, return true only if the target saw the full
2929  * message.  If "full" is false, return true if the target saw at
2930  * least the first byte of the message.
2931  */
2932 static int
2933 ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
2934 {
2935         int found;
2936         u_int index;
2937
2938         found = FALSE;
2939         index = 0;
2940
2941         while (index < ahc->msgout_len) {
2942                 if (ahc->msgout_buf[index] == MSG_EXTENDED) {
2943                         u_int end_index;
2944
2945                         end_index = index + 1 + ahc->msgout_buf[index + 1];
2946                         if (ahc->msgout_buf[index+2] == msgval
2947                          && type == AHCMSG_EXT) {
2948
2949                                 if (full) {
2950                                         if (ahc->msgout_index > end_index)
2951                                                 found = TRUE;
2952                                 } else if (ahc->msgout_index > index)
2953                                         found = TRUE;
2954                         }
2955                         index = end_index;
2956                 } else if (ahc->msgout_buf[index] >= MSG_SIMPLE_TASK
2957                         && ahc->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
2958
2959                         /* Skip tag type and tag id or residue param*/
2960                         index += 2;
2961                 } else {
2962                         /* Single byte message */
2963                         if (type == AHCMSG_1B
2964                          && ahc->msgout_buf[index] == msgval
2965                          && ahc->msgout_index > index)
2966                                 found = TRUE;
2967                         index++;
2968                 }
2969
2970                 if (found)
2971                         break;
2972         }
2973         return (found);
2974 }
2975
2976 /*
2977  * Wait for a complete incoming message, parse it, and respond accordingly.
2978  */
2979 static int
2980 ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2981 {
2982         struct  ahc_initiator_tinfo *tinfo;
2983         struct  ahc_tmode_tstate *tstate;
2984         int     reject;
2985         int     done;
2986         int     response;
2987         u_int   targ_scsirate;
2988
2989         done = MSGLOOP_IN_PROG;
2990         response = FALSE;
2991         reject = FALSE;
2992         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2993                                     devinfo->target, &tstate);
2994         targ_scsirate = tinfo->scsirate;
2995
2996         /*
2997          * Parse as much of the message as is available,
2998          * rejecting it if we don't support it.  When
2999          * the entire message is available and has been
3000          * handled, return MSGLOOP_MSGCOMPLETE, indicating
3001          * that we have parsed an entire message.
3002          *
3003          * In the case of extended messages, we accept the length
3004          * byte outright and perform more checking once we know the
3005          * extended message type.
3006          */
3007         switch (ahc->msgin_buf[0]) {
3008         case MSG_DISCONNECT:
3009         case MSG_SAVEDATAPOINTER:
3010         case MSG_CMDCOMPLETE:
3011         case MSG_RESTOREPOINTERS:
3012         case MSG_IGN_WIDE_RESIDUE:
3013                 /*
3014                  * End our message loop as these are messages
3015                  * the sequencer handles on its own.
3016                  */
3017                 done = MSGLOOP_TERMINATED;
3018                 break;
3019         case MSG_MESSAGE_REJECT:
3020                 response = ahc_handle_msg_reject(ahc, devinfo);
3021                 /* FALLTHROUGH */
3022         case MSG_NOOP:
3023                 done = MSGLOOP_MSGCOMPLETE;
3024                 break;
3025         case MSG_EXTENDED:
3026         {
3027                 /* Wait for enough of the message to begin validation */
3028                 if (ahc->msgin_index < 2)
3029                         break;
3030                 switch (ahc->msgin_buf[2]) {
3031                 case MSG_EXT_SDTR:
3032                 {
3033                         struct   ahc_syncrate *syncrate;
3034                         u_int    period;
3035                         u_int    ppr_options;
3036                         u_int    offset;
3037                         u_int    saved_offset;
3038
3039                         if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3040                                 reject = TRUE;
3041                                 break;
3042                         }
3043
3044                         /*
3045                          * Wait until we have both args before validating
3046                          * and acting on this message.
3047                          *
3048                          * Add one to MSG_EXT_SDTR_LEN to account for
3049                          * the extended message preamble.
3050                          */
3051                         if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3052                                 break;
3053
3054                         period = ahc->msgin_buf[3];
3055                         ppr_options = 0;
3056                         saved_offset = offset = ahc->msgin_buf[4];
3057                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3058                                                            &ppr_options,
3059                                                            devinfo->role);
3060                         ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3061                                             targ_scsirate & WIDEXFER,
3062                                             devinfo->role);
3063                         if (bootverbose) {
3064                                 printf("(%s:%c:%d:%d): Received "
3065                                        "SDTR period %x, offset %x\n\t"
3066                                        "Filtered to period %x, offset %x\n",
3067                                        ahc_name(ahc), devinfo->channel,
3068                                        devinfo->target, devinfo->lun,
3069                                        ahc->msgin_buf[3], saved_offset,
3070                                        period, offset);
3071                         }
3072                         ahc_set_syncrate(ahc, devinfo,
3073                                          syncrate, period,
3074                                          offset, ppr_options,
3075                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3076                                          /*paused*/TRUE);
3077
3078                         /*
3079                          * See if we initiated Sync Negotiation
3080                          * and didn't have to fall down to async
3081                          * transfers.
3082                          */
3083                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3084                                 /* We started it */
3085                                 if (saved_offset != offset) {
3086                                         /* Went too low - force async */
3087                                         reject = TRUE;
3088                                 }
3089                         } else {
3090                                 /*
3091                                  * Send our own SDTR in reply
3092                                  */
3093                                 if (bootverbose
3094                                  && devinfo->role == ROLE_INITIATOR) {
3095                                         printf("(%s:%c:%d:%d): Target "
3096                                                "Initiated SDTR\n",
3097                                                ahc_name(ahc), devinfo->channel,
3098                                                devinfo->target, devinfo->lun);
3099                                 }
3100                                 ahc->msgout_index = 0;
3101                                 ahc->msgout_len = 0;
3102                                 ahc_construct_sdtr(ahc, devinfo,
3103                                                    period, offset);
3104                                 ahc->msgout_index = 0;
3105                                 response = TRUE;
3106                         }
3107                         done = MSGLOOP_MSGCOMPLETE;
3108                         break;
3109                 }
3110                 case MSG_EXT_WDTR:
3111                 {
3112                         u_int bus_width;
3113                         u_int saved_width;
3114                         u_int sending_reply;
3115
3116                         sending_reply = FALSE;
3117                         if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3118                                 reject = TRUE;
3119                                 break;
3120                         }
3121
3122                         /*
3123                          * Wait until we have our arg before validating
3124                          * and acting on this message.
3125                          *
3126                          * Add one to MSG_EXT_WDTR_LEN to account for
3127                          * the extended message preamble.
3128                          */
3129                         if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3130                                 break;
3131
3132                         bus_width = ahc->msgin_buf[3];
3133                         saved_width = bus_width;
3134                         ahc_validate_width(ahc, tinfo, &bus_width,
3135                                            devinfo->role);
3136                         if (bootverbose) {
3137                                 printf("(%s:%c:%d:%d): Received WDTR "
3138                                        "%x filtered to %x\n",
3139                                        ahc_name(ahc), devinfo->channel,
3140                                        devinfo->target, devinfo->lun,
3141                                        saved_width, bus_width);
3142                         }
3143
3144                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3145                                 /*
3146                                  * Don't send a WDTR back to the
3147                                  * target, since we asked first.
3148                                  * If the width went higher than our
3149                                  * request, reject it.
3150                                  */
3151                                 if (saved_width > bus_width) {
3152                                         reject = TRUE;
3153                                         printf("(%s:%c:%d:%d): requested %dBit "
3154                                                "transfers.  Rejecting...\n",
3155                                                ahc_name(ahc), devinfo->channel,
3156                                                devinfo->target, devinfo->lun,
3157                                                8 * (0x01 << bus_width));
3158                                         bus_width = 0;
3159                                 }
3160                         } else {
3161                                 /*
3162                                  * Send our own WDTR in reply
3163                                  */
3164                                 if (bootverbose
3165                                  && devinfo->role == ROLE_INITIATOR) {
3166                                         printf("(%s:%c:%d:%d): Target "
3167                                                "Initiated WDTR\n",
3168                                                ahc_name(ahc), devinfo->channel,
3169                                                devinfo->target, devinfo->lun);
3170                                 }
3171                                 ahc->msgout_index = 0;
3172                                 ahc->msgout_len = 0;
3173                                 ahc_construct_wdtr(ahc, devinfo, bus_width);
3174                                 ahc->msgout_index = 0;
3175                                 response = TRUE;
3176                                 sending_reply = TRUE;
3177                         }
3178                         /*
3179                          * After a wide message, we are async, but
3180                          * some devices don't seem to honor this portion
3181                          * of the spec.  Force a renegotiation of the
3182                          * sync component of our transfer agreement even
3183                          * if our goal is async.  By updating our width
3184                          * after forcing the negotiation, we avoid
3185                          * renegotiating for width.
3186                          */
3187                         ahc_update_neg_request(ahc, devinfo, tstate,
3188                                                tinfo, AHC_NEG_ALWAYS);
3189                         ahc_set_width(ahc, devinfo, bus_width,
3190                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3191                                       /*paused*/TRUE);
3192                         if (sending_reply == FALSE && reject == FALSE) {
3193
3194                                 /*
3195                                  * We will always have an SDTR to send.
3196                                  */
3197                                 ahc->msgout_index = 0;
3198                                 ahc->msgout_len = 0;
3199                                 ahc_build_transfer_msg(ahc, devinfo);
3200                                 ahc->msgout_index = 0;
3201                                 response = TRUE;
3202                         }
3203                         done = MSGLOOP_MSGCOMPLETE;
3204                         break;
3205                 }
3206                 case MSG_EXT_PPR:
3207                 {
3208                         struct  ahc_syncrate *syncrate;
3209                         u_int   period;
3210                         u_int   offset;
3211                         u_int   bus_width;
3212                         u_int   ppr_options;
3213                         u_int   saved_width;
3214                         u_int   saved_offset;
3215                         u_int   saved_ppr_options;
3216
3217                         if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3218                                 reject = TRUE;
3219                                 break;
3220                         }
3221
3222                         /*
3223                          * Wait until we have all args before validating
3224                          * and acting on this message.
3225                          *
3226                          * Add one to MSG_EXT_PPR_LEN to account for
3227                          * the extended message preamble.
3228                          */
3229                         if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3230                                 break;
3231
3232                         period = ahc->msgin_buf[3];
3233                         offset = ahc->msgin_buf[5];
3234                         bus_width = ahc->msgin_buf[6];
3235                         saved_width = bus_width;
3236                         ppr_options = ahc->msgin_buf[7];
3237                         /*
3238                          * According to the spec, a DT only
3239                          * period factor with no DT option
3240                          * set implies async.
3241                          */
3242                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3243                          && period == 9)
3244                                 offset = 0;
3245                         saved_ppr_options = ppr_options;
3246                         saved_offset = offset;
3247
3248                         /*
3249                          * Mask out any options we don't support
3250                          * on any controller.  Transfer options are
3251                          * only available if we are negotiating wide.
3252                          */
3253                         ppr_options &= MSG_EXT_PPR_DT_REQ;
3254                         if (bus_width == 0)
3255                                 ppr_options = 0;
3256
3257                         ahc_validate_width(ahc, tinfo, &bus_width,
3258                                            devinfo->role);
3259                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3260                                                            &ppr_options,
3261                                                            devinfo->role);
3262                         ahc_validate_offset(ahc, tinfo, syncrate,
3263                                             &offset, bus_width,
3264                                             devinfo->role);
3265
3266                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, TRUE)) {
3267                                 /*
3268                                  * If we are unable to do any of the
3269                                  * requested options (we went too low),
3270                                  * then we'll have to reject the message.
3271                                  */
3272                                 if (saved_width > bus_width
3273                                  || saved_offset != offset
3274                                  || saved_ppr_options != ppr_options) {
3275                                         reject = TRUE;
3276                                         period = 0;
3277                                         offset = 0;
3278                                         bus_width = 0;
3279                                         ppr_options = 0;
3280                                         syncrate = NULL;
3281                                 }
3282                         } else {
3283                                 if (devinfo->role != ROLE_TARGET)
3284                                         printf("(%s:%c:%d:%d): Target "
3285                                                "Initiated PPR\n",
3286                                                ahc_name(ahc), devinfo->channel,
3287                                                devinfo->target, devinfo->lun);
3288                                 else
3289                                         printf("(%s:%c:%d:%d): Initiator "
3290                                                "Initiated PPR\n",
3291                                                ahc_name(ahc), devinfo->channel,
3292                                                devinfo->target, devinfo->lun);
3293                                 ahc->msgout_index = 0;
3294                                 ahc->msgout_len = 0;
3295                                 ahc_construct_ppr(ahc, devinfo, period, offset,
3296                                                   bus_width, ppr_options);
3297                                 ahc->msgout_index = 0;
3298                                 response = TRUE;
3299                         }
3300                         if (bootverbose) {
3301                                 printf("(%s:%c:%d:%d): Received PPR width %x, "
3302                                        "period %x, offset %x,options %x\n"
3303                                        "\tFiltered to width %x, period %x, "
3304                                        "offset %x, options %x\n",
3305                                        ahc_name(ahc), devinfo->channel,
3306                                        devinfo->target, devinfo->lun,
3307                                        saved_width, ahc->msgin_buf[3],
3308                                        saved_offset, saved_ppr_options,
3309                                        bus_width, period, offset, ppr_options);
3310                         }
3311                         ahc_set_width(ahc, devinfo, bus_width,
3312                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3313                                       /*paused*/TRUE);
3314                         ahc_set_syncrate(ahc, devinfo,
3315                                          syncrate, period,
3316                                          offset, ppr_options,
3317                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3318                                          /*paused*/TRUE);
3319                         done = MSGLOOP_MSGCOMPLETE;
3320                         break;
3321                 }
3322                 default:
3323                         /* Unknown extended message.  Reject it. */
3324                         reject = TRUE;
3325                         break;
3326                 }
3327                 break;
3328         }
3329 #ifdef AHC_TARGET_MODE
3330         case MSG_BUS_DEV_RESET:
3331                 ahc_handle_devreset(ahc, devinfo,
3332                                     CAM_BDR_SENT,
3333                                     "Bus Device Reset Received",
3334                                     /*verbose_level*/0);
3335                 ahc_restart(ahc);
3336                 done = MSGLOOP_TERMINATED;
3337                 break;
3338         case MSG_ABORT_TAG:
3339         case MSG_ABORT:
3340         case MSG_CLEAR_QUEUE:
3341         {
3342                 int tag;
3343
3344                 /* Target mode messages */
3345                 if (devinfo->role != ROLE_TARGET) {
3346                         reject = TRUE;
3347                         break;
3348                 }
3349                 tag = SCB_LIST_NULL;
3350                 if (ahc->msgin_buf[0] == MSG_ABORT_TAG)
3351                         tag = ahc_inb(ahc, INITIATOR_TAG);
3352                 ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3353                                devinfo->lun, tag, ROLE_TARGET,
3354                                CAM_REQ_ABORTED);
3355
3356                 tstate = ahc->enabled_targets[devinfo->our_scsiid];
3357                 if (tstate != NULL) {
3358                         struct ahc_tmode_lstate* lstate;
3359
3360                         lstate = tstate->enabled_luns[devinfo->lun];
3361                         if (lstate != NULL) {
3362                                 ahc_queue_lstate_event(ahc, lstate,
3363                                                        devinfo->our_scsiid,
3364                                                        ahc->msgin_buf[0],
3365                                                        /*arg*/tag);
3366                                 ahc_send_lstate_events(ahc, lstate);
3367                         }
3368                 }
3369                 ahc_restart(ahc);
3370                 done = MSGLOOP_TERMINATED;
3371                 break;
3372         }
3373 #endif
3374         case MSG_TERM_IO_PROC:
3375         default:
3376                 reject = TRUE;
3377                 break;
3378         }
3379
3380         if (reject) {
3381                 /*
3382                  * Setup to reject the message.
3383                  */
3384                 ahc->msgout_index = 0;
3385                 ahc->msgout_len = 1;
3386                 ahc->msgout_buf[0] = MSG_MESSAGE_REJECT;
3387                 done = MSGLOOP_MSGCOMPLETE;
3388                 response = TRUE;
3389         }
3390
3391         if (done != MSGLOOP_IN_PROG && !response)
3392                 /* Clear the outgoing message buffer */
3393                 ahc->msgout_len = 0;
3394
3395         return (done);
3396 }
3397
3398 /*
3399  * Process a message reject message.
3400  */
3401 static int
3402 ahc_handle_msg_reject(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3403 {
3404         /*
3405          * What we care about here is if we had an
3406          * outstanding SDTR or WDTR message for this
3407          * target.  If we did, this is a signal that
3408          * the target is refusing negotiation.
3409          */
3410         struct scb *scb;
3411         struct ahc_initiator_tinfo *tinfo;
3412         struct ahc_tmode_tstate *tstate;
3413         u_int scb_index;
3414         u_int last_msg;
3415         int   response = 0;
3416
3417         scb_index = ahc_inb(ahc, SCB_TAG);
3418         scb = ahc_lookup_scb(ahc, scb_index);
3419         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel,
3420                                     devinfo->our_scsiid,
3421                                     devinfo->target, &tstate);
3422         /* Might be necessary */
3423         last_msg = ahc_inb(ahc, LAST_MSG);
3424
3425         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
3426                 /*
3427                  * Target does not support the PPR message.
3428                  * Attempt to negotiate SPI-2 style.
3429                  */
3430                 if (bootverbose) {
3431                         printf("(%s:%c:%d:%d): PPR Rejected. "
3432                                "Trying WDTR/SDTR\n",
3433                                ahc_name(ahc), devinfo->channel,
3434                                devinfo->target, devinfo->lun);
3435                 }
3436                 tinfo->goal.ppr_options = 0;
3437                 tinfo->curr.transport_version = 2;
3438                 tinfo->goal.transport_version = 2;
3439                 ahc->msgout_index = 0;
3440                 ahc->msgout_len = 0;
3441                 ahc_build_transfer_msg(ahc, devinfo);
3442                 ahc->msgout_index = 0;
3443                 response = 1;
3444         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
3445
3446                 /* note 8bit xfers */
3447                 printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
3448                        "8bit transfers\n", ahc_name(ahc),
3449                        devinfo->channel, devinfo->target, devinfo->lun);
3450                 ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3451                               AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3452                               /*paused*/TRUE);
3453                 /*
3454                  * No need to clear the sync rate.  If the target
3455                  * did not accept the command, our syncrate is
3456                  * unaffected.  If the target started the negotiation,
3457                  * but rejected our response, we already cleared the
3458                  * sync rate before sending our WDTR.
3459                  */
3460                 if (tinfo->goal.offset != tinfo->curr.offset) {
3461
3462                         /* Start the sync negotiation */
3463                         ahc->msgout_index = 0;
3464                         ahc->msgout_len = 0;
3465                         ahc_build_transfer_msg(ahc, devinfo);
3466                         ahc->msgout_index = 0;
3467                         response = 1;
3468                 }
3469         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
3470                 /* note asynch xfers and clear flag */
3471                 ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL, /*period*/0,
3472                                  /*offset*/0, /*ppr_options*/0,
3473                                  AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3474                                  /*paused*/TRUE);
3475                 printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
3476                        "Using asynchronous transfers\n",
3477                        ahc_name(ahc), devinfo->channel,
3478                        devinfo->target, devinfo->lun);
3479         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
3480                 int tag_type;
3481                 int mask;
3482
3483                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
3484
3485                 if (tag_type == MSG_SIMPLE_TASK) {
3486                         printf("(%s:%c:%d:%d): refuses tagged commands.  "
3487                                "Performing non-tagged I/O\n", ahc_name(ahc),
3488                                devinfo->channel, devinfo->target, devinfo->lun);
3489                         ahc_set_tags(ahc, devinfo, AHC_QUEUE_NONE);
3490                         mask = ~0x23;
3491                 } else {
3492                         printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
3493                                "Performing simple queue tagged I/O only\n",
3494                                ahc_name(ahc), devinfo->channel, devinfo->target,
3495                                devinfo->lun, tag_type == MSG_ORDERED_TASK
3496                                ? "ordered" : "head of queue");
3497                         ahc_set_tags(ahc, devinfo, AHC_QUEUE_BASIC);
3498                         mask = ~0x03;
3499                 }
3500
3501                 /*
3502                  * Resend the identify for this CCB as the target
3503                  * may believe that the selection is invalid otherwise.
3504                  */
3505                 ahc_outb(ahc, SCB_CONTROL,
3506                          ahc_inb(ahc, SCB_CONTROL) & mask);
3507                 scb->hscb->control &= mask;
3508                 ahc_set_transaction_tag(scb, /*enabled*/FALSE,
3509                                         /*type*/MSG_SIMPLE_TASK);
3510                 ahc_outb(ahc, MSG_OUT, MSG_IDENTIFYFLAG);
3511                 ahc_assert_atn(ahc);
3512
3513                 /*
3514                  * This transaction is now at the head of
3515                  * the untagged queue for this target.
3516                  */
3517                 if ((ahc->flags & AHC_SCB_BTT) == 0) {
3518                         struct scb_tailq *untagged_q;
3519
3520                         untagged_q =
3521                             &(ahc->untagged_queues[devinfo->target_offset]);
3522                         TAILQ_INSERT_HEAD(untagged_q, scb, links.tqe);
3523                         scb->flags |= SCB_UNTAGGEDQ;
3524                 }
3525                 ahc_busy_tcl(ahc, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
3526                              scb->hscb->tag);
3527
3528                 /*
3529                  * Requeue all tagged commands for this target
3530                  * currently in our posession so they can be
3531                  * converted to untagged commands.
3532                  */
3533                 ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
3534                                    SCB_GET_CHANNEL(ahc, scb),
3535                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
3536                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
3537                                    SEARCH_COMPLETE);
3538         } else {
3539                 /*
3540                  * Otherwise, we ignore it.
3541                  */
3542                 printf("%s:%c:%d: Message reject for %x -- ignored\n",
3543                        ahc_name(ahc), devinfo->channel, devinfo->target,
3544                        last_msg);
3545         }
3546         return (response);
3547 }
3548
3549 /*
3550  * Process an ingnore wide residue message.
3551  */
3552 static void
3553 ahc_handle_ign_wide_residue(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3554 {
3555         u_int scb_index;
3556         struct scb *scb;
3557
3558         scb_index = ahc_inb(ahc, SCB_TAG);
3559         scb = ahc_lookup_scb(ahc, scb_index);
3560         /*
3561          * XXX Actually check data direction in the sequencer?
3562          * Perhaps add datadir to some spare bits in the hscb?
3563          */
3564         if ((ahc_inb(ahc, SEQ_FLAGS) & DPHASE) == 0
3565          || ahc_get_transfer_dir(scb) != CAM_DIR_IN) {
3566                 /*
3567                  * Ignore the message if we haven't
3568                  * seen an appropriate data phase yet.
3569                  */
3570         } else {
3571                 /*
3572                  * If the residual occurred on the last
3573                  * transfer and the transfer request was
3574                  * expected to end on an odd count, do
3575                  * nothing.  Otherwise, subtract a byte
3576                  * and update the residual count accordingly.
3577                  */
3578                 uint32_t sgptr;
3579
3580                 sgptr = ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
3581                 if ((sgptr & SG_LIST_NULL) != 0
3582                  && (ahc_inb(ahc, SCB_LUN) & SCB_XFERLEN_ODD) != 0) {
3583                         /*
3584                          * If the residual occurred on the last
3585                          * transfer and the transfer request was
3586                          * expected to end on an odd count, do
3587                          * nothing.
3588                          */
3589                 } else {
3590                         struct ahc_dma_seg *sg;
3591                         uint32_t data_cnt;
3592                         uint32_t data_addr;
3593                         uint32_t sglen;
3594
3595                         /* Pull in all of the sgptr */
3596                         sgptr = ahc_inl(ahc, SCB_RESIDUAL_SGPTR);
3597                         data_cnt = ahc_inl(ahc, SCB_RESIDUAL_DATACNT);
3598
3599                         if ((sgptr & SG_LIST_NULL) != 0) {
3600                                 /*
3601                                  * The residual data count is not updated
3602                                  * for the command run to completion case.
3603                                  * Explicitly zero the count.
3604                                  */
3605                                 data_cnt &= ~AHC_SG_LEN_MASK;
3606                         }
3607
3608                         data_addr = ahc_inl(ahc, SHADDR);
3609
3610                         data_cnt += 1;
3611                         data_addr -= 1;
3612                         sgptr &= SG_PTR_MASK;
3613
3614                         sg = ahc_sg_bus_to_virt(scb, sgptr);
3615
3616                         /*
3617                          * The residual sg ptr points to the next S/G
3618                          * to load so we must go back one.
3619                          */
3620                         sg--;
3621                         sglen = ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
3622                         if (sg != scb->sg_list
3623                          && sglen < (data_cnt & AHC_SG_LEN_MASK)) {
3624
3625                                 sg--;
3626                                 sglen = ahc_le32toh(sg->len);
3627                                 /*
3628                                  * Preserve High Address and SG_LIST bits
3629                                  * while setting the count to 1.
3630                                  */
3631                                 data_cnt = 1 | (sglen & (~AHC_SG_LEN_MASK));
3632                                 data_addr = ahc_le32toh(sg->addr)
3633                                           + (sglen & AHC_SG_LEN_MASK) - 1;
3634
3635                                 /*
3636                                  * Increment sg so it points to the
3637                                  * "next" sg.
3638                                  */
3639                                 sg++;
3640                                 sgptr = ahc_sg_virt_to_bus(scb, sg);
3641                         }
3642                         ahc_outl(ahc, SCB_RESIDUAL_SGPTR, sgptr);
3643                         ahc_outl(ahc, SCB_RESIDUAL_DATACNT, data_cnt);
3644                         /*
3645                          * Toggle the "oddness" of the transfer length
3646                          * to handle this mid-transfer ignore wide
3647                          * residue.  This ensures that the oddness is
3648                          * correct for subsequent data transfers.
3649                          */
3650                         ahc_outb(ahc, SCB_LUN,
3651                                  ahc_inb(ahc, SCB_LUN) ^ SCB_XFERLEN_ODD);
3652                 }
3653         }
3654 }
3655
3656
3657 /*
3658  * Reinitialize the data pointers for the active transfer
3659  * based on its current residual.
3660  */
3661 static void
3662 ahc_reinitialize_dataptrs(struct ahc_softc *ahc)
3663 {
3664         struct   scb *scb;
3665         struct   ahc_dma_seg *sg;
3666         u_int    scb_index;
3667         uint32_t sgptr;
3668         uint32_t resid;
3669         uint32_t dataptr;
3670
3671         scb_index = ahc_inb(ahc, SCB_TAG);
3672         scb = ahc_lookup_scb(ahc, scb_index);
3673         sgptr = (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 3) << 24)
3674               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 2) << 16)
3675               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 1) << 8)
3676               | ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
3677
3678         sgptr &= SG_PTR_MASK;
3679         sg = ahc_sg_bus_to_virt(scb, sgptr);
3680
3681         /* The residual sg_ptr always points to the next sg */
3682         sg--;
3683
3684         resid = (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 2) << 16)
3685               | (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 1) << 8)
3686               | ahc_inb(ahc, SCB_RESIDUAL_DATACNT);
3687
3688         dataptr = ahc_le32toh(sg->addr)
3689                 + (ahc_le32toh(sg->len) & AHC_SG_LEN_MASK)
3690                 - resid;
3691         if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
3692                 u_int dscommand1;
3693
3694                 dscommand1 = ahc_inb(ahc, DSCOMMAND1);
3695                 ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
3696                 ahc_outb(ahc, HADDR,
3697                          (ahc_le32toh(sg->len) >> 24) & SG_HIGH_ADDR_BITS);
3698                 ahc_outb(ahc, DSCOMMAND1, dscommand1);
3699         }
3700         ahc_outb(ahc, HADDR + 3, dataptr >> 24);
3701         ahc_outb(ahc, HADDR + 2, dataptr >> 16);
3702         ahc_outb(ahc, HADDR + 1, dataptr >> 8);
3703         ahc_outb(ahc, HADDR, dataptr);
3704         ahc_outb(ahc, HCNT + 2, resid >> 16);
3705         ahc_outb(ahc, HCNT + 1, resid >> 8);
3706         ahc_outb(ahc, HCNT, resid);
3707         if ((ahc->features & AHC_ULTRA2) == 0) {
3708                 ahc_outb(ahc, STCNT + 2, resid >> 16);
3709                 ahc_outb(ahc, STCNT + 1, resid >> 8);
3710                 ahc_outb(ahc, STCNT, resid);
3711         }
3712 }
3713
3714 /*
3715  * Handle the effects of issuing a bus device reset message.
3716  */
3717 static void
3718 ahc_handle_devreset(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3719                     cam_status status, char *message, int verbose_level)
3720 {
3721 #ifdef AHC_TARGET_MODE
3722         struct ahc_tmode_tstate* tstate;
3723         u_int lun;
3724 #endif
3725         int found;
3726
3727         found = ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3728                                CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
3729                                status);
3730
3731 #ifdef AHC_TARGET_MODE
3732         /*
3733          * Send an immediate notify ccb to all target mord peripheral
3734          * drivers affected by this action.
3735          */
3736         tstate = ahc->enabled_targets[devinfo->our_scsiid];
3737         if (tstate != NULL) {
3738                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
3739                         struct ahc_tmode_lstate* lstate;
3740
3741                         lstate = tstate->enabled_luns[lun];
3742                         if (lstate == NULL)
3743                                 continue;
3744
3745                         ahc_queue_lstate_event(ahc, lstate, devinfo->our_scsiid,
3746                                                MSG_BUS_DEV_RESET, /*arg*/0);
3747                         ahc_send_lstate_events(ahc, lstate);
3748                 }
3749         }
3750 #endif
3751
3752         /*
3753          * Go back to async/narrow transfers and renegotiate.
3754          */
3755         ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3756                       AHC_TRANS_CUR, /*paused*/TRUE);
3757         ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL,
3758                          /*period*/0, /*offset*/0, /*ppr_options*/0,
3759                          AHC_TRANS_CUR, /*paused*/TRUE);
3760
3761         ahc_send_async(ahc, devinfo->channel, devinfo->target,
3762                        CAM_LUN_WILDCARD, AC_SENT_BDR, NULL);
3763
3764         if (message != NULL
3765          && (verbose_level <= bootverbose))
3766                 printf("%s: %s on %c:%d. %d SCBs aborted\n", ahc_name(ahc),
3767                        message, devinfo->channel, devinfo->target, found);
3768 }
3769
3770 #ifdef AHC_TARGET_MODE
3771 static void
3772 ahc_setup_target_msgin(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3773                        struct scb *scb)
3774 {
3775
3776         /*
3777          * To facilitate adding multiple messages together,
3778          * each routine should increment the index and len
3779          * variables instead of setting them explicitly.
3780          */
3781         ahc->msgout_index = 0;
3782         ahc->msgout_len = 0;
3783
3784         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
3785                 ahc_build_transfer_msg(ahc, devinfo);
3786         else
3787                 panic("ahc_intr: AWAITING target message with no message");
3788
3789         ahc->msgout_index = 0;
3790         ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3791 }
3792 #endif
3793 /**************************** Initialization **********************************/
3794 /*
3795  * Allocate a controller structure for a new device
3796  * and perform initial initializion.
3797  */
3798 struct ahc_softc *
3799 ahc_alloc(void *platform_arg, char *name)
3800 {
3801         struct  ahc_softc *ahc;
3802         int     i;
3803
3804 #if !defined(__DragonFly__) && !defined(__FreeBSD__)
3805         ahc = kmalloc(sizeof(*ahc), M_DEVBUF, M_WAITOK);
3806 #else
3807         ahc = device_get_softc((device_t)platform_arg);
3808 #endif
3809         memset(ahc, 0, sizeof(*ahc));
3810         ahc->seep_config = kmalloc(sizeof(*ahc->seep_config),
3811                                   M_DEVBUF, M_WAITOK);
3812         LIST_INIT(&ahc->pending_scbs);
3813         /* We don't know our unit number until the OSM sets it */
3814         ahc->name = name;
3815         ahc->unit = -1;
3816         ahc->description = NULL;
3817         ahc->channel = 'A';
3818         ahc->channel_b = 'B';
3819         ahc->chip = AHC_NONE;
3820         ahc->features = AHC_FENONE;
3821         ahc->bugs = AHC_BUGNONE;
3822         ahc->flags = AHC_FNONE;
3823         /*
3824          * Default to all error reporting enabled with the
3825          * sequencer operating at its fastest speed.
3826          * The bus attach code may modify this.
3827          */
3828         ahc->seqctl = FASTMODE;
3829
3830         for (i = 0; i < AHC_NUM_TARGETS; i++)
3831                 TAILQ_INIT(&ahc->untagged_queues[i]);
3832         if (ahc_platform_alloc(ahc, platform_arg) != 0) {
3833                 ahc_free(ahc);
3834                 ahc = NULL;
3835         }
3836         return (ahc);
3837 }
3838
3839 int
3840 ahc_softc_init(struct ahc_softc *ahc)
3841 {
3842
3843         /* The IRQMS bit is only valid on VL and EISA chips */
3844         if ((ahc->chip & AHC_PCI) == 0)
3845                 ahc->unpause = ahc_inb(ahc, HCNTRL) & IRQMS;
3846         else
3847                 ahc->unpause = 0;
3848         ahc->pause = ahc->unpause | PAUSE;
3849         /* XXX The shared scb data stuff should be deprecated */
3850         if (ahc->scb_data == NULL) {
3851                 ahc->scb_data = kmalloc(sizeof(*ahc->scb_data),
3852                                        M_DEVBUF, M_WAITOK | M_ZERO);
3853         }
3854
3855         return (0);
3856 }
3857
3858 void
3859 ahc_softc_insert(struct ahc_softc *ahc)
3860 {
3861         struct ahc_softc *list_ahc;
3862
3863 #if AHC_PCI_CONFIG > 0
3864         /*
3865          * Second Function PCI devices need to inherit some
3866          * settings from function 0.
3867          */
3868         if ((ahc->chip & AHC_BUS_MASK) == AHC_PCI
3869          && (ahc->features & AHC_MULTI_FUNC) != 0) {
3870                 TAILQ_FOREACH(list_ahc, &ahc_tailq, links) {
3871                         ahc_dev_softc_t list_pci;
3872                         ahc_dev_softc_t pci;
3873
3874                         list_pci = list_ahc->dev_softc;
3875                         pci = ahc->dev_softc;
3876                         if (ahc_get_pci_slot(list_pci) == ahc_get_pci_slot(pci)
3877                          && ahc_get_pci_bus(list_pci) == ahc_get_pci_bus(pci)) {
3878                                 struct ahc_softc *master;
3879                                 struct ahc_softc *slave;
3880
3881                                 if (ahc_get_pci_function(list_pci) == 0) {
3882                                         master = list_ahc;
3883                                         slave = ahc;
3884                                 } else {
3885                                         master = ahc;
3886                                         slave = list_ahc;
3887                                 }
3888                                 slave->flags &= ~AHC_BIOS_ENABLED;
3889                                 slave->flags |=
3890                                     master->flags & AHC_BIOS_ENABLED;
3891                                 slave->flags &= ~AHC_PRIMARY_CHANNEL;
3892                                 slave->flags |=
3893                                     master->flags & AHC_PRIMARY_CHANNEL;
3894                                 break;
3895                         }
3896                 }
3897         }
3898 #endif
3899
3900         /*
3901          * Insertion sort into our list of softcs.
3902          */
3903         list_ahc = TAILQ_FIRST(&ahc_tailq);
3904         while (list_ahc != NULL
3905             && ahc_softc_comp(ahc, list_ahc) <= 0)
3906                 list_ahc = TAILQ_NEXT(list_ahc, links);
3907         if (list_ahc != NULL)
3908                 TAILQ_INSERT_BEFORE(list_ahc, ahc, links);
3909         else
3910                 TAILQ_INSERT_TAIL(&ahc_tailq, ahc, links);
3911         ahc->init_level++;
3912 }
3913
3914 /*
3915  * Verify that the passed in softc pointer is for a
3916  * controller that is still configured.
3917  */
3918 struct ahc_softc *
3919 ahc_find_softc(struct ahc_softc *ahc)
3920 {
3921         struct ahc_softc *list_ahc;
3922
3923         TAILQ_FOREACH(list_ahc, &ahc_tailq, links) {
3924                 if (list_ahc == ahc)
3925                         return (ahc);
3926         }
3927         return (NULL);
3928 }
3929
3930 void
3931 ahc_set_unit(struct ahc_softc *ahc, int unit)
3932 {
3933         ahc->unit = unit;
3934 }
3935
3936 void
3937 ahc_set_name(struct ahc_softc *ahc, char *name)
3938 {
3939         if (ahc->name != NULL)
3940                 kfree(ahc->name, M_DEVBUF);
3941         ahc->name = name;
3942 }
3943
3944 void
3945 ahc_free(struct ahc_softc *ahc)
3946 {
3947         int i;
3948
3949         switch (ahc->init_level) {
3950         default:
3951         case 5:
3952                 ahc_shutdown(ahc);
3953                 TAILQ_REMOVE(&ahc_tailq, ahc, links);
3954                 /* FALLTHROUGH */
3955         case 4:
3956                 ahc_dmamap_unload(ahc, ahc->shared_data_dmat,
3957                                   ahc->shared_data_dmamap);
3958                 /* FALLTHROUGH */
3959         case 3:
3960                 ahc_dmamem_free(ahc, ahc->shared_data_dmat, ahc->qoutfifo,
3961                                 ahc->shared_data_dmamap);
3962                 ahc_dmamap_destroy(ahc, ahc->shared_data_dmat,
3963                                    ahc->shared_data_dmamap);
3964                 /* FALLTHROUGH */
3965         case 2:
3966                 ahc_dma_tag_destroy(ahc, ahc->shared_data_dmat);
3967         case 1:
3968 #ifndef __linux__
3969                 ahc_dma_tag_destroy(ahc, ahc->buffer_dmat);
3970 #endif
3971                 break;
3972         case 0:
3973                 break;
3974         }
3975
3976 #ifndef __linux__
3977         ahc_dma_tag_destroy(ahc, ahc->parent_dmat);
3978 #endif
3979         ahc_platform_free(ahc);
3980         ahc_fini_scbdata(ahc);
3981         for (i = 0; i < AHC_NUM_TARGETS; i++) {
3982                 struct ahc_tmode_tstate *tstate;
3983
3984                 tstate = ahc->enabled_targets[i];
3985                 if (tstate != NULL) {
3986 #ifdef AHC_TARGET_MODE
3987                         int j;
3988
3989                         for (j = 0; j < AHC_NUM_LUNS; j++) {
3990                                 struct ahc_tmode_lstate *lstate;
3991
3992                                 lstate = tstate->enabled_luns[j];
3993                                 if (lstate != NULL) {
3994                                         xpt_free_path(lstate->path);
3995                                         kfree(lstate, M_DEVBUF);
3996                                 }
3997                         }
3998 #endif
3999                         kfree(tstate, M_DEVBUF);
4000                 }
4001         }
4002 #ifdef AHC_TARGET_MODE
4003         if (ahc->black_hole != NULL) {
4004                 xpt_free_path(ahc->black_hole->path);
4005                 kfree(ahc->black_hole, M_DEVBUF);
4006         }
4007 #endif
4008         if (ahc->name != NULL)
4009                 kfree(ahc->name, M_DEVBUF);
4010         if (ahc->seep_config != NULL)
4011                 kfree(ahc->seep_config, M_DEVBUF);
4012 #if !defined(__DragonFly__) && !defined(__FreeBSD__)
4013         kfree(ahc, M_DEVBUF);
4014 #endif
4015         return;
4016 }
4017
4018 void
4019 ahc_shutdown(void *arg)
4020 {
4021         struct  ahc_softc *ahc;
4022         int     i;
4023
4024         ahc = (struct ahc_softc *)arg;
4025
4026         /* This will reset most registers to 0, but not all */
4027         ahc_reset(ahc, /*reinit*/FALSE);
4028         ahc_outb(ahc, SCSISEQ, 0);
4029         ahc_outb(ahc, SXFRCTL0, 0);
4030         ahc_outb(ahc, DSPCISTATUS, 0);
4031
4032         for (i = TARG_SCSIRATE; i < SCSICONF; i++)
4033                 ahc_outb(ahc, i, 0);
4034 }
4035
4036 /*
4037  * Reset the controller and record some information about it
4038  * that is only available just after a reset.  If "reinit" is
4039  * non-zero, this reset occured after initial configuration
4040  * and the caller requests that the chip be fully reinitialized
4041  * to a runable state.  Chip interrupts are *not* enabled after
4042  * a reinitialization.  The caller must enable interrupts via
4043  * ahc_intr_enable().
4044  */
4045 int
4046 ahc_reset(struct ahc_softc *ahc, int reinit)
4047 {
4048         u_int   sblkctl;
4049         u_int   sxfrctl1_a, sxfrctl1_b;
4050         int     error;
4051         int     wait;
4052
4053         /*
4054          * Preserve the value of the SXFRCTL1 register for all channels.
4055          * It contains settings that affect termination and we don't want
4056          * to disturb the integrity of the bus.
4057          */
4058         ahc_pause(ahc);
4059         if ((ahc_inb(ahc, HCNTRL) & CHIPRST) != 0) {
4060                 /*
4061                  * The chip has not been initialized since
4062                  * PCI/EISA/VLB bus reset.  Don't trust
4063                  * "left over BIOS data".
4064                  */
4065                 ahc->flags |= AHC_NO_BIOS_INIT;
4066         }
4067         sxfrctl1_b = 0;
4068         if ((ahc->chip & AHC_CHIPID_MASK) == AHC_AIC7770) {
4069                 u_int sblkctl;
4070
4071                 /*
4072                  * Save channel B's settings in case this chip
4073                  * is setup for TWIN channel operation.
4074                  */
4075                 sblkctl = ahc_inb(ahc, SBLKCTL);
4076                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4077                 sxfrctl1_b = ahc_inb(ahc, SXFRCTL1);
4078                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4079         }
4080         sxfrctl1_a = ahc_inb(ahc, SXFRCTL1);
4081
4082         ahc_outb(ahc, HCNTRL, CHIPRST | ahc->pause);
4083
4084         /*
4085          * Ensure that the reset has finished.  We delay 1000us
4086          * prior to reading the register to make sure the chip
4087          * has sufficiently completed its reset to handle register
4088          * accesses.
4089          */
4090         wait = 1000;
4091         do {
4092                 ahc_delay(1000);
4093         } while (--wait && !(ahc_inb(ahc, HCNTRL) & CHIPRSTACK));
4094
4095         if (wait == 0) {
4096                 printf("%s: WARNING - Failed chip reset!  "
4097                        "Trying to initialize anyway.\n", ahc_name(ahc));
4098         }
4099         ahc_outb(ahc, HCNTRL, ahc->pause);
4100
4101         /* Determine channel configuration */
4102         sblkctl = ahc_inb(ahc, SBLKCTL) & (SELBUSB|SELWIDE);
4103         /* No Twin Channel PCI cards */
4104         if ((ahc->chip & AHC_PCI) != 0)
4105                 sblkctl &= ~SELBUSB;
4106         switch (sblkctl) {
4107         case 0:
4108                 /* Single Narrow Channel */
4109                 break;
4110         case 2:
4111                 /* Wide Channel */
4112                 ahc->features |= AHC_WIDE;
4113                 break;
4114         case 8:
4115                 /* Twin Channel */
4116                 ahc->features |= AHC_TWIN;
4117                 break;
4118         default:
4119                 printf(" Unsupported adapter type.  Ignoring\n");
4120                 return(-1);
4121         }
4122
4123         /*
4124          * Reload sxfrctl1.
4125          *
4126          * We must always initialize STPWEN to 1 before we
4127          * restore the saved values.  STPWEN is initialized
4128          * to a tri-state condition which can only be cleared
4129          * by turning it on.
4130          */
4131         if ((ahc->features & AHC_TWIN) != 0) {
4132                 u_int sblkctl;
4133
4134                 sblkctl = ahc_inb(ahc, SBLKCTL);
4135                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4136                 ahc_outb(ahc, SXFRCTL1, sxfrctl1_b);
4137                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4138         }
4139         ahc_outb(ahc, SXFRCTL1, sxfrctl1_a);
4140
4141         error = 0;
4142         if (reinit != 0)
4143                 /*
4144                  * If a recovery action has forced a chip reset,
4145                  * re-initialize the chip to our liking.
4146                  */
4147                 error = ahc->bus_chip_init(ahc);
4148 #ifdef AHC_DUMP_SEQ
4149         else
4150                 ahc_dumpseq(ahc);
4151 #endif
4152
4153         return (error);
4154 }
4155
4156 /*
4157  * Determine the number of SCBs available on the controller
4158  */
4159 int
4160 ahc_probe_scbs(struct ahc_softc *ahc) {
4161         int i;
4162
4163         for (i = 0; i < AHC_SCB_MAX; i++) {
4164
4165                 ahc_outb(ahc, SCBPTR, i);
4166                 ahc_outb(ahc, SCB_BASE, i);
4167                 if (ahc_inb(ahc, SCB_BASE) != i)
4168                         break;
4169                 ahc_outb(ahc, SCBPTR, 0);
4170                 if (ahc_inb(ahc, SCB_BASE) != 0)
4171                         break;
4172         }
4173         return (i);
4174 }
4175
4176 static void
4177 ahc_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4178 {
4179         bus_addr_t *baddr;
4180
4181         baddr = (bus_addr_t *)arg;
4182         *baddr = segs->ds_addr;
4183 }
4184
4185 static void
4186 ahc_build_free_scb_list(struct ahc_softc *ahc)
4187 {
4188         int scbsize;
4189         int i;
4190
4191         scbsize = 32;
4192         if ((ahc->flags & AHC_LSCBS_ENABLED) != 0)
4193                 scbsize = 64;
4194
4195         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
4196                 int j;
4197
4198                 ahc_outb(ahc, SCBPTR, i);
4199
4200                 /*
4201                  * Touch all SCB bytes to avoid parity errors
4202                  * should one of our debugging routines read
4203                  * an otherwise uninitiatlized byte.
4204                  */
4205                 for (j = 0; j < scbsize; j++)
4206                         ahc_outb(ahc, SCB_BASE+j, 0xFF);
4207
4208                 /* Clear the control byte. */
4209                 ahc_outb(ahc, SCB_CONTROL, 0);
4210
4211                 /* Set the next pointer */
4212                 if ((ahc->flags & AHC_PAGESCBS) != 0)
4213                         ahc_outb(ahc, SCB_NEXT, i+1);
4214                 else
4215                         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4216
4217                 /* Make the tag number, SCSIID, and lun invalid */
4218                 ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
4219                 ahc_outb(ahc, SCB_SCSIID, 0xFF);
4220                 ahc_outb(ahc, SCB_LUN, 0xFF);
4221         }
4222
4223         if ((ahc->flags & AHC_PAGESCBS) != 0) {
4224                 /* SCB 0 heads the free list. */
4225                 ahc_outb(ahc, FREE_SCBH, 0);
4226         } else {
4227                 /* No free list. */
4228                 ahc_outb(ahc, FREE_SCBH, SCB_LIST_NULL);
4229         }
4230
4231         /* Make sure that the last SCB terminates the free list */
4232         ahc_outb(ahc, SCBPTR, i-1);
4233         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4234 }
4235
4236 static int
4237 ahc_init_scbdata(struct ahc_softc *ahc)
4238 {
4239         struct scb_data *scb_data;
4240
4241         scb_data = ahc->scb_data;
4242         SLIST_INIT(&scb_data->free_scbs);
4243         SLIST_INIT(&scb_data->sg_maps);
4244
4245         /* Allocate SCB resources */
4246         scb_data->scbarray = kmalloc(sizeof(struct scb) * AHC_SCB_MAX_ALLOC,
4247                                     M_DEVBUF, M_INTWAIT | M_ZERO);
4248
4249         /* Determine the number of hardware SCBs and initialize them */
4250
4251         scb_data->maxhscbs = ahc_probe_scbs(ahc);
4252         if (ahc->scb_data->maxhscbs == 0) {
4253                 printf("%s: No SCB space found\n", ahc_name(ahc));
4254                 return (ENXIO);
4255         }
4256
4257         /*
4258          * Create our DMA tags.  These tags define the kinds of device
4259          * accessible memory allocations and memory mappings we will
4260          * need to perform during normal operation.
4261          *
4262          * Unless we need to further restrict the allocation, we rely
4263          * on the restrictions of the parent dmat, hence the common
4264          * use of MAXADDR and MAXSIZE.
4265          */
4266
4267         /* DMA tag for our hardware scb structures */
4268         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4269                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4270                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4271                                /*highaddr*/BUS_SPACE_MAXADDR,
4272                                /*filter*/NULL, /*filterarg*/NULL,
4273                                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4274                                /*nsegments*/1,
4275                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4276                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
4277                 goto error_exit;
4278         }
4279
4280         scb_data->init_level++;
4281
4282         /* Allocation for our hscbs */
4283         if (ahc_dmamem_alloc(ahc, scb_data->hscb_dmat,
4284                              (void **)&scb_data->hscbs,
4285                              BUS_DMA_NOWAIT, &scb_data->hscb_dmamap) != 0) {
4286                 goto error_exit;
4287         }
4288
4289         scb_data->init_level++;
4290
4291         /* And permanently map them */
4292         ahc_dmamap_load(ahc, scb_data->hscb_dmat, scb_data->hscb_dmamap,
4293                         scb_data->hscbs,
4294                         AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4295                         ahc_dmamap_cb, &scb_data->hscb_busaddr, /*flags*/0);
4296
4297         scb_data->init_level++;
4298
4299         /* DMA tag for our sense buffers */
4300         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4301                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4302                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4303                                /*highaddr*/BUS_SPACE_MAXADDR,
4304                                /*filter*/NULL, /*filterarg*/NULL,
4305                                AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4306                                /*nsegments*/1,
4307                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4308                                /*flags*/0, &scb_data->sense_dmat) != 0) {
4309                 goto error_exit;
4310         }
4311
4312         scb_data->init_level++;
4313
4314         /* Allocate them */
4315         if (ahc_dmamem_alloc(ahc, scb_data->sense_dmat,
4316                              (void **)&scb_data->sense,
4317                              BUS_DMA_NOWAIT, &scb_data->sense_dmamap) != 0) {
4318                 goto error_exit;
4319         }
4320
4321         scb_data->init_level++;
4322
4323         /* And permanently map them */
4324         ahc_dmamap_load(ahc, scb_data->sense_dmat, scb_data->sense_dmamap,
4325                         scb_data->sense,
4326                         AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4327                         ahc_dmamap_cb, &scb_data->sense_busaddr, /*flags*/0);
4328
4329         scb_data->init_level++;
4330
4331         /* DMA tag for our S/G structures.  We allocate in page sized chunks */
4332         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/8,
4333                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4334                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4335                                /*highaddr*/BUS_SPACE_MAXADDR,
4336                                /*filter*/NULL, /*filterarg*/NULL,
4337                                PAGE_SIZE, /*nsegments*/1,
4338                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4339                                /*flags*/0, &scb_data->sg_dmat) != 0) {
4340                 goto error_exit;
4341         }
4342
4343         scb_data->init_level++;
4344
4345         /* Perform initial CCB allocation */
4346         memset(scb_data->hscbs, 0,
4347                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb));
4348         ahc_alloc_scbs(ahc);
4349
4350         if (scb_data->numscbs == 0) {
4351                 printf("%s: ahc_init_scbdata - "
4352                        "Unable to allocate initial scbs\n",
4353                        ahc_name(ahc));
4354                 goto error_exit;
4355         }
4356
4357         /*
4358          * Reserve the next queued SCB.
4359          */
4360         ahc->next_queued_scb = ahc_get_scb(ahc);
4361
4362         /*
4363          * Note that we were successful
4364          */
4365         return (0);
4366
4367 error_exit:
4368
4369         return (ENOMEM);
4370 }
4371
4372 static void
4373 ahc_fini_scbdata(struct ahc_softc *ahc)
4374 {
4375         struct scb_data *scb_data;
4376
4377         scb_data = ahc->scb_data;
4378         if (scb_data == NULL)
4379                 return;
4380
4381         switch (scb_data->init_level) {
4382         default:
4383         case 7:
4384         {
4385                 struct sg_map_node *sg_map;
4386
4387                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps))!= NULL) {
4388                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4389                         ahc_dmamap_unload(ahc, scb_data->sg_dmat,
4390                                           sg_map->sg_dmamap);
4391                         ahc_dmamem_free(ahc, scb_data->sg_dmat,
4392                                         sg_map->sg_vaddr,
4393                                         sg_map->sg_dmamap);
4394                         kfree(sg_map, M_DEVBUF);
4395                 }
4396                 ahc_dma_tag_destroy(ahc, scb_data->sg_dmat);
4397         }
4398         case 6:
4399                 ahc_dmamap_unload(ahc, scb_data->sense_dmat,
4400                                   scb_data->sense_dmamap);
4401         case 5:
4402                 ahc_dmamem_free(ahc, scb_data->sense_dmat, scb_data->sense,
4403                                 scb_data->sense_dmamap);
4404                 ahc_dmamap_destroy(ahc, scb_data->sense_dmat,
4405                                    scb_data->sense_dmamap);
4406         case 4:
4407                 ahc_dma_tag_destroy(ahc, scb_data->sense_dmat);
4408         case 3:
4409                 ahc_dmamap_unload(ahc, scb_data->hscb_dmat,
4410                                   scb_data->hscb_dmamap);
4411         case 2:
4412                 ahc_dmamem_free(ahc, scb_data->hscb_dmat, scb_data->hscbs,
4413                                 scb_data->hscb_dmamap);
4414                 ahc_dmamap_destroy(ahc, scb_data->hscb_dmat,
4415                                    scb_data->hscb_dmamap);
4416         case 1:
4417                 ahc_dma_tag_destroy(ahc, scb_data->hscb_dmat);
4418                 break;
4419         case 0:
4420                 break;
4421         }
4422         if (scb_data->scbarray != NULL)
4423                 kfree(scb_data->scbarray, M_DEVBUF);
4424 }
4425
4426 void
4427 ahc_alloc_scbs(struct ahc_softc *ahc)
4428 {
4429         struct scb_data *scb_data;
4430         struct scb *next_scb;
4431         struct sg_map_node *sg_map;
4432         bus_addr_t physaddr;
4433         struct ahc_dma_seg *segs;
4434         int newcount;
4435         int i;
4436
4437         scb_data = ahc->scb_data;
4438         if (scb_data->numscbs >= AHC_SCB_MAX_ALLOC)
4439                 /* Can't allocate any more */
4440                 return;
4441
4442         next_scb = &scb_data->scbarray[scb_data->numscbs];
4443
4444         sg_map = kmalloc(sizeof(*sg_map), M_DEVBUF, M_INTWAIT);
4445
4446         /* Allocate S/G space for the next batch of SCBS */
4447         if (ahc_dmamem_alloc(ahc, scb_data->sg_dmat,
4448                              (void **)&sg_map->sg_vaddr,
4449                              BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
4450                 kfree(sg_map, M_DEVBUF);
4451                 return;
4452         }
4453
4454         SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
4455
4456         ahc_dmamap_load(ahc, scb_data->sg_dmat, sg_map->sg_dmamap,
4457                         sg_map->sg_vaddr, PAGE_SIZE, ahc_dmamap_cb,
4458                         &sg_map->sg_physaddr, /*flags*/0);
4459
4460         segs = sg_map->sg_vaddr;
4461         physaddr = sg_map->sg_physaddr;
4462
4463         newcount = (PAGE_SIZE / (AHC_NSEG * sizeof(struct ahc_dma_seg)));
4464         newcount = MIN(newcount, (AHC_SCB_MAX_ALLOC - scb_data->numscbs));
4465         for (i = 0; i < newcount; i++) {
4466                 struct scb_platform_data *pdata;
4467 #ifndef __linux__
4468                 int error;
4469 #endif
4470                 pdata = kmalloc(sizeof(*pdata), M_DEVBUF, M_INTWAIT);
4471                 next_scb->platform_data = pdata;
4472                 next_scb->sg_map = sg_map;
4473                 next_scb->sg_list = segs;
4474                 /*
4475                  * The sequencer always starts with the second entry.
4476                  * The first entry is embedded in the scb.
4477                  */
4478                 next_scb->sg_list_phys = physaddr + sizeof(struct ahc_dma_seg);
4479                 next_scb->ahc_softc = ahc;
4480                 next_scb->flags = SCB_FREE;
4481 #ifndef __linux__
4482                 error = ahc_dmamap_create(ahc, ahc->buffer_dmat, /*flags*/0,
4483                                           &next_scb->dmamap);
4484                 if (error != 0)
4485                         break;
4486 #endif
4487                 next_scb->hscb = &scb_data->hscbs[scb_data->numscbs];
4488                 next_scb->hscb->tag = ahc->scb_data->numscbs;
4489                 SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs,
4490                                   next_scb, links.sle);
4491                 segs += AHC_NSEG;
4492                 physaddr += (AHC_NSEG * sizeof(struct ahc_dma_seg));
4493                 next_scb++;
4494                 ahc->scb_data->numscbs++;
4495         }
4496 }
4497
4498 void
4499 ahc_controller_info(struct ahc_softc *ahc, char *buf)
4500 {
4501         int len;
4502
4503         len = sprintf(buf, "%s: ", ahc_chip_names[ahc->chip & AHC_CHIPID_MASK]);
4504         buf += len;
4505         if ((ahc->features & AHC_TWIN) != 0)
4506                 len = sprintf(buf, "Twin Channel, A SCSI Id=%d, "
4507                               "B SCSI Id=%d, primary %c, ",
4508                               ahc->our_id, ahc->our_id_b,
4509                               (ahc->flags & AHC_PRIMARY_CHANNEL) + 'A');
4510         else {
4511                 const char *speed;
4512                 const char *type;
4513
4514                 speed = "";
4515                 if ((ahc->features & AHC_ULTRA) != 0) {
4516                         speed = "Ultra ";
4517                 } else if ((ahc->features & AHC_DT) != 0) {
4518                         speed = "Ultra160 ";
4519                 } else if ((ahc->features & AHC_ULTRA2) != 0) {
4520                         speed = "Ultra2 ";
4521                 }
4522                 if ((ahc->features & AHC_WIDE) != 0) {
4523                         type = "Wide";
4524                 } else {
4525                         type = "Single";
4526                 }
4527                 len = sprintf(buf, "%s%s Channel %c, SCSI Id=%d, ",
4528                               speed, type, ahc->channel, ahc->our_id);
4529         }
4530         buf += len;
4531
4532         if ((ahc->flags & AHC_PAGESCBS) != 0)
4533                 sprintf(buf, "%d/%d SCBs",
4534                         ahc->scb_data->maxhscbs, AHC_MAX_QUEUE);
4535         else
4536                 sprintf(buf, "%d SCBs", ahc->scb_data->maxhscbs);
4537 }
4538
4539 int
4540 ahc_chip_init(struct ahc_softc *ahc)
4541 {
4542         int      term;
4543         int      error;
4544         u_int    i;
4545         u_int    scsi_conf;
4546         u_int    scsiseq_template;
4547         uint32_t physaddr;
4548
4549         ahc_outb(ahc, SEQ_FLAGS, 0);
4550         ahc_outb(ahc, SEQ_FLAGS2, 0);
4551
4552         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1, for both channels*/
4553         if (ahc->features & AHC_TWIN) {
4554
4555                 /*
4556                  * Setup Channel B first.
4557                  */
4558                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) | SELBUSB);
4559                 term = (ahc->flags & AHC_TERM_ENB_B) != 0 ? STPWEN : 0;
4560                 ahc_outb(ahc, SCSIID, ahc->our_id_b);
4561                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
4562                 ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
4563                                         |term|ahc->seltime_b|ENSTIMER|ACTNEGEN);
4564                 if ((ahc->features & AHC_ULTRA2) != 0)
4565                         ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
4566                 ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
4567                 ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
4568
4569                 /* Select Channel A */
4570                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) & ~SELBUSB);
4571         }
4572         term = (ahc->flags & AHC_TERM_ENB_A) != 0 ? STPWEN : 0;
4573         if ((ahc->features & AHC_ULTRA2) != 0)
4574                 ahc_outb(ahc, SCSIID_ULTRA2, ahc->our_id);
4575         else
4576                 ahc_outb(ahc, SCSIID, ahc->our_id);
4577         scsi_conf = ahc_inb(ahc, SCSICONF);
4578         ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
4579                                 |term|ahc->seltime
4580                                 |ENSTIMER|ACTNEGEN);
4581         if ((ahc->features & AHC_ULTRA2) != 0)
4582                 ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
4583         ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
4584         ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
4585
4586         /* There are no untagged SCBs active yet. */
4587         for (i = 0; i < 16; i++) {
4588                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, 0));
4589                 if ((ahc->flags & AHC_SCB_BTT) != 0) {
4590                         int lun;
4591
4592                         /*
4593                          * The SCB based BTT allows an entry per
4594                          * target and lun pair.
4595                          */
4596                         for (lun = 1; lun < AHC_NUM_LUNS; lun++)
4597                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, lun));
4598                 }
4599         }
4600
4601         /* All of our queues are empty */
4602         for (i = 0; i < 256; i++)
4603                 ahc->qoutfifo[i] = SCB_LIST_NULL;
4604         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_PREREAD);
4605
4606         for (i = 0; i < 256; i++)
4607                 ahc->qinfifo[i] = SCB_LIST_NULL;
4608
4609         if ((ahc->features & AHC_MULTI_TID) != 0) {
4610                 ahc_outb(ahc, TARGID, 0);
4611                 ahc_outb(ahc, TARGID + 1, 0);
4612         }
4613
4614         /*
4615          * Tell the sequencer where it can find our arrays in memory.
4616          */
4617         physaddr = ahc->scb_data->hscb_busaddr;
4618         ahc_outb(ahc, HSCB_ADDR, physaddr & 0xFF);
4619         ahc_outb(ahc, HSCB_ADDR + 1, (physaddr >> 8) & 0xFF);
4620         ahc_outb(ahc, HSCB_ADDR + 2, (physaddr >> 16) & 0xFF);
4621         ahc_outb(ahc, HSCB_ADDR + 3, (physaddr >> 24) & 0xFF);
4622
4623         physaddr = ahc->shared_data_busaddr;
4624         ahc_outb(ahc, SHARED_DATA_ADDR, physaddr & 0xFF);
4625         ahc_outb(ahc, SHARED_DATA_ADDR + 1, (physaddr >> 8) & 0xFF);
4626         ahc_outb(ahc, SHARED_DATA_ADDR + 2, (physaddr >> 16) & 0xFF);
4627         ahc_outb(ahc, SHARED_DATA_ADDR + 3, (physaddr >> 24) & 0xFF);
4628
4629         /*
4630          * Initialize the group code to command length table.
4631          * This overrides the values in TARG_SCSIRATE, so only
4632          * setup the table after we have processed that information.
4633          */
4634         ahc_outb(ahc, CMDSIZE_TABLE, 5);
4635         ahc_outb(ahc, CMDSIZE_TABLE + 1, 9);
4636         ahc_outb(ahc, CMDSIZE_TABLE + 2, 9);
4637         ahc_outb(ahc, CMDSIZE_TABLE + 3, 0);
4638         ahc_outb(ahc, CMDSIZE_TABLE + 4, 15);
4639         ahc_outb(ahc, CMDSIZE_TABLE + 5, 11);
4640         ahc_outb(ahc, CMDSIZE_TABLE + 6, 0);
4641         ahc_outb(ahc, CMDSIZE_TABLE + 7, 0);
4642
4643         if ((ahc->features & AHC_HS_MAILBOX) != 0)
4644                 ahc_outb(ahc, HS_MAILBOX, 0);
4645
4646         /* Tell the sequencer of our initial queue positions */
4647         if ((ahc->features & AHC_TARGETMODE) != 0) {
4648                 ahc->tqinfifonext = 1;
4649                 ahc_outb(ahc, KERNEL_TQINPOS, ahc->tqinfifonext - 1);
4650                 ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
4651         }
4652         ahc->qinfifonext = 0;
4653         ahc->qoutfifonext = 0;
4654         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
4655                 ahc_outb(ahc, QOFF_CTLSTA, SCB_QSIZE_256);
4656                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
4657                 ahc_outb(ahc, SNSCB_QOFF, ahc->qinfifonext);
4658                 ahc_outb(ahc, SDSCB_QOFF, 0);
4659         } else {
4660                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
4661                 ahc_outb(ahc, QINPOS, ahc->qinfifonext);
4662                 ahc_outb(ahc, QOUTPOS, ahc->qoutfifonext);
4663         }
4664
4665         /* We don't have any waiting selections */
4666         ahc_outb(ahc, WAITING_SCBH, SCB_LIST_NULL);
4667
4668         /* Our disconnection list is empty too */
4669         ahc_outb(ahc, DISCONNECTED_SCBH, SCB_LIST_NULL);
4670
4671         /* Message out buffer starts empty */
4672         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
4673
4674         /*
4675          * Setup the allowed SCSI Sequences based on operational mode.
4676          * If we are a target, we'll enalbe select in operations once
4677          * we've had a lun enabled.
4678          */
4679         scsiseq_template = ENSELO|ENAUTOATNO|ENAUTOATNP;
4680         if ((ahc->flags & AHC_INITIATORROLE) != 0)
4681                 scsiseq_template |= ENRSELI;
4682         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq_template);
4683
4684         /* Initialize our list of free SCBs. */
4685         ahc_build_free_scb_list(ahc);
4686
4687         /*
4688          * Tell the sequencer which SCB will be the next one it receives.
4689          */
4690         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
4691
4692         /*
4693          * Load the Sequencer program and Enable the adapter
4694          * in "fast" mode.
4695          */
4696         if (bootverbose)
4697                 printf("%s: Downloading Sequencer Program...",
4698                        ahc_name(ahc));
4699
4700         error = ahc_loadseq(ahc);
4701         if (error != 0)
4702                 return (error);
4703
4704         if ((ahc->features & AHC_ULTRA2) != 0) {
4705                 int wait;
4706
4707                 /*
4708                  * Wait for up to 500ms for our transceivers
4709                  * to settle.  If the adapter does not have
4710                  * a cable attached, the transceivers may
4711                  * never settle, so don't complain if we
4712                  * fail here.
4713                  */
4714                 for (wait = 5000;
4715                      (ahc_inb(ahc, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
4716                      wait--)
4717                         ahc_delay(100);
4718         }
4719         ahc_restart(ahc);
4720         return (0);
4721 }
4722
4723 /*
4724  * Start the board, ready for normal operation
4725  */
4726 int
4727 ahc_init(struct ahc_softc *ahc)
4728 {
4729         int      max_targ;
4730         u_int    i;
4731         u_int    scsi_conf;
4732         u_int    ultraenb;
4733         u_int    discenable;
4734         u_int    tagenable;
4735         size_t   driver_data_size;
4736
4737 #ifdef AHC_DEBUG
4738         if ((ahc_debug & AHC_DEBUG_SEQUENCER) != 0)
4739                 ahc->flags |= AHC_SEQUENCER_DEBUG;
4740 #endif
4741
4742 #ifdef AHC_PRINT_SRAM
4743         printf("Scratch Ram:");
4744         for (i = 0x20; i < 0x5f; i++) {
4745                 if (((i % 8) == 0) && (i != 0)) {
4746                         printf ("\n              ");
4747                 }
4748                 printf (" 0x%x", ahc_inb(ahc, i));
4749         }
4750         if ((ahc->features & AHC_MORE_SRAM) != 0) {
4751                 for (i = 0x70; i < 0x7f; i++) {
4752                         if (((i % 8) == 0) && (i != 0)) {
4753                                 printf ("\n              ");
4754                         }
4755                         printf (" 0x%x", ahc_inb(ahc, i));
4756                 }
4757         }
4758         printf ("\n");
4759         /*
4760          * Reading uninitialized scratch ram may
4761          * generate parity errors.
4762          */
4763         ahc_outb(ahc, CLRINT, CLRPARERR);
4764         ahc_outb(ahc, CLRINT, CLRBRKADRINT);
4765 #endif
4766         max_targ = 15;
4767
4768         /*
4769          * Assume we have a board at this stage and it has been reset.
4770          */
4771         if ((ahc->flags & AHC_USEDEFAULTS) != 0)
4772                 ahc->our_id = ahc->our_id_b = 7;
4773
4774         /*
4775          * Default to allowing initiator operations.
4776          */
4777         ahc->flags |= AHC_INITIATORROLE;
4778
4779         /*
4780          * Only allow target mode features if this unit has them enabled.
4781          */
4782         if ((AHC_TMODE_ENABLE & (0x1 << ahc->unit)) == 0)
4783                 ahc->features &= ~AHC_TARGETMODE;
4784
4785 #ifndef __linux__
4786         /* DMA tag for mapping buffers into device visible space. */
4787         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4788                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4789                                /*lowaddr*/ahc->flags & AHC_39BIT_ADDRESSING
4790                                         ? (bus_addr_t)0x7FFFFFFFFFULL
4791                                         : BUS_SPACE_MAXADDR_32BIT,
4792                                /*highaddr*/BUS_SPACE_MAXADDR,
4793                                /*filter*/NULL, /*filterarg*/NULL,
4794                                /*maxsize*/(AHC_NSEG - 1) * PAGE_SIZE,
4795                                /*nsegments*/AHC_NSEG,
4796                                /*maxsegsz*/AHC_MAXTRANSFER_SIZE,
4797                                /*flags*/BUS_DMA_ALLOCNOW,
4798                                &ahc->buffer_dmat) != 0) {
4799                 return (ENOMEM);
4800         }
4801 #endif
4802
4803         ahc->init_level++;
4804
4805         /*
4806          * DMA tag for our command fifos and other data in system memory
4807          * the card's sequencer must be able to access.  For initiator
4808          * roles, we need to allocate space for the qinfifo and qoutfifo.
4809          * The qinfifo and qoutfifo are composed of 256 1 byte elements.
4810          * When providing for the target mode role, we must additionally
4811          * provide space for the incoming target command fifo and an extra
4812          * byte to deal with a dma bug in some chip versions.
4813          */
4814         driver_data_size = 2 * 256 * sizeof(uint8_t);
4815         if ((ahc->features & AHC_TARGETMODE) != 0)
4816                 driver_data_size += AHC_TMODE_CMDS * sizeof(struct target_cmd)
4817                                  + /*DMA WideOdd Bug Buffer*/1;
4818         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4819                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4820                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4821                                /*highaddr*/BUS_SPACE_MAXADDR,
4822                                /*filter*/NULL, /*filterarg*/NULL,
4823                                driver_data_size,
4824                                /*nsegments*/1,
4825                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4826                                /*flags*/0, &ahc->shared_data_dmat) != 0) {
4827                 return (ENOMEM);
4828         }
4829
4830         ahc->init_level++;
4831
4832         /* Allocation of driver data */
4833         if (ahc_dmamem_alloc(ahc, ahc->shared_data_dmat,
4834                              (void **)&ahc->qoutfifo,
4835                              BUS_DMA_NOWAIT, &ahc->shared_data_dmamap) != 0) {
4836                 return (ENOMEM);
4837         }
4838
4839         ahc->init_level++;
4840
4841         /* And permanently map it in */
4842         ahc_dmamap_load(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
4843                         ahc->qoutfifo, driver_data_size, ahc_dmamap_cb,
4844                         &ahc->shared_data_busaddr, /*flags*/0);
4845
4846         if ((ahc->features & AHC_TARGETMODE) != 0) {
4847                 ahc->targetcmds = (struct target_cmd *)ahc->qoutfifo;
4848                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[AHC_TMODE_CMDS];
4849                 ahc->dma_bug_buf = ahc->shared_data_busaddr
4850                                  + driver_data_size - 1;
4851                 /* All target command blocks start out invalid. */
4852                 for (i = 0; i < AHC_TMODE_CMDS; i++)
4853                         ahc->targetcmds[i].cmd_valid = 0;
4854                 ahc_sync_tqinfifo(ahc, BUS_DMASYNC_PREREAD);
4855                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[256];
4856         }
4857         ahc->qinfifo = &ahc->qoutfifo[256];
4858
4859         ahc->init_level++;
4860
4861         /* Allocate SCB data now that buffer_dmat is initialized */
4862         if (ahc->scb_data->maxhscbs == 0)
4863                 if (ahc_init_scbdata(ahc) != 0)
4864                         return (ENOMEM);
4865
4866         /*
4867          * Allocate a tstate to house information for our
4868          * initiator presence on the bus as well as the user
4869          * data for any target mode initiator.
4870          */
4871         if (ahc_alloc_tstate(ahc, ahc->our_id, 'A') == NULL) {
4872                 printf("%s: unable to allocate ahc_tmode_tstate.  "
4873                        "Failing attach\n", ahc_name(ahc));
4874                 return (ENOMEM);
4875         }
4876
4877         if ((ahc->features & AHC_TWIN) != 0) {
4878                 if (ahc_alloc_tstate(ahc, ahc->our_id_b, 'B') == NULL) {
4879                         printf("%s: unable to allocate ahc_tmode_tstate.  "
4880                                "Failing attach\n", ahc_name(ahc));
4881                         return (ENOMEM);
4882                 }
4883         }
4884
4885         if (ahc->scb_data->maxhscbs < AHC_SCB_MAX_ALLOC) {
4886                 ahc->flags |= AHC_PAGESCBS;
4887         } else {
4888                 ahc->flags &= ~AHC_PAGESCBS;
4889         }
4890
4891 #ifdef AHC_DEBUG
4892         if (ahc_debug & AHC_SHOW_MISC) {
4893                 printf("%s: hardware scb %u bytes; kernel scb %u bytes; "
4894                        "ahc_dma %u bytes\n",
4895                         ahc_name(ahc),
4896                         (u_int)sizeof(struct hardware_scb),
4897                         (u_int)sizeof(struct scb),
4898                         (u_int)sizeof(struct ahc_dma_seg));
4899         }
4900 #endif /* AHC_DEBUG */
4901
4902         /*
4903          * Look at the information that board initialization or
4904          * the board bios has left us.
4905          */
4906         if (ahc->features & AHC_TWIN) {
4907                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
4908                 if ((scsi_conf & RESET_SCSI) != 0
4909                  && (ahc->flags & AHC_INITIATORROLE) != 0)
4910                         ahc->flags |= AHC_RESET_BUS_B;
4911         }
4912
4913         scsi_conf = ahc_inb(ahc, SCSICONF);
4914         if ((scsi_conf & RESET_SCSI) != 0
4915          && (ahc->flags & AHC_INITIATORROLE) != 0)
4916                 ahc->flags |= AHC_RESET_BUS_A;
4917
4918         ultraenb = 0;
4919         tagenable = ALL_TARGETS_MASK;
4920
4921         /* Grab the disconnection disable table and invert it for our needs */
4922         if ((ahc->flags & AHC_USEDEFAULTS) != 0) {
4923                 printf("%s: Host Adapter Bios disabled.  Using default SCSI "
4924                         "device parameters\n", ahc_name(ahc));
4925                 ahc->flags |= AHC_EXTENDED_TRANS_A|AHC_EXTENDED_TRANS_B|
4926                               AHC_TERM_ENB_A|AHC_TERM_ENB_B;
4927                 discenable = ALL_TARGETS_MASK;
4928                 if ((ahc->features & AHC_ULTRA) != 0)
4929                         ultraenb = ALL_TARGETS_MASK;
4930         } else {
4931                 discenable = ~((ahc_inb(ahc, DISC_DSB + 1) << 8)
4932                            | ahc_inb(ahc, DISC_DSB));
4933                 if ((ahc->features & (AHC_ULTRA|AHC_ULTRA2)) != 0)
4934                         ultraenb = (ahc_inb(ahc, ULTRA_ENB + 1) << 8)
4935                                       | ahc_inb(ahc, ULTRA_ENB);
4936         }
4937
4938         if ((ahc->features & (AHC_WIDE|AHC_TWIN)) == 0)
4939                 max_targ = 7;
4940
4941         for (i = 0; i <= max_targ; i++) {
4942                 struct ahc_initiator_tinfo *tinfo;
4943                 struct ahc_tmode_tstate *tstate;
4944                 u_int our_id;
4945                 u_int target_id;
4946                 char channel;
4947
4948                 channel = 'A';
4949                 our_id = ahc->our_id;
4950                 target_id = i;
4951                 if (i > 7 && (ahc->features & AHC_TWIN) != 0) {
4952                         channel = 'B';
4953                         our_id = ahc->our_id_b;
4954                         target_id = i % 8;
4955                 }
4956                 tinfo = ahc_fetch_transinfo(ahc, channel, our_id,
4957                                             target_id, &tstate);
4958                 /* Default to async narrow across the board */
4959                 memset(tinfo, 0, sizeof(*tinfo));
4960                 if (ahc->flags & AHC_USEDEFAULTS) {
4961                         if ((ahc->features & AHC_WIDE) != 0)
4962                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
4963
4964                         /*
4965                          * These will be truncated when we determine the
4966                          * connection type we have with the target.
4967                          */
4968                         tinfo->user.period = ahc_syncrates->period;
4969                         tinfo->user.offset = MAX_OFFSET;
4970                 } else {
4971                         u_int scsirate;
4972                         uint16_t mask;
4973
4974                         /* Take the settings leftover in scratch RAM. */
4975                         scsirate = ahc_inb(ahc, TARG_SCSIRATE + i);
4976                         mask = (0x01 << i);
4977                         if ((ahc->features & AHC_ULTRA2) != 0) {
4978                                 u_int offset;
4979                                 u_int maxsync;
4980
4981                                 if ((scsirate & SOFS) == 0x0F) {
4982                                         /*
4983                                          * Haven't negotiated yet,
4984                                          * so the format is different.
4985                                          */
4986                                         scsirate = (scsirate & SXFR) >> 4
4987                                                  | (ultraenb & mask)
4988                                                   ? 0x08 : 0x0
4989                                                  | (scsirate & WIDEXFER);
4990                                         offset = MAX_OFFSET_ULTRA2;
4991                                 } else
4992                                         offset = ahc_inb(ahc, TARG_OFFSET + i);
4993                                 if ((scsirate & ~WIDEXFER) == 0 && offset != 0)
4994                                         /* Set to the lowest sync rate, 5MHz */
4995                                         scsirate |= 0x1c;
4996                                 maxsync = AHC_SYNCRATE_ULTRA2;
4997                                 if ((ahc->features & AHC_DT) != 0)
4998                                         maxsync = AHC_SYNCRATE_DT;
4999                                 tinfo->user.period =
5000                                     ahc_find_period(ahc, scsirate, maxsync);
5001                                 if (offset == 0)
5002                                         tinfo->user.period = 0;
5003                                 else
5004                                         tinfo->user.offset = MAX_OFFSET;
5005                                 if ((scsirate & SXFR_ULTRA2) <= 8/*10MHz*/
5006                                  && (ahc->features & AHC_DT) != 0)
5007                                         tinfo->user.ppr_options =
5008                                             MSG_EXT_PPR_DT_REQ;
5009                         } else if ((scsirate & SOFS) != 0) {
5010                                 if ((scsirate & SXFR) == 0x40
5011                                  && (ultraenb & mask) != 0) {
5012                                         /* Treat 10MHz as a non-ultra speed */
5013                                         scsirate &= ~SXFR;
5014                                         ultraenb &= ~mask;
5015                                 }
5016                                 tinfo->user.period =
5017                                     ahc_find_period(ahc, scsirate,
5018                                                     (ultraenb & mask)
5019                                                    ? AHC_SYNCRATE_ULTRA
5020                                                    : AHC_SYNCRATE_FAST);
5021                                 if (tinfo->user.period != 0)
5022                                         tinfo->user.offset = MAX_OFFSET;
5023                         }
5024                         if (tinfo->user.period == 0)
5025                                 tinfo->user.offset = 0;
5026                         if ((scsirate & WIDEXFER) != 0
5027                          && (ahc->features & AHC_WIDE) != 0)
5028                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5029                         tinfo->user.protocol_version = 4;
5030                         if ((ahc->features & AHC_DT) != 0)
5031                                 tinfo->user.transport_version = 3;
5032                         else
5033                                 tinfo->user.transport_version = 2;
5034                         tinfo->goal.protocol_version = 2;
5035                         tinfo->goal.transport_version = 2;
5036                         tinfo->curr.protocol_version = 2;
5037                         tinfo->curr.transport_version = 2;
5038                 }
5039                 tstate->ultraenb = 0;
5040         }
5041         ahc->user_discenable = discenable;
5042         ahc->user_tagenable = tagenable;
5043
5044         return (ahc->bus_chip_init(ahc));
5045 }
5046
5047 void
5048 ahc_intr_enable(struct ahc_softc *ahc, int enable)
5049 {
5050         u_int hcntrl;
5051
5052         hcntrl = ahc_inb(ahc, HCNTRL);
5053         hcntrl &= ~INTEN;
5054         ahc->pause &= ~INTEN;
5055         ahc->unpause &= ~INTEN;
5056         if (enable) {
5057                 hcntrl |= INTEN;
5058                 ahc->pause |= INTEN;
5059                 ahc->unpause |= INTEN;
5060         }
5061         ahc_outb(ahc, HCNTRL, hcntrl);
5062 }
5063
5064 /*
5065  * Ensure that the card is paused in a location
5066  * outside of all critical sections and that all
5067  * pending work is completed prior to returning.
5068  * This routine should only be called from outside
5069  * an interrupt context.
5070  */
5071 void
5072 ahc_pause_and_flushwork(struct ahc_softc *ahc)
5073 {
5074         int intstat;
5075         int maxloops;
5076         int paused;
5077
5078         maxloops = 1000;
5079         ahc->flags |= AHC_ALL_INTERRUPTS;
5080         paused = FALSE;
5081         do {
5082                 if (paused)
5083                         ahc_unpause(ahc);
5084                 ahc_intr(ahc);
5085                 ahc_pause(ahc);
5086                 paused = TRUE;
5087                 ahc_outb(ahc, SCSISEQ, ahc_inb(ahc, SCSISEQ) & ~ENSELO);
5088                 ahc_clear_critical_section(ahc);
5089                 intstat = ahc_inb(ahc, INTSTAT);
5090         } while (--maxloops
5091               && (intstat != 0xFF || (ahc->features & AHC_REMOVABLE) == 0)
5092               && ((intstat & INT_PEND) != 0
5093                || (ahc_inb(ahc, SSTAT0) & (SELDO|SELINGO)) != 0));
5094         if (maxloops == 0) {
5095                 printf("Infinite interrupt loop, INTSTAT = %x",
5096                        ahc_inb(ahc, INTSTAT));
5097         }
5098         ahc_platform_flushwork(ahc);
5099         ahc->flags &= ~AHC_ALL_INTERRUPTS;
5100 }
5101
5102 int
5103 ahc_suspend(struct ahc_softc *ahc)
5104 {
5105
5106         ahc_pause_and_flushwork(ahc);
5107
5108         if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
5109                 ahc_unpause(ahc);
5110                 return (EBUSY);
5111         }
5112
5113 #ifdef AHC_TARGET_MODE
5114         /*
5115          * XXX What about ATIOs that have not yet been serviced?
5116          * Perhaps we should just refuse to be suspended if we
5117          * are acting in a target role.
5118          */
5119         if (ahc->pending_device != NULL) {
5120                 ahc_unpause(ahc);
5121                 return (EBUSY);
5122         }
5123 #endif
5124         ahc_shutdown(ahc);
5125         return (0);
5126 }
5127
5128 int
5129 ahc_resume(struct ahc_softc *ahc)
5130 {
5131
5132         ahc_reset(ahc, /*reinit*/TRUE);
5133         ahc_intr_enable(ahc, TRUE);
5134         ahc_restart(ahc);
5135         return (0);
5136 }
5137
5138 /************************** Busy Target Table *********************************/
5139 /*
5140  * Return the untagged transaction id for a given target/channel lun.
5141  * Optionally, clear the entry.
5142  */
5143 u_int
5144 ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl)
5145 {
5146         u_int scbid;
5147         u_int target_offset;
5148
5149         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5150                 u_int saved_scbptr;
5151
5152                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5153                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5154                 scbid = ahc_inb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl));
5155                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5156         } else {
5157                 target_offset = TCL_TARGET_OFFSET(tcl);
5158                 scbid = ahc_inb(ahc, BUSY_TARGETS + target_offset);
5159         }
5160
5161         return (scbid);
5162 }
5163
5164 void
5165 ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl)
5166 {
5167         u_int target_offset;
5168
5169         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5170                 u_int saved_scbptr;
5171
5172                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5173                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5174                 ahc_outb(ahc, SCB_64_BTT+TCL_TARGET_OFFSET(tcl), SCB_LIST_NULL);
5175                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5176         } else {
5177                 target_offset = TCL_TARGET_OFFSET(tcl);
5178                 ahc_outb(ahc, BUSY_TARGETS + target_offset, SCB_LIST_NULL);
5179         }
5180 }
5181
5182 void
5183 ahc_busy_tcl(struct ahc_softc *ahc, u_int tcl, u_int scbid)
5184 {
5185         u_int target_offset;
5186
5187         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5188                 u_int saved_scbptr;
5189
5190                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5191                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5192                 ahc_outb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl), scbid);
5193                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5194         } else {
5195                 target_offset = TCL_TARGET_OFFSET(tcl);
5196                 ahc_outb(ahc, BUSY_TARGETS + target_offset, scbid);
5197         }
5198 }
5199
5200 /************************** SCB and SCB queue management **********************/
5201 int
5202 ahc_match_scb(struct ahc_softc *ahc, struct scb *scb, int target,
5203               char channel, int lun, u_int tag, role_t role)
5204 {
5205         int targ = SCB_GET_TARGET(ahc, scb);
5206         char chan = SCB_GET_CHANNEL(ahc, scb);
5207         int slun = SCB_GET_LUN(scb);
5208         int match;
5209
5210         match = ((chan == channel) || (channel == ALL_CHANNELS));
5211         if (match != 0)
5212                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
5213         if (match != 0)
5214                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
5215         if (match != 0) {
5216 #ifdef AHC_TARGET_MODE
5217                 int group;
5218
5219                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
5220                 if (role == ROLE_INITIATOR) {
5221                         match = (group != XPT_FC_GROUP_TMODE)
5222                               && ((tag == scb->hscb->tag)
5223                                || (tag == SCB_LIST_NULL));
5224                 } else if (role == ROLE_TARGET) {
5225                         match = (group == XPT_FC_GROUP_TMODE)
5226                               && ((tag == scb->io_ctx->csio.tag_id)
5227                                || (tag == SCB_LIST_NULL));
5228                 }
5229 #else /* !AHC_TARGET_MODE */
5230                 match = ((tag == scb->hscb->tag) || (tag == SCB_LIST_NULL));
5231 #endif /* AHC_TARGET_MODE */
5232         }
5233
5234         return match;
5235 }
5236
5237 void
5238 ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb)
5239 {
5240         int     target;
5241         char    channel;
5242         int     lun;
5243
5244         target = SCB_GET_TARGET(ahc, scb);
5245         lun = SCB_GET_LUN(scb);
5246         channel = SCB_GET_CHANNEL(ahc, scb);
5247
5248         ahc_search_qinfifo(ahc, target, channel, lun,
5249                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
5250                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5251
5252         ahc_platform_freeze_devq(ahc, scb);
5253 }
5254
5255 void
5256 ahc_qinfifo_requeue_tail(struct ahc_softc *ahc, struct scb *scb)
5257 {
5258         struct scb *prev_scb;
5259
5260         prev_scb = NULL;
5261         if (ahc_qinfifo_count(ahc) != 0) {
5262                 u_int prev_tag;
5263                 uint8_t prev_pos;
5264
5265                 prev_pos = ahc->qinfifonext - 1;
5266                 prev_tag = ahc->qinfifo[prev_pos];
5267                 prev_scb = ahc_lookup_scb(ahc, prev_tag);
5268         }
5269         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5270         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5271                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5272         } else {
5273                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5274         }
5275 }
5276
5277 static void
5278 ahc_qinfifo_requeue(struct ahc_softc *ahc, struct scb *prev_scb,
5279                     struct scb *scb)
5280 {
5281         if (prev_scb == NULL) {
5282                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5283         } else {
5284                 prev_scb->hscb->next = scb->hscb->tag;
5285                 ahc_sync_scb(ahc, prev_scb,
5286                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5287         }
5288         ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
5289         scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5290         ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5291 }
5292
5293 static int
5294 ahc_qinfifo_count(struct ahc_softc *ahc)
5295 {
5296         uint8_t qinpos;
5297         uint8_t diff;
5298
5299         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5300                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
5301                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
5302         } else
5303                 qinpos = ahc_inb(ahc, QINPOS);
5304         diff = ahc->qinfifonext - qinpos;
5305         return (diff);
5306 }
5307
5308 int
5309 ahc_search_qinfifo(struct ahc_softc *ahc, int target, char channel,
5310                    int lun, u_int tag, role_t role, uint32_t status,
5311                    ahc_search_action action)
5312 {
5313         struct  scb *scb;
5314         struct  scb *prev_scb;
5315         uint8_t qinstart;
5316         uint8_t qinpos;
5317         uint8_t qintail;
5318         uint8_t next;
5319         uint8_t prev;
5320         uint8_t curscbptr;
5321         int     found;
5322         int     have_qregs;
5323
5324         qintail = ahc->qinfifonext;
5325         have_qregs = (ahc->features & AHC_QUEUE_REGS) != 0;
5326         if (have_qregs) {
5327                 qinstart = ahc_inb(ahc, SNSCB_QOFF);
5328                 ahc_outb(ahc, SNSCB_QOFF, qinstart);
5329         } else
5330                 qinstart = ahc_inb(ahc, QINPOS);
5331         qinpos = qinstart;
5332         found = 0;
5333         prev_scb = NULL;
5334
5335         if (action == SEARCH_COMPLETE) {
5336                 /*
5337                  * Don't attempt to run any queued untagged transactions
5338                  * until we are done with the abort process.
5339                  */
5340                 ahc_freeze_untagged_queues(ahc);
5341         }
5342
5343         /*
5344          * Start with an empty queue.  Entries that are not chosen
5345          * for removal will be re-added to the queue as we go.
5346          */
5347         ahc->qinfifonext = qinpos;
5348         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5349
5350         while (qinpos != qintail) {
5351                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinpos]);
5352                 if (scb == NULL) {
5353                         printf("qinpos = %d, SCB index = %d\n",
5354                                 qinpos, ahc->qinfifo[qinpos]);
5355                         panic("Loop 1\n");
5356                 }
5357
5358                 if (ahc_match_scb(ahc, scb, target, channel, lun, tag, role)) {
5359                         /*
5360                          * We found an scb that needs to be acted on.
5361                          */
5362                         found++;
5363                         switch (action) {
5364                         case SEARCH_COMPLETE:
5365                         {
5366                                 cam_status ostat;
5367                                 cam_status cstat;
5368
5369                                 ostat = ahc_get_transaction_status(scb);
5370                                 if (ostat == CAM_REQ_INPROG)
5371                                         ahc_set_transaction_status(scb, status);
5372                                 cstat = ahc_get_transaction_status(scb);
5373                                 if (cstat != CAM_REQ_CMP)
5374                                         ahc_freeze_scb(scb);
5375                                 if ((scb->flags & SCB_ACTIVE) == 0)
5376                                         printf("Inactive SCB in qinfifo\n");
5377                                 ahc_done(ahc, scb);
5378
5379                                 /* FALLTHROUGH */
5380                         }
5381                         case SEARCH_REMOVE:
5382                                 break;
5383                         case SEARCH_COUNT:
5384                                 ahc_qinfifo_requeue(ahc, prev_scb, scb);
5385                                 prev_scb = scb;
5386                                 break;
5387                         }
5388                 } else {
5389                         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5390                         prev_scb = scb;
5391                 }
5392                 qinpos++;
5393         }
5394
5395         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5396                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5397         } else {
5398                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5399         }
5400
5401         if (action != SEARCH_COUNT
5402          && (found != 0)
5403          && (qinstart != ahc->qinfifonext)) {
5404                 /*
5405                  * The sequencer may be in the process of dmaing
5406                  * down the SCB at the beginning of the queue.
5407                  * This could be problematic if either the first,
5408                  * or the second SCB is removed from the queue
5409                  * (the first SCB includes a pointer to the "next"
5410                  * SCB to dma). If we have removed any entries, swap
5411                  * the first element in the queue with the next HSCB
5412                  * so the sequencer will notice that NEXT_QUEUED_SCB
5413                  * has changed during its dma attempt and will retry
5414                  * the DMA.
5415                  */
5416                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinstart]);
5417
5418                 if (scb == NULL) {
5419                         printf("found = %d, qinstart = %d, qinfifionext = %d\n",
5420                                 found, qinstart, ahc->qinfifonext);
5421                         panic("First/Second Qinfifo fixup\n");
5422                 }
5423                 /*
5424                  * ahc_swap_with_next_hscb forces our next pointer to
5425                  * point to the reserved SCB for future commands.  Save
5426                  * and restore our original next pointer to maintain
5427                  * queue integrity.
5428                  */
5429                 next = scb->hscb->next;
5430                 ahc->scb_data->scbindex[scb->hscb->tag] = NULL;
5431                 ahc_swap_with_next_hscb(ahc, scb);
5432                 scb->hscb->next = next;
5433                 ahc->qinfifo[qinstart] = scb->hscb->tag;
5434
5435                 /* Tell the card about the new head of the qinfifo. */
5436                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5437
5438                 /* Fixup the tail "next" pointer. */
5439                 qintail = ahc->qinfifonext - 1;
5440                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qintail]);
5441                 scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5442         }
5443
5444         /*
5445          * Search waiting for selection list.
5446          */
5447         curscbptr = ahc_inb(ahc, SCBPTR);
5448         next = ahc_inb(ahc, WAITING_SCBH);  /* Start at head of list. */
5449         prev = SCB_LIST_NULL;
5450
5451         while (next != SCB_LIST_NULL) {
5452                 uint8_t scb_index;
5453
5454                 ahc_outb(ahc, SCBPTR, next);
5455                 scb_index = ahc_inb(ahc, SCB_TAG);
5456                 if (scb_index >= ahc->scb_data->numscbs) {
5457                         printf("Waiting List inconsistency. "
5458                                "SCB index == %d, yet numscbs == %d.",
5459                                scb_index, ahc->scb_data->numscbs);
5460                         ahc_dump_card_state(ahc);
5461                         panic("for safety");
5462                 }
5463                 scb = ahc_lookup_scb(ahc, scb_index);
5464                 if (scb == NULL) {
5465                         printf("scb_index = %d, next = %d\n",
5466                                 scb_index, next);
5467                         panic("Waiting List traversal\n");
5468                 }
5469                 if (ahc_match_scb(ahc, scb, target, channel,
5470                                   lun, SCB_LIST_NULL, role)) {
5471                         /*
5472                          * We found an scb that needs to be acted on.
5473                          */
5474                         found++;
5475                         switch (action) {
5476                         case SEARCH_COMPLETE:
5477                         {
5478                                 cam_status ostat;
5479                                 cam_status cstat;
5480
5481                                 ostat = ahc_get_transaction_status(scb);
5482                                 if (ostat == CAM_REQ_INPROG)
5483                                         ahc_set_transaction_status(scb,
5484                                                                    status);
5485                                 cstat = ahc_get_transaction_status(scb);
5486                                 if (cstat != CAM_REQ_CMP)
5487                                         ahc_freeze_scb(scb);
5488                                 if ((scb->flags & SCB_ACTIVE) == 0)
5489                                         printf("Inactive SCB in Waiting List\n");
5490                                 ahc_done(ahc, scb);
5491                                 /* FALLTHROUGH */
5492                         }
5493                         case SEARCH_REMOVE:
5494                                 next = ahc_rem_wscb(ahc, next, prev);
5495                                 break;
5496                         case SEARCH_COUNT:
5497                                 prev = next;
5498                                 next = ahc_inb(ahc, SCB_NEXT);
5499                                 break;
5500                         }
5501                 } else {
5502
5503                         prev = next;
5504                         next = ahc_inb(ahc, SCB_NEXT);
5505                 }
5506         }
5507         ahc_outb(ahc, SCBPTR, curscbptr);
5508
5509         found += ahc_search_untagged_queues(ahc, /*ahc_io_ctx_t*/NULL, target,
5510                                             channel, lun, status, action);
5511
5512         if (action == SEARCH_COMPLETE)
5513                 ahc_release_untagged_queues(ahc);
5514         return (found);
5515 }
5516
5517 int
5518 ahc_search_untagged_queues(struct ahc_softc *ahc, ahc_io_ctx_t ctx,
5519                            int target, char channel, int lun, uint32_t status,
5520                            ahc_search_action action)
5521 {
5522         struct  scb *scb;
5523         int     maxtarget;
5524         int     found;
5525         int     i;
5526
5527         if (action == SEARCH_COMPLETE) {
5528                 /*
5529                  * Don't attempt to run any queued untagged transactions
5530                  * until we are done with the abort process.
5531                  */
5532                 ahc_freeze_untagged_queues(ahc);
5533         }
5534
5535         found = 0;
5536         i = 0;
5537         if ((ahc->flags & AHC_SCB_BTT) == 0) {
5538
5539                 maxtarget = 16;
5540                 if (target != CAM_TARGET_WILDCARD) {
5541
5542                         i = target;
5543                         if (channel == 'B')
5544                                 i += 8;
5545                         maxtarget = i + 1;
5546                 }
5547         } else {
5548                 maxtarget = 0;
5549         }
5550
5551         for (; i < maxtarget; i++) {
5552                 struct scb_tailq *untagged_q;
5553                 struct scb *next_scb;
5554
5555                 untagged_q = &(ahc->untagged_queues[i]);
5556                 next_scb = TAILQ_FIRST(untagged_q);
5557                 while (next_scb != NULL) {
5558
5559                         scb = next_scb;
5560                         next_scb = TAILQ_NEXT(scb, links.tqe);
5561
5562                         /*
5563                          * The head of the list may be the currently
5564                          * active untagged command for a device.
5565                          * We're only searching for commands that
5566                          * have not been started.  A transaction
5567                          * marked active but still in the qinfifo
5568                          * is removed by the qinfifo scanning code
5569                          * above.
5570                          */
5571                         if ((scb->flags & SCB_ACTIVE) != 0)
5572                                 continue;
5573
5574                         if (ahc_match_scb(ahc, scb, target, channel, lun,
5575                                           SCB_LIST_NULL, ROLE_INITIATOR) == 0
5576                          || (ctx != NULL && ctx != scb->io_ctx))
5577                                 continue;
5578
5579                         /*
5580                          * We found an scb that needs to be acted on.
5581                          */
5582                         found++;
5583                         switch (action) {
5584                         case SEARCH_COMPLETE:
5585                         {
5586                                 cam_status ostat;
5587                                 cam_status cstat;
5588
5589                                 ostat = ahc_get_transaction_status(scb);
5590                                 if (ostat == CAM_REQ_INPROG)
5591                                         ahc_set_transaction_status(scb, status);
5592                                 cstat = ahc_get_transaction_status(scb);
5593                                 if (cstat != CAM_REQ_CMP)
5594                                         ahc_freeze_scb(scb);
5595                                 if ((scb->flags & SCB_ACTIVE) == 0)
5596                                         printf("Inactive SCB in untaggedQ\n");
5597                                 ahc_done(ahc, scb);
5598                                 break;
5599                         }
5600                         case SEARCH_REMOVE:
5601                                 scb->flags &= ~SCB_UNTAGGEDQ;
5602                                 TAILQ_REMOVE(untagged_q, scb, links.tqe);
5603                                 break;
5604                         case SEARCH_COUNT:
5605                                 break;
5606                         }
5607                 }
5608         }
5609
5610         if (action == SEARCH_COMPLETE)
5611                 ahc_release_untagged_queues(ahc);
5612         return (found);
5613 }
5614
5615 int
5616 ahc_search_disc_list(struct ahc_softc *ahc, int target, char channel,
5617                      int lun, u_int tag, int stop_on_first, int remove,
5618                      int save_state)
5619 {
5620         struct  scb *scbp;
5621         u_int   next;
5622         u_int   prev;
5623         u_int   count;
5624         u_int   active_scb;
5625
5626         count = 0;
5627         next = ahc_inb(ahc, DISCONNECTED_SCBH);
5628         prev = SCB_LIST_NULL;
5629
5630         if (save_state) {
5631                 /* restore this when we're done */
5632                 active_scb = ahc_inb(ahc, SCBPTR);
5633         } else
5634                 /* Silence compiler */
5635                 active_scb = SCB_LIST_NULL;
5636
5637         while (next != SCB_LIST_NULL) {
5638                 u_int scb_index;
5639
5640                 ahc_outb(ahc, SCBPTR, next);
5641                 scb_index = ahc_inb(ahc, SCB_TAG);
5642                 if (scb_index >= ahc->scb_data->numscbs) {
5643                         printf("Disconnected List inconsistency. "
5644                                "SCB index == %d, yet numscbs == %d.",
5645                                scb_index, ahc->scb_data->numscbs);
5646                         ahc_dump_card_state(ahc);
5647                         panic("for safety");
5648                 }
5649
5650                 if (next == prev) {
5651                         panic("Disconnected List Loop. "
5652                               "cur SCBPTR == %x, prev SCBPTR == %x.",
5653                               next, prev);
5654                 }
5655                 scbp = ahc_lookup_scb(ahc, scb_index);
5656                 if (ahc_match_scb(ahc, scbp, target, channel, lun,
5657                                   tag, ROLE_INITIATOR)) {
5658                         count++;
5659                         if (remove) {
5660                                 next =
5661                                     ahc_rem_scb_from_disc_list(ahc, prev, next);
5662                         } else {
5663                                 prev = next;
5664                                 next = ahc_inb(ahc, SCB_NEXT);
5665                         }
5666                         if (stop_on_first)
5667                                 break;
5668                 } else {
5669                         prev = next;
5670                         next = ahc_inb(ahc, SCB_NEXT);
5671                 }
5672         }
5673         if (save_state)
5674                 ahc_outb(ahc, SCBPTR, active_scb);
5675         return (count);
5676 }
5677
5678 /*
5679  * Remove an SCB from the on chip list of disconnected transactions.
5680  * This is empty/unused if we are not performing SCB paging.
5681  */
5682 static u_int
5683 ahc_rem_scb_from_disc_list(struct ahc_softc *ahc, u_int prev, u_int scbptr)
5684 {
5685         u_int next;
5686
5687         ahc_outb(ahc, SCBPTR, scbptr);
5688         next = ahc_inb(ahc, SCB_NEXT);
5689
5690         ahc_outb(ahc, SCB_CONTROL, 0);
5691
5692         ahc_add_curscb_to_free_list(ahc);
5693
5694         if (prev != SCB_LIST_NULL) {
5695                 ahc_outb(ahc, SCBPTR, prev);
5696                 ahc_outb(ahc, SCB_NEXT, next);
5697         } else
5698                 ahc_outb(ahc, DISCONNECTED_SCBH, next);
5699
5700         return (next);
5701 }
5702
5703 /*
5704  * Add the SCB as selected by SCBPTR onto the on chip list of
5705  * free hardware SCBs.  This list is empty/unused if we are not
5706  * performing SCB paging.
5707  */
5708 static void
5709 ahc_add_curscb_to_free_list(struct ahc_softc *ahc)
5710 {
5711         /*
5712          * Invalidate the tag so that our abort
5713          * routines don't think it's active.
5714          */
5715         ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
5716
5717         if ((ahc->flags & AHC_PAGESCBS) != 0) {
5718                 ahc_outb(ahc, SCB_NEXT, ahc_inb(ahc, FREE_SCBH));
5719                 ahc_outb(ahc, FREE_SCBH, ahc_inb(ahc, SCBPTR));
5720         }
5721 }
5722
5723 /*
5724  * Manipulate the waiting for selection list and return the
5725  * scb that follows the one that we remove.
5726  */
5727 static u_int
5728 ahc_rem_wscb(struct ahc_softc *ahc, u_int scbpos, u_int prev)
5729 {
5730         u_int curscb, next;
5731
5732         /*
5733          * Select the SCB we want to abort and
5734          * pull the next pointer out of it.
5735          */
5736         curscb = ahc_inb(ahc, SCBPTR);
5737         ahc_outb(ahc, SCBPTR, scbpos);
5738         next = ahc_inb(ahc, SCB_NEXT);
5739
5740         /* Clear the necessary fields */
5741         ahc_outb(ahc, SCB_CONTROL, 0);
5742
5743         ahc_add_curscb_to_free_list(ahc);
5744
5745         /* update the waiting list */
5746         if (prev == SCB_LIST_NULL) {
5747                 /* First in the list */
5748                 ahc_outb(ahc, WAITING_SCBH, next);
5749
5750                 /*
5751                  * Ensure we aren't attempting to perform
5752                  * selection for this entry.
5753                  */
5754                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
5755         } else {
5756                 /*
5757                  * Select the scb that pointed to us
5758                  * and update its next pointer.
5759                  */
5760                 ahc_outb(ahc, SCBPTR, prev);
5761                 ahc_outb(ahc, SCB_NEXT, next);
5762         }
5763
5764         /*
5765          * Point us back at the original scb position.
5766          */
5767         ahc_outb(ahc, SCBPTR, curscb);
5768         return next;
5769 }
5770
5771 /******************************** Error Handling ******************************/
5772 /*
5773  * Abort all SCBs that match the given description (target/channel/lun/tag),
5774  * setting their status to the passed in status if the status has not already
5775  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
5776  * is paused before it is called.
5777  */
5778 int
5779 ahc_abort_scbs(struct ahc_softc *ahc, int target, char channel,
5780                int lun, u_int tag, role_t role, uint32_t status)
5781 {
5782         struct  scb *scbp;
5783         struct  scb *scbp_next;
5784         u_int   active_scb;
5785         int     i, j;
5786         int     maxtarget;
5787         int     minlun;
5788         int     maxlun;
5789
5790         int     found;
5791
5792         /*
5793          * Don't attempt to run any queued untagged transactions
5794          * until we are done with the abort process.
5795          */
5796         ahc_freeze_untagged_queues(ahc);
5797
5798         /* restore this when we're done */
5799         active_scb = ahc_inb(ahc, SCBPTR);
5800
5801         found = ahc_search_qinfifo(ahc, target, channel, lun, SCB_LIST_NULL,
5802                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5803
5804         /*
5805          * Clean out the busy target table for any untagged commands.
5806          */
5807         i = 0;
5808         maxtarget = 16;
5809         if (target != CAM_TARGET_WILDCARD) {
5810                 i = target;
5811                 if (channel == 'B')
5812                         i += 8;
5813                 maxtarget = i + 1;
5814         }
5815
5816         if (lun == CAM_LUN_WILDCARD) {
5817
5818                 /*
5819                  * Unless we are using an SCB based
5820                  * busy targets table, there is only
5821                  * one table entry for all luns of
5822                  * a target.
5823                  */
5824                 minlun = 0;
5825                 maxlun = 1;
5826                 if ((ahc->flags & AHC_SCB_BTT) != 0)
5827                         maxlun = AHC_NUM_LUNS;
5828         } else {
5829                 minlun = lun;
5830                 maxlun = lun + 1;
5831         }
5832
5833         if (role != ROLE_TARGET) {
5834                 for (;i < maxtarget; i++) {
5835                         for (j = minlun;j < maxlun; j++) {
5836                                 u_int scbid;
5837                                 u_int tcl;
5838
5839                                 tcl = BUILD_TCL(i << 4, j);
5840                                 scbid = ahc_index_busy_tcl(ahc, tcl);
5841                                 scbp = ahc_lookup_scb(ahc, scbid);
5842                                 if (scbp == NULL
5843                                  || ahc_match_scb(ahc, scbp, target, channel,
5844                                                   lun, tag, role) == 0)
5845                                         continue;
5846                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, j));
5847                         }
5848                 }
5849
5850                 /*
5851                  * Go through the disconnected list and remove any entries we
5852                  * have queued for completion, 0'ing their control byte too.
5853                  * We save the active SCB and restore it ourselves, so there
5854                  * is no reason for this search to restore it too.
5855                  */
5856                 ahc_search_disc_list(ahc, target, channel, lun, tag,
5857                                      /*stop_on_first*/FALSE, /*remove*/TRUE,
5858                                      /*save_state*/FALSE);
5859         }
5860
5861         /*
5862          * Go through the hardware SCB array looking for commands that
5863          * were active but not on any list.  In some cases, these remnants
5864          * might not still have mappings in the scbindex array (e.g. unexpected
5865          * bus free with the same scb queued for an abort).  Don't hold this
5866          * against them.
5867          */
5868         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
5869                 u_int scbid;
5870
5871                 ahc_outb(ahc, SCBPTR, i);
5872                 scbid = ahc_inb(ahc, SCB_TAG);
5873                 scbp = ahc_lookup_scb(ahc, scbid);
5874                 if ((scbp == NULL && scbid != SCB_LIST_NULL)
5875                  || (scbp != NULL
5876                   && ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)))
5877                         ahc_add_curscb_to_free_list(ahc);
5878         }
5879
5880         /*
5881          * Go through the pending CCB list and look for
5882          * commands for this target that are still active.
5883          * These are other tagged commands that were
5884          * disconnected when the reset occurred.
5885          */
5886         scbp_next = LIST_FIRST(&ahc->pending_scbs);
5887         while (scbp_next != NULL) {
5888                 scbp = scbp_next;
5889                 scbp_next = LIST_NEXT(scbp, pending_links);
5890                 if (ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)) {
5891                         cam_status ostat;
5892
5893                         ostat = ahc_get_transaction_status(scbp);
5894                         if (ostat == CAM_REQ_INPROG)
5895                                 ahc_set_transaction_status(scbp, status);
5896                         if (ahc_get_transaction_status(scbp) != CAM_REQ_CMP)
5897                                 ahc_freeze_scb(scbp);
5898                         if ((scbp->flags & SCB_ACTIVE) == 0)
5899                                 printf("Inactive SCB on pending list\n");
5900                         ahc_done(ahc, scbp);
5901                         found++;
5902                 }
5903         }
5904         ahc_outb(ahc, SCBPTR, active_scb);
5905         ahc_platform_abort_scbs(ahc, target, channel, lun, tag, role, status);
5906         ahc_release_untagged_queues(ahc);
5907         return found;
5908 }
5909
5910 static void
5911 ahc_reset_current_bus(struct ahc_softc *ahc)
5912 {
5913         uint8_t scsiseq;
5914
5915         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENSCSIRST);
5916         scsiseq = ahc_inb(ahc, SCSISEQ);
5917         ahc_outb(ahc, SCSISEQ, scsiseq | SCSIRSTO);
5918         ahc_flush_device_writes(ahc);
5919         ahc_delay(AHC_BUSRESET_DELAY);
5920         /* Turn off the bus reset */
5921         ahc_outb(ahc, SCSISEQ, scsiseq & ~SCSIRSTO);
5922
5923         ahc_clear_intstat(ahc);
5924
5925         /* Re-enable reset interrupts */
5926         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) | ENSCSIRST);
5927 }
5928
5929 int
5930 ahc_reset_channel(struct ahc_softc *ahc, char channel, int initiate_reset)
5931 {
5932         struct  ahc_devinfo devinfo;
5933         u_int   initiator, target, max_scsiid;
5934         u_int   sblkctl;
5935         u_int   scsiseq;
5936         u_int   simode1;
5937         int     found;
5938         int     restart_needed;
5939         char    cur_channel;
5940
5941         ahc->pending_device = NULL;
5942
5943         ahc_compile_devinfo(&devinfo,
5944                             CAM_TARGET_WILDCARD,
5945                             CAM_TARGET_WILDCARD,
5946                             CAM_LUN_WILDCARD,
5947                             channel, ROLE_UNKNOWN);
5948         ahc_pause(ahc);
5949
5950         /* Make sure the sequencer is in a safe location. */
5951         ahc_clear_critical_section(ahc);
5952
5953         /*
5954          * Run our command complete fifos to ensure that we perform
5955          * completion processing on any commands that 'completed'
5956          * before the reset occurred.
5957          */
5958         ahc_run_qoutfifo(ahc);
5959 #ifdef AHC_TARGET_MODE
5960         /*
5961          * XXX - In Twin mode, the tqinfifo may have commands
5962          *       for an unaffected channel in it.  However, if
5963          *       we have run out of ATIO resources to drain that
5964          *       queue, we may not get them all out here.  Further,
5965          *       the blocked transactions for the reset channel
5966          *       should just be killed off, irrespecitve of whether
5967          *       we are blocked on ATIO resources.  Write a routine
5968          *       to compact the tqinfifo appropriately.
5969          */
5970         if ((ahc->flags & AHC_TARGETROLE) != 0) {
5971                 ahc_run_tqinfifo(ahc, /*paused*/TRUE);
5972         }
5973 #endif
5974
5975         /*
5976          * Reset the bus if we are initiating this reset
5977          */
5978         sblkctl = ahc_inb(ahc, SBLKCTL);
5979         cur_channel = 'A';
5980         if ((ahc->features & AHC_TWIN) != 0
5981          && ((sblkctl & SELBUSB) != 0))
5982             cur_channel = 'B';
5983         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
5984         if (cur_channel != channel) {
5985                 /* Case 1: Command for another bus is active
5986                  * Stealthily reset the other bus without
5987                  * upsetting the current bus.
5988                  */
5989                 ahc_outb(ahc, SBLKCTL, sblkctl ^ SELBUSB);
5990                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
5991 #ifdef AHC_TARGET_MODE
5992                 /*
5993                  * Bus resets clear ENSELI, so we cannot
5994                  * defer re-enabling bus reset interrupts
5995                  * if we are in target mode.
5996                  */
5997                 if ((ahc->flags & AHC_TARGETROLE) != 0)
5998                         simode1 |= ENSCSIRST;
5999 #endif
6000                 ahc_outb(ahc, SIMODE1, simode1);
6001                 if (initiate_reset)
6002                         ahc_reset_current_bus(ahc);
6003                 ahc_clear_intstat(ahc);
6004                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6005                 ahc_outb(ahc, SBLKCTL, sblkctl);
6006                 restart_needed = FALSE;
6007         } else {
6008                 /* Case 2: A command from this bus is active or we're idle */
6009                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6010 #ifdef AHC_TARGET_MODE
6011                 /*
6012                  * Bus resets clear ENSELI, so we cannot
6013                  * defer re-enabling bus reset interrupts
6014                  * if we are in target mode.
6015                  */
6016                 if ((ahc->flags & AHC_TARGETROLE) != 0)
6017                         simode1 |= ENSCSIRST;
6018 #endif
6019                 ahc_outb(ahc, SIMODE1, simode1);
6020                 if (initiate_reset)
6021                         ahc_reset_current_bus(ahc);
6022                 ahc_clear_intstat(ahc);
6023                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6024                 restart_needed = TRUE;
6025         }
6026
6027         /*
6028          * Clean up all the state information for the
6029          * pending transactions on this bus.
6030          */
6031         found = ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, channel,
6032                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
6033                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6034
6035         max_scsiid = (ahc->features & AHC_WIDE) ? 15 : 7;
6036
6037 #ifdef AHC_TARGET_MODE
6038         /*
6039          * Send an immediate notify ccb to all target more peripheral
6040          * drivers affected by this action.
6041          */
6042         for (target = 0; target <= max_scsiid; target++) {
6043                 struct ahc_tmode_tstate* tstate;
6044                 u_int lun;
6045
6046                 tstate = ahc->enabled_targets[target];
6047                 if (tstate == NULL)
6048                         continue;
6049                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
6050                         struct ahc_tmode_lstate* lstate;
6051
6052                         lstate = tstate->enabled_luns[lun];
6053                         if (lstate == NULL)
6054                                 continue;
6055
6056                         ahc_queue_lstate_event(ahc, lstate, CAM_TARGET_WILDCARD,
6057                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
6058                         ahc_send_lstate_events(ahc, lstate);
6059                 }
6060         }
6061 #endif
6062         /* Notify the XPT that a bus reset occurred */
6063         ahc_send_async(ahc, devinfo.channel, CAM_TARGET_WILDCARD,
6064                        CAM_LUN_WILDCARD, AC_BUS_RESET, NULL);
6065
6066         /*
6067          * Revert to async/narrow transfers until we renegotiate.
6068          */
6069         for (target = 0; target <= max_scsiid; target++) {
6070
6071                 if (ahc->enabled_targets[target] == NULL)
6072                         continue;
6073                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
6074                         struct ahc_devinfo devinfo;
6075
6076                         ahc_compile_devinfo(&devinfo, target, initiator,
6077                                             CAM_LUN_WILDCARD,
6078                                             channel, ROLE_UNKNOWN);
6079                         ahc_set_width(ahc, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6080                                       AHC_TRANS_CUR, /*paused*/TRUE);
6081                         ahc_set_syncrate(ahc, &devinfo, /*syncrate*/NULL,
6082                                          /*period*/0, /*offset*/0,
6083                                          /*ppr_options*/0, AHC_TRANS_CUR,
6084                                          /*paused*/TRUE);
6085                 }
6086         }
6087
6088         if (restart_needed)
6089                 ahc_restart(ahc);
6090         else
6091                 ahc_unpause(ahc);
6092         return found;
6093 }
6094
6095
6096 /***************************** Residual Processing ****************************/
6097 /*
6098  * Calculate the residual for a just completed SCB.
6099  */
6100 void
6101 ahc_calc_residual(struct ahc_softc *ahc, struct scb *scb)
6102 {
6103         struct hardware_scb *hscb;
6104         struct status_pkt *spkt;
6105         uint32_t sgptr;
6106         uint32_t resid_sgptr;
6107         uint32_t resid;
6108
6109         /*
6110          * 5 cases.
6111          * 1) No residual.
6112          *    SG_RESID_VALID clear in sgptr.
6113          * 2) Transferless command
6114          * 3) Never performed any transfers.
6115          *    sgptr has SG_FULL_RESID set.
6116          * 4) No residual but target did not
6117          *    save data pointers after the
6118          *    last transfer, so sgptr was
6119          *    never updated.
6120          * 5) We have a partial residual.
6121          *    Use residual_sgptr to determine
6122          *    where we are.
6123          */
6124
6125         hscb = scb->hscb;
6126         sgptr = ahc_le32toh(hscb->sgptr);
6127         if ((sgptr & SG_RESID_VALID) == 0)
6128                 /* Case 1 */
6129                 return;
6130         sgptr &= ~SG_RESID_VALID;
6131
6132         if ((sgptr & SG_LIST_NULL) != 0)
6133                 /* Case 2 */
6134                 return;
6135
6136         spkt = &hscb->shared_data.status;
6137         resid_sgptr = ahc_le32toh(spkt->residual_sg_ptr);
6138         if ((sgptr & SG_FULL_RESID) != 0) {
6139                 /* Case 3 */
6140                 resid = ahc_get_transfer_length(scb);
6141         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
6142                 /* Case 4 */
6143                 return;
6144         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
6145                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
6146         } else {
6147                 struct ahc_dma_seg *sg;
6148
6149                 /*
6150                  * Remainder of the SG where the transfer
6151                  * stopped.
6152                  */
6153                 resid = ahc_le32toh(spkt->residual_datacnt) & AHC_SG_LEN_MASK;
6154                 sg = ahc_sg_bus_to_virt(scb, resid_sgptr & SG_PTR_MASK);
6155
6156                 /* The residual sg_ptr always points to the next sg */
6157                 sg--;
6158
6159                 /*
6160                  * Add up the contents of all residual
6161                  * SG segments that are after the SG where
6162                  * the transfer stopped.
6163                  */
6164                 while ((ahc_le32toh(sg->len) & AHC_DMA_LAST_SEG) == 0) {
6165                         sg++;
6166                         resid += ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
6167                 }
6168         }
6169         if ((scb->flags & SCB_SENSE) == 0)
6170                 ahc_set_residual(scb, resid);
6171         else
6172                 ahc_set_sense_residual(scb, resid);
6173
6174 #ifdef AHC_DEBUG
6175         if ((ahc_debug & AHC_SHOW_MISC) != 0) {
6176                 ahc_print_path(ahc, scb);
6177                 printf("Handled %sResidual of %d bytes\n",
6178                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
6179         }
6180 #endif
6181 }
6182
6183 /******************************* Target Mode **********************************/
6184 #ifdef AHC_TARGET_MODE
6185 /*
6186  * Add a target mode event to this lun's queue
6187  */
6188 static void
6189 ahc_queue_lstate_event(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate,
6190                        u_int initiator_id, u_int event_type, u_int event_arg)
6191 {
6192         struct ahc_tmode_event *event;
6193         int pending;
6194
6195         xpt_freeze_devq(lstate->path, /*count*/1);
6196         if (lstate->event_w_idx >= lstate->event_r_idx)
6197                 pending = lstate->event_w_idx - lstate->event_r_idx;
6198         else
6199                 pending = AHC_TMODE_EVENT_BUFFER_SIZE + 1
6200                         - (lstate->event_r_idx - lstate->event_w_idx);
6201
6202         if (event_type == EVENT_TYPE_BUS_RESET
6203          || event_type == MSG_BUS_DEV_RESET) {
6204                 /*
6205                  * Any earlier events are irrelevant, so reset our buffer.
6206                  * This has the effect of allowing us to deal with reset
6207                  * floods (an external device holding down the reset line)
6208                  * without losing the event that is really interesting.
6209                  */
6210                 lstate->event_r_idx = 0;
6211                 lstate->event_w_idx = 0;
6212                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
6213         }
6214
6215         if (pending == AHC_TMODE_EVENT_BUFFER_SIZE) {
6216                 xpt_print_path(lstate->path);
6217                 printf("immediate event %x:%x lost\n",
6218                        lstate->event_buffer[lstate->event_r_idx].event_type,
6219                        lstate->event_buffer[lstate->event_r_idx].event_arg);
6220                 lstate->event_r_idx++;
6221                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6222                         lstate->event_r_idx = 0;
6223                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
6224         }
6225
6226         event = &lstate->event_buffer[lstate->event_w_idx];
6227         event->initiator_id = initiator_id;
6228         event->event_type = event_type;
6229         event->event_arg = event_arg;
6230         lstate->event_w_idx++;
6231         if (lstate->event_w_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6232                 lstate->event_w_idx = 0;
6233 }
6234
6235 /*
6236  * Send any target mode events queued up waiting
6237  * for immediate notify resources.
6238  */
6239 void
6240 ahc_send_lstate_events(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate)
6241 {
6242         struct ccb_hdr *ccbh;
6243         struct ccb_immed_notify *inot;
6244
6245         while (lstate->event_r_idx != lstate->event_w_idx
6246             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
6247                 struct ahc_tmode_event *event;
6248
6249                 event = &lstate->event_buffer[lstate->event_r_idx];
6250                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
6251                 inot = (struct ccb_immed_notify *)ccbh;
6252                 switch (event->event_type) {
6253                 case EVENT_TYPE_BUS_RESET:
6254                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
6255                         break;
6256                 default:
6257                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
6258                         inot->message_args[0] = event->event_type;
6259                         inot->message_args[1] = event->event_arg;
6260                         break;
6261                 }
6262                 inot->initiator_id = event->initiator_id;
6263                 inot->sense_len = 0;
6264                 xpt_done((union ccb *)inot);
6265                 lstate->event_r_idx++;
6266                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6267                         lstate->event_r_idx = 0;
6268         }
6269 }
6270 #endif
6271
6272 /******************** Sequencer Program Patching/Download *********************/
6273
6274 #ifdef AHC_DUMP_SEQ
6275 void
6276 ahc_dumpseq(struct ahc_softc* ahc)
6277 {
6278         int i;
6279
6280         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6281         ahc_outb(ahc, SEQADDR0, 0);
6282         ahc_outb(ahc, SEQADDR1, 0);
6283         for (i = 0; i < ahc->instruction_ram_size; i++) {
6284                 uint8_t ins_bytes[4];
6285
6286                 ahc_insb(ahc, SEQRAM, ins_bytes, 4);
6287                 printf("0x%08x\n", ins_bytes[0] << 24
6288                                  | ins_bytes[1] << 16
6289                                  | ins_bytes[2] << 8
6290                                  | ins_bytes[3]);
6291         }
6292 }
6293 #endif
6294
6295 static int
6296 ahc_loadseq(struct ahc_softc *ahc)
6297 {
6298         struct  cs cs_table[num_critical_sections];
6299         u_int   begin_set[num_critical_sections];
6300         u_int   end_set[num_critical_sections];
6301         struct  patch *cur_patch;
6302         u_int   cs_count;
6303         u_int   cur_cs;
6304         u_int   i;
6305         u_int   skip_addr;
6306         u_int   sg_prefetch_cnt;
6307         int     downloaded;
6308         uint8_t download_consts[7];
6309
6310         /*
6311          * Start out with 0 critical sections
6312          * that apply to this firmware load.
6313          */
6314         cs_count = 0;
6315         cur_cs = 0;
6316         memset(begin_set, 0, sizeof(begin_set));
6317         memset(end_set, 0, sizeof(end_set));
6318
6319         /* Setup downloadable constant table */
6320         download_consts[QOUTFIFO_OFFSET] = 0;
6321         if (ahc->targetcmds != NULL)
6322                 download_consts[QOUTFIFO_OFFSET] += 32;
6323         download_consts[QINFIFO_OFFSET] = download_consts[QOUTFIFO_OFFSET] + 1;
6324         download_consts[CACHESIZE_MASK] = ahc->pci_cachesize - 1;
6325         download_consts[INVERTED_CACHESIZE_MASK] = ~(ahc->pci_cachesize - 1);
6326         sg_prefetch_cnt = ahc->pci_cachesize;
6327         if (sg_prefetch_cnt < (2 * sizeof(struct ahc_dma_seg)))
6328                 sg_prefetch_cnt = 2 * sizeof(struct ahc_dma_seg);
6329         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
6330         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_cnt - 1);
6331         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_cnt - 1);
6332
6333         cur_patch = patches;
6334         downloaded = 0;
6335         skip_addr = 0;
6336         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6337         ahc_outb(ahc, SEQADDR0, 0);
6338         ahc_outb(ahc, SEQADDR1, 0);
6339
6340         for (i = 0; i < sizeof(seqprog)/4; i++) {
6341                 if (ahc_check_patch(ahc, &cur_patch, i, &skip_addr) == 0) {
6342                         /*
6343                          * Don't download this instruction as it
6344                          * is in a patch that was removed.
6345                          */
6346                         continue;
6347                 }
6348
6349                 if (downloaded == ahc->instruction_ram_size) {
6350                         /*
6351                          * We're about to exceed the instruction
6352                          * storage capacity for this chip.  Fail
6353                          * the load.
6354                          */
6355                         printf("\n%s: Program too large for instruction memory "
6356                                "size of %d!\n", ahc_name(ahc),
6357                                ahc->instruction_ram_size);
6358                         return (ENOMEM);
6359                 }
6360
6361                 /*
6362                  * Move through the CS table until we find a CS
6363                  * that might apply to this instruction.
6364                  */
6365                 for (; cur_cs < num_critical_sections; cur_cs++) {
6366                         if (critical_sections[cur_cs].end <= i) {
6367                                 if (begin_set[cs_count] == TRUE
6368                                  && end_set[cs_count] == FALSE) {
6369                                         cs_table[cs_count].end = downloaded;
6370                                         end_set[cs_count] = TRUE;
6371                                         cs_count++;
6372                                 }
6373                                 continue;
6374                         }
6375                         if (critical_sections[cur_cs].begin <= i
6376                          && begin_set[cs_count] == FALSE) {
6377                                 cs_table[cs_count].begin = downloaded;
6378                                 begin_set[cs_count] = TRUE;
6379                         }
6380                         break;
6381                 }
6382                 ahc_download_instr(ahc, i, download_consts);
6383                 downloaded++;
6384         }
6385
6386         ahc->num_critical_sections = cs_count;
6387         if (cs_count != 0) {
6388                 cs_count *= sizeof(struct cs);
6389                 ahc->critical_sections = kmalloc(cs_count, M_DEVBUF, M_INTWAIT);
6390                 memcpy(ahc->critical_sections, cs_table, cs_count);
6391         }
6392         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE);
6393
6394         if (bootverbose) {
6395                 printf(" %d instructions downloaded\n", downloaded);
6396                 printf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
6397                        ahc_name(ahc), ahc->features, ahc->bugs, ahc->flags);
6398         }
6399         return (0);
6400 }
6401
6402 static int
6403 ahc_check_patch(struct ahc_softc *ahc, struct patch **start_patch,
6404                 u_int start_instr, u_int *skip_addr)
6405 {
6406         struct  patch *cur_patch;
6407         struct  patch *last_patch;
6408         u_int   num_patches;
6409
6410         num_patches = sizeof(patches)/sizeof(struct patch);
6411         last_patch = &patches[num_patches];
6412         cur_patch = *start_patch;
6413
6414         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
6415
6416                 if (cur_patch->patch_func(ahc) == 0) {
6417
6418                         /* Start rejecting code */
6419                         *skip_addr = start_instr + cur_patch->skip_instr;
6420                         cur_patch += cur_patch->skip_patch;
6421                 } else {
6422                         /* Accepted this patch.  Advance to the next
6423                          * one and wait for our intruction pointer to
6424                          * hit this point.
6425                          */
6426                         cur_patch++;
6427                 }
6428         }
6429
6430         *start_patch = cur_patch;
6431         if (start_instr < *skip_addr)
6432                 /* Still skipping */
6433                 return (0);
6434
6435         return (1);
6436 }
6437
6438 static void
6439 ahc_download_instr(struct ahc_softc *ahc, u_int instrptr, uint8_t *dconsts)
6440 {
6441         union   ins_formats instr;
6442         struct  ins_format1 *fmt1_ins;
6443         struct  ins_format3 *fmt3_ins;
6444         u_int   opcode;
6445
6446         /*
6447          * The firmware is always compiled into a little endian format.
6448          */
6449         instr.integer = ahc_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
6450
6451         fmt1_ins = &instr.format1;
6452         fmt3_ins = NULL;
6453
6454         /* Pull the opcode */
6455         opcode = instr.format1.opcode;
6456         switch (opcode) {
6457         case AIC_OP_JMP:
6458         case AIC_OP_JC:
6459         case AIC_OP_JNC:
6460         case AIC_OP_CALL:
6461         case AIC_OP_JNE:
6462         case AIC_OP_JNZ:
6463         case AIC_OP_JE:
6464         case AIC_OP_JZ:
6465         {
6466                 struct patch *cur_patch;
6467                 int address_offset;
6468                 u_int address;
6469                 u_int skip_addr;
6470                 u_int i;
6471
6472                 fmt3_ins = &instr.format3;
6473                 address_offset = 0;
6474                 address = fmt3_ins->address;
6475                 cur_patch = patches;
6476                 skip_addr = 0;
6477
6478                 for (i = 0; i < address;) {
6479
6480                         ahc_check_patch(ahc, &cur_patch, i, &skip_addr);
6481
6482                         if (skip_addr > i) {
6483                                 int end_addr;
6484
6485                                 end_addr = MIN(address, skip_addr);
6486                                 address_offset += end_addr - i;
6487                                 i = skip_addr;
6488                         } else {
6489                                 i++;
6490                         }
6491                 }
6492                 address -= address_offset;
6493                 fmt3_ins->address = address;
6494                 /* FALLTHROUGH */
6495         }
6496         case AIC_OP_OR:
6497         case AIC_OP_AND:
6498         case AIC_OP_XOR:
6499         case AIC_OP_ADD:
6500         case AIC_OP_ADC:
6501         case AIC_OP_BMOV:
6502                 if (fmt1_ins->parity != 0) {
6503                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
6504                 }
6505                 fmt1_ins->parity = 0;
6506                 if ((ahc->features & AHC_CMD_CHAN) == 0
6507                  && opcode == AIC_OP_BMOV) {
6508                         /*
6509                          * Block move was added at the same time
6510                          * as the command channel.  Verify that
6511                          * this is only a move of a single element
6512                          * and convert the BMOV to a MOV
6513                          * (AND with an immediate of FF).
6514                          */
6515                         if (fmt1_ins->immediate != 1)
6516                                 panic("%s: BMOV not supported\n",
6517                                       ahc_name(ahc));
6518                         fmt1_ins->opcode = AIC_OP_AND;
6519                         fmt1_ins->immediate = 0xff;
6520                 }
6521                 /* FALLTHROUGH */
6522         case AIC_OP_ROL:
6523                 if ((ahc->features & AHC_ULTRA2) != 0) {
6524                         int i, count;
6525
6526                         /* Calculate odd parity for the instruction */
6527                         for (i = 0, count = 0; i < 31; i++) {
6528                                 uint32_t mask;
6529
6530                                 mask = 0x01 << i;
6531                                 if ((instr.integer & mask) != 0)
6532                                         count++;
6533                         }
6534                         if ((count & 0x01) == 0)
6535                                 instr.format1.parity = 1;
6536                 } else {
6537                         /* Compress the instruction for older sequencers */
6538                         if (fmt3_ins != NULL) {
6539                                 instr.integer =
6540                                         fmt3_ins->immediate
6541                                       | (fmt3_ins->source << 8)
6542                                       | (fmt3_ins->address << 16)
6543                                       | (fmt3_ins->opcode << 25);
6544                         } else {
6545                                 instr.integer =
6546                                         fmt1_ins->immediate
6547                                       | (fmt1_ins->source << 8)
6548                                       | (fmt1_ins->destination << 16)
6549                                       | (fmt1_ins->ret << 24)
6550                                       | (fmt1_ins->opcode << 25);
6551                         }
6552                 }
6553                 /* The sequencer is a little endian cpu */
6554                 instr.integer = ahc_htole32(instr.integer);
6555                 ahc_outsb(ahc, SEQRAM, instr.bytes, 4);
6556                 break;
6557         default:
6558                 panic("Unknown opcode encountered in seq program");
6559                 break;
6560         }
6561 }
6562
6563 int
6564 ahc_print_register(ahc_reg_parse_entry_t *table, u_int num_entries,
6565                    const char *name, u_int address, u_int value,
6566                    u_int *cur_column, u_int wrap_point)
6567 {
6568         int     printed;
6569         u_int   printed_mask;
6570
6571         if (cur_column != NULL && *cur_column >= wrap_point) {
6572                 printf("\n");
6573                 *cur_column = 0;
6574         }
6575         printed = printf("%s[0x%x]", name, value);
6576         if (table == NULL) {
6577                 printed += printf(" ");
6578                 *cur_column += printed;
6579                 return (printed);
6580         }
6581         printed_mask = 0;
6582         while (printed_mask != 0xFF) {
6583                 int entry;
6584
6585                 for (entry = 0; entry < num_entries; entry++) {
6586                         if (((value & table[entry].mask)
6587                           != table[entry].value)
6588                          || ((printed_mask & table[entry].mask)
6589                           == table[entry].mask))
6590                                 continue;
6591
6592                         printed += printf("%s%s",
6593                                           printed_mask == 0 ? ":(" : "|",
6594                                           table[entry].name);
6595                         printed_mask |= table[entry].mask;
6596
6597                         break;
6598                 }
6599                 if (entry >= num_entries)
6600                         break;
6601         }
6602         if (printed_mask != 0)
6603                 printed += printf(") ");
6604         else
6605                 printed += printf(" ");
6606         if (cur_column != NULL)
6607                 *cur_column += printed;
6608         return (printed);
6609 }
6610
6611 void
6612 ahc_dump_card_state(struct ahc_softc *ahc)
6613 {
6614         struct  scb *scb;
6615         struct  scb_tailq *untagged_q;
6616         u_int   cur_col;
6617         int     paused;
6618         int     target;
6619         int     maxtarget;
6620         int     i;
6621         uint8_t last_phase;
6622         uint8_t qinpos;
6623         uint8_t qintail;
6624         uint8_t qoutpos;
6625         uint8_t scb_index;
6626         uint8_t saved_scbptr;
6627
6628         if (ahc_is_paused(ahc)) {
6629                 paused = 1;
6630         } else {
6631                 paused = 0;
6632                 ahc_pause(ahc);
6633         }
6634
6635         saved_scbptr = ahc_inb(ahc, SCBPTR);
6636         last_phase = ahc_inb(ahc, LASTPHASE);
6637         printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
6638                "%s: Dumping Card State %s, at SEQADDR 0x%x\n",
6639                ahc_name(ahc), ahc_lookup_phase_entry(last_phase)->phasemsg,
6640                ahc_inb(ahc, SEQADDR0) | (ahc_inb(ahc, SEQADDR1) << 8));
6641         if (paused)
6642                 printf("Card was paused\n");
6643         printf("ACCUM = 0x%x, SINDEX = 0x%x, DINDEX = 0x%x, ARG_2 = 0x%x\n",
6644                ahc_inb(ahc, ACCUM), ahc_inb(ahc, SINDEX), ahc_inb(ahc, DINDEX),
6645                ahc_inb(ahc, ARG_2));
6646         printf("HCNT = 0x%x SCBPTR = 0x%x\n", ahc_inb(ahc, HCNT),
6647                ahc_inb(ahc, SCBPTR));
6648         cur_col = 0;
6649         if ((ahc->features & AHC_DT) != 0)
6650                 ahc_scsiphase_print(ahc_inb(ahc, SCSIPHASE), &cur_col, 50);
6651         ahc_scsisigi_print(ahc_inb(ahc, SCSISIGI), &cur_col, 50);
6652         ahc_error_print(ahc_inb(ahc, ERROR), &cur_col, 50);
6653         ahc_scsibusl_print(ahc_inb(ahc, SCSIBUSL), &cur_col, 50);
6654         ahc_lastphase_print(ahc_inb(ahc, LASTPHASE), &cur_col, 50);
6655         ahc_scsiseq_print(ahc_inb(ahc, SCSISEQ), &cur_col, 50);
6656         ahc_sblkctl_print(ahc_inb(ahc, SBLKCTL), &cur_col, 50);
6657         ahc_scsirate_print(ahc_inb(ahc, SCSIRATE), &cur_col, 50);
6658         ahc_seqctl_print(ahc_inb(ahc, SEQCTL), &cur_col, 50);
6659         ahc_seq_flags_print(ahc_inb(ahc, SEQ_FLAGS), &cur_col, 50);
6660         ahc_sstat0_print(ahc_inb(ahc, SSTAT0), &cur_col, 50);
6661         ahc_sstat1_print(ahc_inb(ahc, SSTAT1), &cur_col, 50);
6662         ahc_sstat2_print(ahc_inb(ahc, SSTAT2), &cur_col, 50);
6663         ahc_sstat3_print(ahc_inb(ahc, SSTAT3), &cur_col, 50);
6664         ahc_simode0_print(ahc_inb(ahc, SIMODE0), &cur_col, 50);
6665         ahc_simode1_print(ahc_inb(ahc, SIMODE1), &cur_col, 50);
6666         ahc_sxfrctl0_print(ahc_inb(ahc, SXFRCTL0), &cur_col, 50);
6667         ahc_dfcntrl_print(ahc_inb(ahc, DFCNTRL), &cur_col, 50);
6668         ahc_dfstatus_print(ahc_inb(ahc, DFSTATUS), &cur_col, 50);
6669         if (cur_col != 0)
6670                 printf("\n");
6671         printf("STACK:");
6672         for (i = 0; i < STACK_SIZE; i++)
6673                printf(" 0x%x", ahc_inb(ahc, STACK)|(ahc_inb(ahc, STACK) << 8));
6674         printf("\nSCB count = %d\n", ahc->scb_data->numscbs);
6675         printf("Kernel NEXTQSCB = %d\n", ahc->next_queued_scb->hscb->tag);
6676         printf("Card NEXTQSCB = %d\n", ahc_inb(ahc, NEXT_QUEUED_SCB));
6677         /* QINFIFO */
6678         printf("QINFIFO entries: ");
6679         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
6680                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
6681                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
6682         } else
6683                 qinpos = ahc_inb(ahc, QINPOS);
6684         qintail = ahc->qinfifonext;
6685         while (qinpos != qintail) {
6686                 printf("%d ", ahc->qinfifo[qinpos]);
6687                 qinpos++;
6688         }
6689         printf("\n");
6690
6691         printf("Waiting Queue entries: ");
6692         scb_index = ahc_inb(ahc, WAITING_SCBH);
6693         i = 0;
6694         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6695                 ahc_outb(ahc, SCBPTR, scb_index);
6696                 printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
6697                 scb_index = ahc_inb(ahc, SCB_NEXT);
6698         }
6699         printf("\n");
6700
6701         printf("Disconnected Queue entries: ");
6702         scb_index = ahc_inb(ahc, DISCONNECTED_SCBH);
6703         i = 0;
6704         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6705                 ahc_outb(ahc, SCBPTR, scb_index);
6706                 printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
6707                 scb_index = ahc_inb(ahc, SCB_NEXT);
6708         }
6709         printf("\n");
6710
6711         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
6712         printf("QOUTFIFO entries: ");
6713         qoutpos = ahc->qoutfifonext;
6714         i = 0;
6715         while (ahc->qoutfifo[qoutpos] != SCB_LIST_NULL && i++ < 256) {
6716                 printf("%d ", ahc->qoutfifo[qoutpos]);
6717                 qoutpos++;
6718         }
6719         printf("\n");
6720
6721         printf("Sequencer Free SCB List: ");
6722         scb_index = ahc_inb(ahc, FREE_SCBH);
6723         i = 0;
6724         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6725                 ahc_outb(ahc, SCBPTR, scb_index);
6726                 printf("%d ", scb_index);
6727                 scb_index = ahc_inb(ahc, SCB_NEXT);
6728         }
6729         printf("\n");
6730
6731         printf("Sequencer SCB Info: ");
6732         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
6733                 ahc_outb(ahc, SCBPTR, i);
6734                 cur_col = printf("\n%3d ", i);
6735
6736                 ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL), &cur_col, 60);
6737                 ahc_scb_scsiid_print(ahc_inb(ahc, SCB_SCSIID), &cur_col, 60);
6738                 ahc_scb_lun_print(ahc_inb(ahc, SCB_LUN), &cur_col, 60);
6739                 ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
6740         }
6741         printf("\n");
6742
6743         printf("Pending list: ");
6744         i = 0;
6745         LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
6746                 if (i++ > 256)
6747                         break;
6748                 cur_col = printf("\n%3d ", scb->hscb->tag);
6749                 ahc_scb_control_print(scb->hscb->control, &cur_col, 60);
6750                 ahc_scb_scsiid_print(scb->hscb->scsiid, &cur_col, 60);
6751                 ahc_scb_lun_print(scb->hscb->lun, &cur_col, 60);
6752                 if ((ahc->flags & AHC_PAGESCBS) == 0) {
6753                         ahc_outb(ahc, SCBPTR, scb->hscb->tag);
6754                         printf("(");
6755                         ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL),
6756                                               &cur_col, 60);
6757                         ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
6758                         printf(")");
6759                 }
6760         }
6761         printf("\n");
6762
6763         printf("Kernel Free SCB list: ");
6764         i = 0;
6765         SLIST_FOREACH(scb, &ahc->scb_data->free_scbs, links.sle) {
6766                 if (i++ > 256)
6767                         break;
6768                 printf("%d ", scb->hscb->tag);
6769         }
6770         printf("\n");
6771
6772         maxtarget = (ahc->features & (AHC_WIDE|AHC_TWIN)) ? 15 : 7;
6773         for (target = 0; target <= maxtarget; target++) {
6774                 untagged_q = &ahc->untagged_queues[target];
6775                 if (TAILQ_FIRST(untagged_q) == NULL)
6776                         continue;
6777                 printf("Untagged Q(%d): ", target);
6778                 i = 0;
6779                 TAILQ_FOREACH(scb, untagged_q, links.tqe) {
6780                         if (i++ > 256)
6781                                 break;
6782                         printf("%d ", scb->hscb->tag);
6783                 }
6784                 printf("\n");
6785         }
6786
6787         ahc_platform_dump_card_state(ahc);
6788         printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
6789         ahc_outb(ahc, SCBPTR, saved_scbptr);
6790         if (paused == 0)
6791                 ahc_unpause(ahc);
6792 }
6793
6794 /************************* Target Mode ****************************************/
6795 #ifdef AHC_TARGET_MODE
6796 cam_status
6797 ahc_find_tmode_devs(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb,
6798                     struct ahc_tmode_tstate **tstate,
6799                     struct ahc_tmode_lstate **lstate,
6800                     int notfound_failure)
6801 {
6802
6803         if ((ahc->features & AHC_TARGETMODE) == 0)
6804                 return (CAM_REQ_INVALID);
6805
6806         /*
6807          * Handle the 'black hole' device that sucks up
6808          * requests to unattached luns on enabled targets.
6809          */
6810         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
6811          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
6812                 *tstate = NULL;
6813                 *lstate = ahc->black_hole;
6814         } else {
6815                 u_int max_id;
6816
6817                 max_id = (ahc->features & AHC_WIDE) ? 15 : 7;
6818                 if (ccb->ccb_h.target_id > max_id)
6819                         return (CAM_TID_INVALID);
6820
6821                 if (ccb->ccb_h.target_lun >= AHC_NUM_LUNS)
6822                         return (CAM_LUN_INVALID);
6823
6824                 *tstate = ahc->enabled_targets[ccb->ccb_h.target_id];
6825                 *lstate = NULL;
6826                 if (*tstate != NULL)
6827                         *lstate =
6828                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
6829         }
6830
6831         if (notfound_failure != 0 && *lstate == NULL)
6832                 return (CAM_PATH_INVALID);
6833
6834         return (CAM_REQ_CMP);
6835 }
6836
6837 void
6838 ahc_handle_en_lun(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb)
6839 {
6840         struct     ahc_tmode_tstate *tstate;
6841         struct     ahc_tmode_lstate *lstate;
6842         struct     ccb_en_lun *cel;
6843         cam_status status;
6844         u_long     s;
6845         u_int      target;
6846         u_int      lun;
6847         u_int      target_mask;
6848         u_int      our_id;
6849         int        error;
6850         char       channel;
6851
6852         status = ahc_find_tmode_devs(ahc, sim, ccb, &tstate, &lstate,
6853                                      /*notfound_failure*/FALSE);
6854
6855         if (status != CAM_REQ_CMP) {
6856                 ccb->ccb_h.status = status;
6857                 return;
6858         }
6859
6860         if (cam_sim_bus(sim) == 0)
6861                 our_id = ahc->our_id;
6862         else
6863                 our_id = ahc->our_id_b;
6864
6865         if (ccb->ccb_h.target_id != our_id) {
6866                 /*
6867                  * our_id represents our initiator ID, or
6868                  * the ID of the first target to have an
6869                  * enabled lun in target mode.  There are
6870                  * two cases that may preclude enabling a
6871                  * target id other than our_id.
6872                  *
6873                  *   o our_id is for an active initiator role.
6874                  *     Since the hardware does not support
6875                  *     reselections to the initiator role at
6876                  *     anything other than our_id, and our_id
6877                  *     is used by the hardware to indicate the
6878                  *     ID to use for both select-out and
6879                  *     reselect-out operations, the only target
6880                  *     ID we can support in this mode is our_id.
6881                  *
6882                  *   o The MULTARGID feature is not available and
6883                  *     a previous target mode ID has been enabled.
6884                  */
6885                 if ((ahc->features & AHC_MULTIROLE) != 0) {
6886
6887                         if ((ahc->features & AHC_MULTI_TID) != 0
6888                          && (ahc->flags & AHC_INITIATORROLE) != 0) {
6889                                 /*
6890                                  * Only allow additional targets if
6891                                  * the initiator role is disabled.
6892                                  * The hardware cannot handle a re-select-in
6893                                  * on the initiator id during a re-select-out
6894                                  * on a different target id.
6895                                  */
6896                                 status = CAM_TID_INVALID;
6897                         } else if ((ahc->flags & AHC_INITIATORROLE) != 0
6898                                 || ahc->enabled_luns > 0) {
6899                                 /*
6900                                  * Only allow our target id to change
6901                                  * if the initiator role is not configured
6902                                  * and there are no enabled luns which
6903                                  * are attached to the currently registered
6904                                  * scsi id.
6905                                  */
6906                                 status = CAM_TID_INVALID;
6907                         }
6908                 } else if ((ahc->features & AHC_MULTI_TID) == 0
6909                         && ahc->enabled_luns > 0) {
6910
6911                         status = CAM_TID_INVALID;
6912                 }
6913         }
6914
6915         if (status != CAM_REQ_CMP) {
6916                 ccb->ccb_h.status = status;
6917                 return;
6918         }
6919
6920         /*
6921          * We now have an id that is valid.
6922          * If we aren't in target mode, switch modes.
6923          */
6924         if ((ahc->flags & AHC_TARGETROLE) == 0
6925          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
6926                 u_long   s;
6927                 ahc_flag saved_flags;
6928
6929                 printf("Configuring Target Mode\n");
6930                 ahc_lock(ahc, &s);
6931                 if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
6932                         ccb->ccb_h.status = CAM_BUSY;
6933                         ahc_unlock(ahc, &s);
6934                         return;
6935                 }
6936                 saved_flags = ahc->flags;
6937                 ahc->flags |= AHC_TARGETROLE;
6938                 if ((ahc->features & AHC_MULTIROLE) == 0)
6939                         ahc->flags &= ~AHC_INITIATORROLE;
6940                 ahc_pause(ahc);
6941                 error = ahc_loadseq(ahc);
6942                 if (error != 0) {
6943                         /*
6944                          * Restore original configuration and notify
6945                          * the caller that we cannot support target mode.
6946                          * Since the adapter started out in this
6947                          * configuration, the firmware load will succeed,
6948                          * so there is no point in checking ahc_loadseq's
6949                          * return value.
6950                          */
6951                         ahc->flags = saved_flags;
6952                         (void)ahc_loadseq(ahc);
6953                         ahc_restart(ahc);
6954                         ahc_unlock(ahc, &s);
6955                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
6956                         return;
6957                 }
6958                 ahc_restart(ahc);
6959                 ahc_unlock(ahc, &s);
6960         }
6961         cel = &ccb->cel;
6962         target = ccb->ccb_h.target_id;
6963         lun = ccb->ccb_h.target_lun;
6964         channel = SIM_CHANNEL(ahc, sim);
6965         target_mask = 0x01 << target;
6966         if (channel == 'B')
6967                 target_mask <<= 8;
6968
6969         if (cel->enable != 0) {
6970                 u_int scsiseq;
6971
6972                 /* Are we already enabled?? */
6973                 if (lstate != NULL) {
6974                         xpt_print_path(ccb->ccb_h.path);
6975                         printf("Lun already enabled\n");
6976                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
6977                         return;
6978                 }
6979
6980                 if (cel->grp6_len != 0
6981                  || cel->grp7_len != 0) {
6982                         /*
6983                          * Don't (yet?) support vendor
6984                          * specific commands.
6985                          */
6986                         ccb->ccb_h.status = CAM_REQ_INVALID;
6987                         printf("Non-zero Group Codes\n");
6988                         return;
6989                 }
6990
6991                 /*
6992                  * Seems to be okay.
6993                  * Setup our data structures.
6994                  */
6995                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
6996                         tstate = ahc_alloc_tstate(ahc, target, channel);
6997                         if (tstate == NULL) {
6998                                 xpt_print_path(ccb->ccb_h.path);
6999                                 printf("Couldn't allocate tstate\n");
7000                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7001                                 return;
7002                         }
7003                 }
7004                 lstate = kmalloc(sizeof(*lstate), M_DEVBUF, M_INTWAIT | M_ZERO);
7005                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
7006                                          xpt_path_path_id(ccb->ccb_h.path),
7007                                          xpt_path_target_id(ccb->ccb_h.path),
7008                                          xpt_path_lun_id(ccb->ccb_h.path));
7009                 if (status != CAM_REQ_CMP) {
7010                         kfree(lstate, M_DEVBUF);
7011                         xpt_print_path(ccb->ccb_h.path);
7012                         printf("Couldn't allocate path\n");
7013                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7014                         return;
7015                 }
7016                 SLIST_INIT(&lstate->accept_tios);
7017                 SLIST_INIT(&lstate->immed_notifies);
7018                 ahc_lock(ahc, &s);
7019                 ahc_pause(ahc);
7020                 if (target != CAM_TARGET_WILDCARD) {
7021                         tstate->enabled_luns[lun] = lstate;
7022                         ahc->enabled_luns++;
7023
7024                         if ((ahc->features & AHC_MULTI_TID) != 0) {
7025                                 u_int targid_mask;
7026
7027                                 targid_mask = ahc_inb(ahc, TARGID)
7028                                             | (ahc_inb(ahc, TARGID + 1) << 8);
7029
7030                                 targid_mask |= target_mask;
7031                                 ahc_outb(ahc, TARGID, targid_mask);
7032                                 ahc_outb(ahc, TARGID+1, (targid_mask >> 8));
7033
7034                                 ahc_update_scsiid(ahc, targid_mask);
7035                         } else {
7036                                 u_int our_id;
7037                                 char  channel;
7038
7039                                 channel = SIM_CHANNEL(ahc, sim);
7040                                 our_id = SIM_SCSI_ID(ahc, sim);
7041
7042                                 /*
7043                                  * This can only happen if selections
7044                                  * are not enabled
7045                                  */
7046                                 if (target != our_id) {
7047                                         u_int sblkctl;
7048                                         char  cur_channel;
7049                                         int   swap;
7050
7051                                         sblkctl = ahc_inb(ahc, SBLKCTL);
7052                                         cur_channel = (sblkctl & SELBUSB)
7053                                                     ? 'B' : 'A';
7054                                         if ((ahc->features & AHC_TWIN) == 0)
7055                                                 cur_channel = 'A';
7056                                         swap = cur_channel != channel;
7057                                         if (channel == 'A')
7058                                                 ahc->our_id = target;
7059                                         else
7060                                                 ahc->our_id_b = target;
7061
7062                                         if (swap)
7063                                                 ahc_outb(ahc, SBLKCTL,
7064                                                          sblkctl ^ SELBUSB);
7065
7066                                         ahc_outb(ahc, SCSIID, target);
7067
7068                                         if (swap)
7069                                                 ahc_outb(ahc, SBLKCTL, sblkctl);
7070                                 }
7071                         }
7072                 } else
7073                         ahc->black_hole = lstate;
7074                 /* Allow select-in operations */
7075                 if (ahc->black_hole != NULL && ahc->enabled_luns > 0) {
7076                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7077                         scsiseq |= ENSELI;
7078                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7079                         scsiseq = ahc_inb(ahc, SCSISEQ);
7080                         scsiseq |= ENSELI;
7081                         ahc_outb(ahc, SCSISEQ, scsiseq);
7082                 }
7083                 ahc_unpause(ahc);
7084                 ahc_unlock(ahc, &s);
7085                 ccb->ccb_h.status = CAM_REQ_CMP;
7086                 xpt_print_path(ccb->ccb_h.path);
7087                 printf("Lun now enabled for target mode\n");
7088         } else {
7089                 struct scb *scb;
7090                 int i, empty;
7091
7092                 if (lstate == NULL) {
7093                         ccb->ccb_h.status = CAM_LUN_INVALID;
7094                         return;
7095                 }
7096
7097                 ahc_lock(ahc, &s);
7098
7099                 ccb->ccb_h.status = CAM_REQ_CMP;
7100                 LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7101                         struct ccb_hdr *ccbh;
7102
7103                         ccbh = &scb->io_ctx->ccb_h;
7104                         if (ccbh->func_code == XPT_CONT_TARGET_IO
7105                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
7106                                 printf("CTIO pending\n");
7107                                 ccb->ccb_h.status = CAM_REQ_INVALID;
7108                                 ahc_unlock(ahc, &s);
7109                                 return;
7110                         }
7111                 }
7112
7113                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
7114                         printf("ATIOs pending\n");
7115                         ccb->ccb_h.status = CAM_REQ_INVALID;
7116                 }
7117
7118                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
7119                         printf("INOTs pending\n");
7120                         ccb->ccb_h.status = CAM_REQ_INVALID;
7121                 }
7122
7123                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
7124                         ahc_unlock(ahc, &s);
7125                         return;
7126                 }
7127
7128                 xpt_print_path(ccb->ccb_h.path);
7129                 printf("Target mode disabled\n");
7130                 xpt_free_path(lstate->path);
7131                 kfree(lstate, M_DEVBUF);
7132
7133                 ahc_pause(ahc);
7134                 /* Can we clean up the target too? */
7135                 if (target != CAM_TARGET_WILDCARD) {
7136                         tstate->enabled_luns[lun] = NULL;
7137                         ahc->enabled_luns--;
7138                         for (empty = 1, i = 0; i < 8; i++)
7139                                 if (tstate->enabled_luns[i] != NULL) {
7140                                         empty = 0;
7141                                         break;
7142                                 }
7143
7144                         if (empty) {
7145                                 ahc_free_tstate(ahc, target, channel,
7146                                                 /*force*/FALSE);
7147                                 if (ahc->features & AHC_MULTI_TID) {
7148                                         u_int targid_mask;
7149
7150                                         targid_mask = ahc_inb(ahc, TARGID)
7151                                                     | (ahc_inb(ahc, TARGID + 1)
7152                                                        << 8);
7153
7154                                         targid_mask &= ~target_mask;
7155                                         ahc_outb(ahc, TARGID, targid_mask);
7156                                         ahc_outb(ahc, TARGID+1,
7157                                                  (targid_mask >> 8));
7158                                         ahc_update_scsiid(ahc, targid_mask);
7159                                 }
7160                         }
7161                 } else {
7162
7163                         ahc->black_hole = NULL;
7164
7165                         /*
7166                          * We can't allow selections without
7167                          * our black hole device.
7168                          */
7169                         empty = TRUE;
7170                 }
7171                 if (ahc->enabled_luns == 0) {
7172                         /* Disallow select-in */
7173                         u_int scsiseq;
7174
7175                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7176                         scsiseq &= ~ENSELI;
7177                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7178                         scsiseq = ahc_inb(ahc, SCSISEQ);
7179                         scsiseq &= ~ENSELI;
7180                         ahc_outb(ahc, SCSISEQ, scsiseq);
7181
7182                         if ((ahc->features & AHC_MULTIROLE) == 0) {
7183                                 printf("Configuring Initiator Mode\n");
7184                                 ahc->flags &= ~AHC_TARGETROLE;
7185                                 ahc->flags |= AHC_INITIATORROLE;
7186                                 /*
7187                                  * Returning to a configuration that
7188                                  * fit previously will always succeed.
7189                                  */
7190                                 (void)ahc_loadseq(ahc);
7191                                 ahc_restart(ahc);
7192                                 /*
7193                                  * Unpaused.  The extra unpause
7194                                  * that follows is harmless.
7195                                  */
7196                         }
7197                 }
7198                 ahc_unpause(ahc);
7199                 ahc_unlock(ahc, &s);
7200         }
7201 }
7202
7203 static void
7204 ahc_update_scsiid(struct ahc_softc *ahc, u_int targid_mask)
7205 {
7206         u_int scsiid_mask;
7207         u_int scsiid;
7208
7209         if ((ahc->features & AHC_MULTI_TID) == 0)
7210                 panic("ahc_update_scsiid called on non-multitid unit\n");
7211
7212         /*
7213          * Since we will rely on the TARGID mask
7214          * for selection enables, ensure that OID
7215          * in SCSIID is not set to some other ID
7216          * that we don't want to allow selections on.
7217          */
7218         if ((ahc->features & AHC_ULTRA2) != 0)
7219                 scsiid = ahc_inb(ahc, SCSIID_ULTRA2);
7220         else
7221                 scsiid = ahc_inb(ahc, SCSIID);
7222         scsiid_mask = 0x1 << (scsiid & OID);
7223         if ((targid_mask & scsiid_mask) == 0) {
7224                 u_int our_id;
7225
7226                 /* ffs counts from 1 */
7227                 our_id = ffs(targid_mask);
7228                 if (our_id == 0)
7229                         our_id = ahc->our_id;
7230                 else
7231                         our_id--;
7232                 scsiid &= TID;
7233                 scsiid |= our_id;
7234         }
7235         if ((ahc->features & AHC_ULTRA2) != 0)
7236                 ahc_outb(ahc, SCSIID_ULTRA2, scsiid);
7237         else
7238                 ahc_outb(ahc, SCSIID, scsiid);
7239 }
7240
7241 void
7242 ahc_run_tqinfifo(struct ahc_softc *ahc, int paused)
7243 {
7244         struct target_cmd *cmd;
7245
7246         /*
7247          * If the card supports auto-access pause,
7248          * we can access the card directly regardless
7249          * of whether it is paused or not.
7250          */
7251         if ((ahc->features & AHC_AUTOPAUSE) != 0)
7252                 paused = TRUE;
7253
7254         ahc_sync_tqinfifo(ahc, BUS_DMASYNC_POSTREAD);
7255         while ((cmd = &ahc->targetcmds[ahc->tqinfifonext])->cmd_valid != 0) {
7256
7257                 /*
7258                  * Only advance through the queue if we
7259                  * have the resources to process the command.
7260                  */
7261                 if (ahc_handle_target_cmd(ahc, cmd) != 0)
7262                         break;
7263
7264                 cmd->cmd_valid = 0;
7265                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
7266                                 ahc->shared_data_dmamap,
7267                                 ahc_targetcmd_offset(ahc, ahc->tqinfifonext),
7268                                 sizeof(struct target_cmd),
7269                                 BUS_DMASYNC_PREREAD);
7270                 ahc->tqinfifonext++;
7271
7272                 /*
7273                  * Lazily update our position in the target mode incoming
7274                  * command queue as seen by the sequencer.
7275                  */
7276                 if ((ahc->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
7277                         if ((ahc->features & AHC_HS_MAILBOX) != 0) {
7278                                 u_int hs_mailbox;
7279
7280                                 hs_mailbox = ahc_inb(ahc, HS_MAILBOX);
7281                                 hs_mailbox &= ~HOST_TQINPOS;
7282                                 hs_mailbox |= ahc->tqinfifonext & HOST_TQINPOS;
7283                                 ahc_outb(ahc, HS_MAILBOX, hs_mailbox);
7284                         } else {
7285                                 if (!paused)
7286                                         ahc_pause(ahc);
7287                                 ahc_outb(ahc, KERNEL_TQINPOS,
7288                                          ahc->tqinfifonext & HOST_TQINPOS);
7289                                 if (!paused)
7290                                         ahc_unpause(ahc);
7291                         }
7292                 }
7293         }
7294 }
7295
7296 static int
7297 ahc_handle_target_cmd(struct ahc_softc *ahc, struct target_cmd *cmd)
7298 {
7299         struct    ahc_tmode_tstate *tstate;
7300         struct    ahc_tmode_lstate *lstate;
7301         struct    ccb_accept_tio *atio;
7302         uint8_t *byte;
7303         int       initiator;
7304         int       target;
7305         int       lun;
7306
7307         initiator = SCSIID_TARGET(ahc, cmd->scsiid);
7308         target = SCSIID_OUR_ID(cmd->scsiid);
7309         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
7310
7311         byte = cmd->bytes;
7312         tstate = ahc->enabled_targets[target];
7313         lstate = NULL;
7314         if (tstate != NULL)
7315                 lstate = tstate->enabled_luns[lun];
7316
7317         /*
7318          * Commands for disabled luns go to the black hole driver.
7319          */
7320         if (lstate == NULL)
7321                 lstate = ahc->black_hole;
7322
7323         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
7324         if (atio == NULL) {
7325                 ahc->flags |= AHC_TQINFIFO_BLOCKED;
7326                 /*
7327                  * Wait for more ATIOs from the peripheral driver for this lun.
7328                  */
7329                 if (bootverbose)
7330                         printf("%s: ATIOs exhausted\n", ahc_name(ahc));
7331                 return (1);
7332         } else
7333                 ahc->flags &= ~AHC_TQINFIFO_BLOCKED;
7334 #if 0
7335         printf("Incoming command from %d for %d:%d%s\n",
7336                initiator, target, lun,
7337                lstate == ahc->black_hole ? "(Black Holed)" : "");
7338 #endif
7339         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
7340
7341         if (lstate == ahc->black_hole) {
7342                 /* Fill in the wildcards */
7343                 atio->ccb_h.target_id = target;
7344                 atio->ccb_h.target_lun = lun;
7345         }
7346
7347         /*
7348          * Package it up and send it off to
7349          * whomever has this lun enabled.
7350          */
7351         atio->sense_len = 0;
7352         atio->init_id = initiator;
7353         if (byte[0] != 0xFF) {
7354                 /* Tag was included */
7355                 atio->tag_action = *byte++;
7356                 atio->tag_id = *byte++;
7357                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
7358         } else {
7359                 atio->ccb_h.flags = 0;
7360         }
7361         byte++;
7362
7363         /* Okay.  Now determine the cdb size based on the command code */
7364         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
7365         case 0:
7366                 atio->cdb_len = 6;
7367                 break;
7368         case 1:
7369         case 2:
7370                 atio->cdb_len = 10;
7371                 break;
7372         case 4:
7373                 atio->cdb_len = 16;
7374                 break;
7375         case 5:
7376                 atio->cdb_len = 12;
7377                 break;
7378         case 3:
7379         default:
7380                 /* Only copy the opcode. */
7381                 atio->cdb_len = 1;
7382                 printf("Reserved or VU command code type encountered\n");
7383                 break;
7384         }
7385
7386         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
7387
7388         atio->ccb_h.status |= CAM_CDB_RECVD;
7389
7390         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
7391                 /*
7392                  * We weren't allowed to disconnect.
7393                  * We're hanging on the bus until a
7394                  * continue target I/O comes in response
7395                  * to this accept tio.
7396                  */
7397 #if 0
7398                 printf("Received Immediate Command %d:%d:%d - %p\n",
7399                        initiator, target, lun, ahc->pending_device);
7400 #endif
7401                 ahc->pending_device = lstate;
7402                 ahc_freeze_ccb((union ccb *)atio);
7403                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
7404         }
7405         xpt_done((union ccb*)atio);
7406         return (0);
7407 }
7408
7409 #endif