sys/dev/disk/aic7xxx/aic7xxx.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2002 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#134 $
  41  *
  42  * $FreeBSD: src/sys/dev/aic7xxx/aic7xxx.c,v 1.41.2.27 2003/06/10 03:26:08 gibbs Exp $
  43  * $DragonFly: src/sys/dev/disk/aic7xxx/aic7xxx.c,v 1.14 2007/05/13 18:33:57 swildner Exp $
  44  */
  45
  46 #include "aic7xxx_osm.h"
  47 #include "aic7xxx_inline.h"
  48 #include "aicasm/aicasm_insformat.h"
  49
  50 /****************************** Softc Data ************************************/
  51 struct ahc_softc_tailq ahc_tailq = TAILQ_HEAD_INITIALIZER(ahc_tailq);
  52
  53 /***************************** Lookup Tables **********************************/
  54 char *ahc_chip_names[] =
  55 {
  56         "NONE",
  57         "aic7770",
  58         "aic7850",
  59         "aic7855",
  60         "aic7859",
  61         "aic7860",
  62         "aic7870",
  63         "aic7880",
  64         "aic7895",
  65         "aic7895C",
  66         "aic7890/91",
  67         "aic7896/97",
  68         "aic7892",
  69         "aic7899"
  70 };
  71 static const u_int num_chip_names = NUM_ELEMENTS(ahc_chip_names);
  72
  73 /*
  74  * Hardware error codes.
  75  */
  76 struct ahc_hard_error_entry {
  77         uint8_t error;
  78         char *errmesg;
  79 };
  80
  81 static struct ahc_hard_error_entry ahc_hard_errors[] = {
  82         { ILLHADDR,     "Illegal Host Access" },
  83         { ILLSADDR,     "Illegal Sequencer Address referenced" },
  84         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  85         { SQPARERR,     "Sequencer Parity Error" },
  86         { DPARERR,      "Data-path Parity Error" },
  87         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  88         { PCIERRSTAT,   "PCI Error detected" },
  89         { CIOPARERR,    "CIOBUS Parity Error" },
  90 };
  91 static const u_int num_errors = NUM_ELEMENTS(ahc_hard_errors);
  92
  93 static struct ahc_phase_table_entry ahc_phase_table[] =
  94 {
  95         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  96         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  97         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  98         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  99         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
 100         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
 101         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
 102         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
 103         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
 104         { 0,            MSG_NOOP,               "in unknown phase"      }
 105 };
 106
 107 /*
 108  * In most cases we only wish to itterate over real phases, so
 109  * exclude the last element from the count.
 110  */
 111 static const u_int num_phases = NUM_ELEMENTS(ahc_phase_table) - 1;
 112
 113 /*
 114  * Valid SCSIRATE values.  (p. 3-17)
 115  * Provides a mapping of tranfer periods in ns to the proper value to
 116  * stick in the scsixfer reg.
 117  */
 118 static struct ahc_syncrate ahc_syncrates[] =
 119 {
 120       /* ultra2    fast/ultra  period     rate */
 121         { 0x42,      0x000,      9,      "80.0" },
 122         { 0x03,      0x000,     10,      "40.0" },
 123         { 0x04,      0x000,     11,      "33.0" },
 124         { 0x05,      0x100,     12,      "20.0" },
 125         { 0x06,      0x110,     15,      "16.0" },
 126         { 0x07,      0x120,     18,      "13.4" },
 127         { 0x08,      0x000,     25,      "10.0" },
 128         { 0x19,      0x010,     31,      "8.0"  },
 129         { 0x1a,      0x020,     37,      "6.67" },
 130         { 0x1b,      0x030,     43,      "5.7"  },
 131         { 0x1c,      0x040,     50,      "5.0"  },
 132         { 0x00,      0x050,     56,      "4.4"  },
 133         { 0x00,      0x060,     62,      "4.0"  },
 134         { 0x00,      0x070,     68,      "3.6"  },
 135         { 0x00,      0x000,      0,      NULL   }
 136 };
 137
 138 /* Our Sequencer Program */
 139 #include "aic7xxx_seq.h"
 140
 141 /**************************** Function Declarations ***************************/
 142 static void             ahc_force_renegotiation(struct ahc_softc *ahc,
 143                                                 struct ahc_devinfo *devinfo);
 144 static struct ahc_tmode_tstate*
 145                         ahc_alloc_tstate(struct ahc_softc *ahc,
 146                                          u_int scsi_id, char channel);
 147 #ifdef AHC_TARGET_MODE
 148 static void             ahc_free_tstate(struct ahc_softc *ahc,
 149                                         u_int scsi_id, char channel, int force);
 150 #endif
 151 static struct ahc_syncrate*
 152                         ahc_devlimited_syncrate(struct ahc_softc *ahc,
 153                                                 struct ahc_initiator_tinfo *,
 154                                                 u_int *period,
 155                                                 u_int *ppr_options,
 156                                                 role_t role);
 157 static void             ahc_update_pending_scbs(struct ahc_softc *ahc);
 158 static void             ahc_fetch_devinfo(struct ahc_softc *ahc,
 159                                           struct ahc_devinfo *devinfo);
 160 static void             ahc_scb_devinfo(struct ahc_softc *ahc,
 161                                         struct ahc_devinfo *devinfo,
 162                                         struct scb *scb);
 163 static void             ahc_assert_atn(struct ahc_softc *ahc);
 164 static void             ahc_setup_initiator_msgout(struct ahc_softc *ahc,
 165                                                    struct ahc_devinfo *devinfo,
 166                                                    struct scb *scb);
 167 static void             ahc_build_transfer_msg(struct ahc_softc *ahc,
 168                                                struct ahc_devinfo *devinfo);
 169 static void             ahc_construct_sdtr(struct ahc_softc *ahc,
 170                                            struct ahc_devinfo *devinfo,
 171                                            u_int period, u_int offset);
 172 static void             ahc_construct_wdtr(struct ahc_softc *ahc,
 173                                            struct ahc_devinfo *devinfo,
 174                                            u_int bus_width);
 175 static void             ahc_construct_ppr(struct ahc_softc *ahc,
 176                                           struct ahc_devinfo *devinfo,
 177                                           u_int period, u_int offset,
 178                                           u_int bus_width, u_int ppr_options);
 179 static void             ahc_clear_msg_state(struct ahc_softc *ahc);
 180 static void             ahc_handle_proto_violation(struct ahc_softc *ahc);
 181 static void             ahc_handle_message_phase(struct ahc_softc *ahc);
 182 typedef enum {
 183         AHCMSG_1B,
 184         AHCMSG_2B,
 185         AHCMSG_EXT
 186 } ahc_msgtype;
 187 static int              ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
 188                                      u_int msgval, int full);
 189 static int              ahc_parse_msg(struct ahc_softc *ahc,
 190                                       struct ahc_devinfo *devinfo);
 191 static int              ahc_handle_msg_reject(struct ahc_softc *ahc,
 192                                               struct ahc_devinfo *devinfo);
 193 static void             ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
 194                                                 struct ahc_devinfo *devinfo);
 195 static void             ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
 196 static void             ahc_handle_devreset(struct ahc_softc *ahc,
 197                                             struct ahc_devinfo *devinfo,
 198                                             cam_status status, char *message,
 199                                             int verbose_level);
 200 #ifdef AHC_TARGET_MODE
 201 static void             ahc_setup_target_msgin(struct ahc_softc *ahc,
 202                                                struct ahc_devinfo *devinfo,
 203                                                struct scb *scb);
 204 #endif
 205
 206 static bus_dmamap_callback_t    ahc_dmamap_cb;
 207 static void                     ahc_build_free_scb_list(struct ahc_softc *ahc);
 208 static int                      ahc_init_scbdata(struct ahc_softc *ahc);
 209 static void                     ahc_fini_scbdata(struct ahc_softc *ahc);
 210 static void             ahc_qinfifo_requeue(struct ahc_softc *ahc,
 211                                             struct scb *prev_scb,
 212                                             struct scb *scb);
 213 static int              ahc_qinfifo_count(struct ahc_softc *ahc);
 214 static u_int            ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
 215                                                    u_int prev, u_int scbptr);
 216 static void             ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
 217 static u_int            ahc_rem_wscb(struct ahc_softc *ahc,
 218                                      u_int scbpos, u_int prev);
 219 static void             ahc_reset_current_bus(struct ahc_softc *ahc);
 220 #ifdef AHC_DUMP_SEQ
 221 static void             ahc_dumpseq(struct ahc_softc *ahc);
 222 #endif
 223 static int              ahc_loadseq(struct ahc_softc *ahc);
 224 static int              ahc_check_patch(struct ahc_softc *ahc,
 225                                         struct patch **start_patch,
 226                                         u_int start_instr, u_int *skip_addr);
 227 static void             ahc_download_instr(struct ahc_softc *ahc,
 228                                            u_int instrptr, uint8_t *dconsts);
 229 #ifdef AHC_TARGET_MODE
 230 static void             ahc_queue_lstate_event(struct ahc_softc *ahc,
 231                                                struct ahc_tmode_lstate *lstate,
 232                                                u_int initiator_id,
 233                                                u_int event_type,
 234                                                u_int event_arg);
 235 static void             ahc_update_scsiid(struct ahc_softc *ahc,
 236                                           u_int targid_mask);
 237 static int              ahc_handle_target_cmd(struct ahc_softc *ahc,
 238                                               struct target_cmd *cmd);
 239 #endif
 240 /************************* Sequencer Execution Control ************************/
 241 /*
 242  * Restart the sequencer program from address zero
 243  */
 244 void
 245 ahc_restart(struct ahc_softc *ahc)
 246 {
 247
 248         ahc_pause(ahc);
 249
 250         /* No more pending messages. */
 251         ahc_clear_msg_state(ahc);
 252
 253         ahc_outb(ahc, SCSISIGO, 0);             /* De-assert BSY */
 254         ahc_outb(ahc, MSG_OUT, MSG_NOOP);       /* No message to send */
 255         ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 256         ahc_outb(ahc, LASTPHASE, P_BUSFREE);
 257         ahc_outb(ahc, SAVED_SCSIID, 0xFF);
 258         ahc_outb(ahc, SAVED_LUN, 0xFF);
 259
 260         /*
 261          * Ensure that the sequencer's idea of TQINPOS
 262          * matches our own.  The sequencer increments TQINPOS
 263          * only after it sees a DMA complete and a reset could
 264          * occur before the increment leaving the kernel to believe
 265          * the command arrived but the sequencer to not.
 266          */
 267         ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
 268
 269         /* Always allow reselection */
 270         ahc_outb(ahc, SCSISEQ,
 271                  ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 272         if ((ahc->features & AHC_CMD_CHAN) != 0) {
 273                 /* Ensure that no DMA operations are in progress */
 274                 ahc_outb(ahc, CCSCBCNT, 0);
 275                 ahc_outb(ahc, CCSGCTL, 0);
 276                 ahc_outb(ahc, CCSCBCTL, 0);
 277         }
 278         /*
 279          * If we were in the process of DMA'ing SCB data into
 280          * an SCB, replace that SCB on the free list.  This prevents
 281          * an SCB leak.
 282          */
 283         if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
 284                 ahc_add_curscb_to_free_list(ahc);
 285                 ahc_outb(ahc, SEQ_FLAGS2,
 286                          ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
 287         }
 288         ahc_outb(ahc, MWI_RESIDUAL, 0);
 289         ahc_outb(ahc, SEQCTL, ahc->seqctl);
 290         ahc_outb(ahc, SEQADDR0, 0);
 291         ahc_outb(ahc, SEQADDR1, 0);
 292         ahc_unpause(ahc);
 293 }
 294
 295 /************************* Input/Output Queues ********************************/
 296 void
 297 ahc_run_qoutfifo(struct ahc_softc *ahc)
 298 {
 299         struct scb *scb;
 300         u_int  scb_index;
 301
 302         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
 303         while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
 304
 305                 scb_index = ahc->qoutfifo[ahc->qoutfifonext];
 306                 if ((ahc->qoutfifonext & 0x03) == 0x03) {
 307                         u_int modnext;
 308
 309                         /*
 310                          * Clear 32bits of QOUTFIFO at a time
 311                          * so that we don't clobber an incoming
 312                          * byte DMA to the array on architectures
 313                          * that only support 32bit load and store
 314                          * operations.
 315                          */
 316                         modnext = ahc->qoutfifonext & ~0x3;
 317                         *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
 318                         ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 319                                         ahc->shared_data_dmamap,
 320                                         /*offset*/modnext, /*len*/4,
 321                                         BUS_DMASYNC_PREREAD);
 322                 }
 323                 ahc->qoutfifonext++;
 324
 325                 scb = ahc_lookup_scb(ahc, scb_index);
 326                 if (scb == NULL) {
 327                         kprintf("%s: WARNING no command for scb %d "
 328                                "(cmdcmplt)\nQOUTPOS = %d\n",
 329                                ahc_name(ahc), scb_index,
 330                                (ahc->qoutfifonext - 1) & 0xFF);
 331                         continue;
 332                 }
 333
 334                 /*
 335                  * Save off the residual
 336                  * if there is one.
 337                  */
 338                 ahc_update_residual(ahc, scb);
 339                 ahc_done(ahc, scb);
 340         }
 341 }
 342
 343 void
 344 ahc_run_untagged_queues(struct ahc_softc *ahc)
 345 {
 346         int i;
 347
 348         for (i = 0; i < 16; i++)
 349                 ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
 350 }
 351
 352 void
 353 ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
 354 {
 355         struct scb *scb;
 356
 357         if (ahc->untagged_queue_lock != 0)
 358                 return;
 359
 360         if ((scb = TAILQ_FIRST(queue)) != NULL
 361          && (scb->flags & SCB_ACTIVE) == 0) {
 362                 scb->flags |= SCB_ACTIVE;
 363                 ahc_queue_scb(ahc, scb);
 364         }
 365 }
 366
 367 /************************* Interrupt Handling *********************************/
 368 void
 369 ahc_handle_brkadrint(struct ahc_softc *ahc)
 370 {
 371         /*
 372          * We upset the sequencer :-(
 373          * Lookup the error message
 374          */
 375         int i;
 376         int error;
 377
 378         error = ahc_inb(ahc, ERROR);
 379         for (i = 0; error != 1 && i < num_errors; i++)
 380                 error >>= 1;
 381         kprintf("%s: brkadrint, %s at seqaddr = 0x%x\n",
 382                ahc_name(ahc), ahc_hard_errors[i].errmesg,
 383                ahc_inb(ahc, SEQADDR0) |
 384                (ahc_inb(ahc, SEQADDR1) << 8));
 385
 386         ahc_dump_card_state(ahc);
 387
 388         /* Tell everyone that this HBA is no longer available */
 389         ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 390                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 391                        CAM_NO_HBA);
 392
 393         /* Disable all interrupt sources by resetting the controller */
 394         ahc_shutdown(ahc);
 395 }
 396
 397 void
 398 ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
 399 {
 400         struct scb *scb;
 401         struct ahc_devinfo devinfo;
 402
 403         ahc_fetch_devinfo(ahc, &devinfo);
 404
 405         /*
 406          * Clear the upper byte that holds SEQINT status
 407          * codes and clear the SEQINT bit. We will unpause
 408          * the sequencer, if appropriate, after servicing
 409          * the request.
 410          */
 411         ahc_outb(ahc, CLRINT, CLRSEQINT);
 412         switch (intstat & SEQINT_MASK) {
 413         case BAD_STATUS:
 414         {
 415                 u_int  scb_index;
 416                 struct hardware_scb *hscb;
 417
 418                 /*
 419                  * Set the default return value to 0 (don't
 420                  * send sense).  The sense code will change
 421                  * this if needed.
 422                  */
 423                 ahc_outb(ahc, RETURN_1, 0);
 424
 425                 /*
 426                  * The sequencer will notify us when a command
 427                  * has an error that would be of interest to
 428                  * the kernel.  This allows us to leave the sequencer
 429                  * running in the common case of command completes
 430                  * without error.  The sequencer will already have
 431                  * dma'd the SCB back up to us, so we can reference
 432                  * the in kernel copy directly.
 433                  */
 434                 scb_index = ahc_inb(ahc, SCB_TAG);
 435                 scb = ahc_lookup_scb(ahc, scb_index);
 436                 if (scb == NULL) {
 437                         ahc_print_devinfo(ahc, &devinfo);
 438                         kprintf("ahc_intr - referenced scb "
 439                                "not valid during seqint 0x%x scb(%d)\n",
 440                                intstat, scb_index);
 441                         ahc_dump_card_state(ahc);
 442                         panic("for safety");
 443                         goto unpause;
 444                 }
 445
 446                 hscb = scb->hscb;
 447
 448                 /* Don't want to clobber the original sense code */
 449                 if ((scb->flags & SCB_SENSE) != 0) {
 450                         /*
 451                          * Clear the SCB_SENSE Flag and have
 452                          * the sequencer do a normal command
 453                          * complete.
 454                          */
 455                         scb->flags &= ~SCB_SENSE;
 456                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
 457                         break;
 458                 }
 459                 ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
 460                 /* Freeze the queue until the client sees the error. */
 461                 ahc_freeze_devq(ahc, scb);
 462                 ahc_freeze_scb(scb);
 463                 ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
 464                 switch (hscb->shared_data.status.scsi_status) {
 465                 case SCSI_STATUS_OK:
 466                         kprintf("%s: Interrupted for staus of 0???\n",
 467                                ahc_name(ahc));
 468                         break;
 469                 case SCSI_STATUS_CMD_TERMINATED:
 470                 case SCSI_STATUS_CHECK_COND:
 471                 {
 472                         struct ahc_dma_seg *sg;
 473                         struct scsi_sense *sc;
 474                         struct ahc_initiator_tinfo *targ_info;
 475                         struct ahc_tmode_tstate *tstate;
 476                         struct ahc_transinfo *tinfo;
 477 #ifdef AHC_DEBUG
 478                         if (ahc_debug & AHC_SHOW_SENSE) {
 479                                 ahc_print_path(ahc, scb);
 480                                 kprintf("SCB %d: requests Check Status\n",
 481                                        scb->hscb->tag);
 482                         }
 483 #endif
 484
 485                         if (ahc_perform_autosense(scb) == 0)
 486                                 break;
 487
 488                         targ_info = ahc_fetch_transinfo(ahc,
 489                                                         devinfo.channel,
 490                                                         devinfo.our_scsiid,
 491                                                         devinfo.target,
 492                                                         &tstate);
 493                         tinfo = &targ_info->curr;
 494                         sg = scb->sg_list;
 495                         sc = (struct scsi_sense *)(&hscb->shared_data.cdb);
 496                         /*
 497                          * Save off the residual if there is one.
 498                          */
 499                         ahc_update_residual(ahc, scb);
 500 #ifdef AHC_DEBUG
 501                         if (ahc_debug & AHC_SHOW_SENSE) {
 502                                 ahc_print_path(ahc, scb);
 503                                 kprintf("Sending Sense\n");
 504                         }
 505 #endif
 506                         sg->addr = ahc_get_sense_bufaddr(ahc, scb);
 507                         sg->len = ahc_get_sense_bufsize(ahc, scb);
 508                         sg->len |= AHC_DMA_LAST_SEG;
 509
 510                         /* Fixup byte order */
 511                         sg->addr = ahc_htole32(sg->addr);
 512                         sg->len = ahc_htole32(sg->len);
 513
 514                         sc->opcode = REQUEST_SENSE;
 515                         sc->byte2 = 0;
 516                         if (tinfo->protocol_version <= SCSI_REV_2
 517                          && SCB_GET_LUN(scb) < 8)
 518                                 sc->byte2 = SCB_GET_LUN(scb) << 5;
 519                         sc->unused[0] = 0;
 520                         sc->unused[1] = 0;
 521                         sc->length = sg->len;
 522                         sc->control = 0;
 523
 524                         /*
 525                          * We can't allow the target to disconnect.
 526                          * This will be an untagged transaction and
 527                          * having the target disconnect will make this
 528                          * transaction indestinguishable from outstanding
 529                          * tagged transactions.
 530                          */
 531                         hscb->control = 0;
 532
 533                         /*
 534                          * This request sense could be because the
 535                          * the device lost power or in some other
 536                          * way has lost our transfer negotiations.
 537                          * Renegotiate if appropriate.  Unit attention
 538                          * errors will be reported before any data
 539                          * phases occur.
 540                          */
 541                         if (ahc_get_residual(scb)
 542                          == ahc_get_transfer_length(scb)) {
 543                                 ahc_update_neg_request(ahc, &devinfo,
 544                                                        tstate, targ_info,
 545                                                        AHC_NEG_IF_NON_ASYNC);
 546                         }
 547                         if (tstate->auto_negotiate & devinfo.target_mask) {
 548                                 hscb->control |= MK_MESSAGE;
 549                                 scb->flags &= ~SCB_NEGOTIATE;
 550                                 scb->flags |= SCB_AUTO_NEGOTIATE;
 551                         }
 552                         hscb->cdb_len = sizeof(*sc);
 553                         hscb->dataptr = sg->addr;
 554                         hscb->datacnt = sg->len;
 555                         hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
 556                         hscb->sgptr = ahc_htole32(hscb->sgptr);
 557                         scb->sg_count = 1;
 558                         scb->flags |= SCB_SENSE;
 559                         ahc_qinfifo_requeue_tail(ahc, scb);
 560                         ahc_outb(ahc, RETURN_1, SEND_SENSE);
 561                         /*
 562                          * Ensure we have enough time to actually
 563                          * retrieve the sense.
 564                          */
 565                         ahc_scb_timer_reset(scb, 5 * 1000000);
 566                         break;
 567                 }
 568                 default:
 569                         break;
 570                 }
 571                 break;
 572         }
 573         case NO_MATCH:
 574         {
 575                 /* Ensure we don't leave the selection hardware on */
 576                 ahc_outb(ahc, SCSISEQ,
 577                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
 578
 579                 kprintf("%s:%c:%d: no active SCB for reconnecting "
 580                        "target - issuing BUS DEVICE RESET\n",
 581                        ahc_name(ahc), devinfo.channel, devinfo.target);
 582                 kprintf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
 583                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
 584                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
 585                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
 586                 kprintf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
 587                        "SINDEX == 0x%x\n",
 588                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
 589                        ahc_index_busy_tcl(ahc,
 590                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
 591                                       ahc_inb(ahc, SAVED_LUN))),
 592                        ahc_inb(ahc, SINDEX));
 593                 kprintf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
 594                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
 595                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
 596                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
 597                        ahc_inb(ahc, SCB_CONTROL));
 598                 kprintf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
 599                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
 600                 kprintf("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
 601                 kprintf("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
 602                 ahc_dump_card_state(ahc);
 603                 ahc->msgout_buf[0] = MSG_BUS_DEV_RESET;
 604                 ahc->msgout_len = 1;
 605                 ahc->msgout_index = 0;
 606                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
 607                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
 608                 ahc_assert_atn(ahc);
 609                 break;
 610         }
 611         case SEND_REJECT:
 612         {
 613                 u_int rejbyte = ahc_inb(ahc, ACCUM);
 614                 kprintf("%s:%c:%d: Warning - unknown message received from "
 615                        "target (0x%x).  Rejecting\n",
 616                        ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
 617                 break;
 618         }
 619         case PROTO_VIOLATION:
 620         {
 621                 ahc_handle_proto_violation(ahc);
 622                 break;
 623         }
 624         case IGN_WIDE_RES:
 625                 ahc_handle_ign_wide_residue(ahc, &devinfo);
 626                 break;
 627         case PDATA_REINIT:
 628                 ahc_reinitialize_dataptrs(ahc);
 629                 break;
 630         case BAD_PHASE:
 631         {
 632                 u_int lastphase;
 633
 634                 lastphase = ahc_inb(ahc, LASTPHASE);
 635                 kprintf("%s:%c:%d: unknown scsi bus phase %x, "
 636                        "lastphase = 0x%x.  Attempting to continue\n",
 637                        ahc_name(ahc), devinfo.channel, devinfo.target,
 638                        lastphase, ahc_inb(ahc, SCSISIGI));
 639                 break;
 640         }
 641         case MISSED_BUSFREE:
 642         {
 643                 u_int lastphase;
 644
 645                 lastphase = ahc_inb(ahc, LASTPHASE);
 646                 kprintf("%s:%c:%d: Missed busfree. "
 647                        "Lastphase = 0x%x, Curphase = 0x%x\n",
 648                        ahc_name(ahc), devinfo.channel, devinfo.target,
 649                        lastphase, ahc_inb(ahc, SCSISIGI));
 650                 ahc_restart(ahc);
 651                 return;
 652         }
 653         case HOST_MSG_LOOP:
 654         {
 655                 /*
 656                  * The sequencer has encountered a message phase
 657                  * that requires host assistance for completion.
 658                  * While handling the message phase(s), we will be
 659                  * notified by the sequencer after each byte is
 660                  * transfered so we can track bus phase changes.
 661                  *
 662                  * If this is the first time we've seen a HOST_MSG_LOOP
 663                  * interrupt, initialize the state of the host message
 664                  * loop.
 665                  */
 666                 if (ahc->msg_type == MSG_TYPE_NONE) {
 667                         struct scb *scb;
 668                         u_int scb_index;
 669                         u_int bus_phase;
 670
 671                         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
 672                         if (bus_phase != P_MESGIN
 673                          && bus_phase != P_MESGOUT) {
 674                                 kprintf("ahc_intr: HOST_MSG_LOOP bad "
 675                                        "phase 0x%x\n",
 676                                       bus_phase);
 677                                 /*
 678                                  * Probably transitioned to bus free before
 679                                  * we got here.  Just punt the message.
 680                                  */
 681                                 ahc_clear_intstat(ahc);
 682                                 ahc_restart(ahc);
 683                                 return;
 684                         }
 685
 686                         scb_index = ahc_inb(ahc, SCB_TAG);
 687                         scb = ahc_lookup_scb(ahc, scb_index);
 688                         if (devinfo.role == ROLE_INITIATOR) {
 689                                 if (scb == NULL)
 690                                         panic("HOST_MSG_LOOP with "
 691                                               "invalid SCB %x\n", scb_index);
 692
 693                                 if (bus_phase == P_MESGOUT)
 694                                         ahc_setup_initiator_msgout(ahc,
 695                                                                    &devinfo,
 696                                                                    scb);
 697                                 else {
 698                                         ahc->msg_type =
 699                                             MSG_TYPE_INITIATOR_MSGIN;
 700                                         ahc->msgin_index = 0;
 701                                 }
 702                         }
 703 #ifdef AHC_TARGET_MODE
 704                         else {
 705                                 if (bus_phase == P_MESGOUT) {
 706                                         ahc->msg_type =
 707                                             MSG_TYPE_TARGET_MSGOUT;
 708                                         ahc->msgin_index = 0;
 709                                 }
 710                                 else
 711                                         ahc_setup_target_msgin(ahc,
 712                                                                &devinfo,
 713                                                                scb);
 714                         }
 715 #endif
 716                 }
 717
 718                 ahc_handle_message_phase(ahc);
 719                 break;
 720         }
 721         case PERR_DETECTED:
 722         {
 723                 /*
 724                  * If we've cleared the parity error interrupt
 725                  * but the sequencer still believes that SCSIPERR
 726                  * is true, it must be that the parity error is
 727                  * for the currently presented byte on the bus,
 728                  * and we are not in a phase (data-in) where we will
 729                  * eventually ack this byte.  Ack the byte and
 730                  * throw it away in the hope that the target will
 731                  * take us to message out to deliver the appropriate
 732                  * error message.
 733                  */
 734                 if ((intstat & SCSIINT) == 0
 735                  && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
 736
 737                         if ((ahc->features & AHC_DT) == 0) {
 738                                 u_int curphase;
 739
 740                                 /*
 741                                  * The hardware will only let you ack bytes
 742                                  * if the expected phase in SCSISIGO matches
 743                                  * the current phase.  Make sure this is
 744                                  * currently the case.
 745                                  */
 746                                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
 747                                 ahc_outb(ahc, LASTPHASE, curphase);
 748                                 ahc_outb(ahc, SCSISIGO, curphase);
 749                         }
 750                         if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
 751                                 int wait;
 752
 753                                 /*
 754                                  * In a data phase.  Faster to bitbucket
 755                                  * the data than to individually ack each
 756                                  * byte.  This is also the only strategy
 757                                  * that will work with AUTOACK enabled.
 758                                  */
 759                                 ahc_outb(ahc, SXFRCTL1,
 760                                          ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
 761                                 wait = 5000;
 762                                 while (--wait != 0) {
 763                                         if ((ahc_inb(ahc, SCSISIGI)
 764                                           & (CDI|MSGI)) != 0)
 765                                                 break;
 766                                         ahc_delay(100);
 767                                 }
 768                                 ahc_outb(ahc, SXFRCTL1,
 769                                          ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 770                                 if (wait == 0) {
 771                                         struct  scb *scb;
 772                                         u_int   scb_index;
 773
 774                                         ahc_print_devinfo(ahc, &devinfo);
 775                                         kprintf("Unable to clear parity error.  "
 776                                                "Resetting bus.\n");
 777                                         scb_index = ahc_inb(ahc, SCB_TAG);
 778                                         scb = ahc_lookup_scb(ahc, scb_index);
 779                                         if (scb != NULL)
 780                                                 ahc_set_transaction_status(scb,
 781                                                     CAM_UNCOR_PARITY);
 782                                         ahc_reset_channel(ahc, devinfo.channel,
 783                                                           /*init reset*/TRUE);
 784                                 }
 785                         } else {
 786                                 ahc_inb(ahc, SCSIDATL);
 787                         }
 788                 }
 789                 break;
 790         }
 791         case DATA_OVERRUN:
 792         {
 793                 /*
 794                  * When the sequencer detects an overrun, it
 795                  * places the controller in "BITBUCKET" mode
 796                  * and allows the target to complete its transfer.
 797                  * Unfortunately, none of the counters get updated
 798                  * when the controller is in this mode, so we have
 799                  * no way of knowing how large the overrun was.
 800                  */
 801                 u_int scbindex = ahc_inb(ahc, SCB_TAG);
 802                 u_int lastphase = ahc_inb(ahc, LASTPHASE);
 803                 u_int i;
 804
 805                 scb = ahc_lookup_scb(ahc, scbindex);
 806                 for (i = 0; i < num_phases; i++) {
 807                         if (lastphase == ahc_phase_table[i].phase)
 808                                 break;
 809                 }
 810                 ahc_print_path(ahc, scb);
 811                 kprintf("data overrun detected %s."
 812                        "  Tag == 0x%x.\n",
 813                        ahc_phase_table[i].phasemsg,
 814                        scb->hscb->tag);
 815                 ahc_print_path(ahc, scb);
 816                 kprintf("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
 817                        ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
 818                        ahc_get_transfer_length(scb), scb->sg_count);
 819                 if (scb->sg_count > 0) {
 820                         for (i = 0; i < scb->sg_count; i++) {
 821
 822                                 kprintf("sg[%d] - Addr 0x%x%x : Length %d\n",
 823                                        i,
 824                                        (ahc_le32toh(scb->sg_list[i].len) >> 24
 825                                         & SG_HIGH_ADDR_BITS),
 826                                        ahc_le32toh(scb->sg_list[i].addr),
 827                                        ahc_le32toh(scb->sg_list[i].len)
 828                                        & AHC_SG_LEN_MASK);
 829                         }
 830                 }
 831                 /*
 832                  * Set this and it will take effect when the
 833                  * target does a command complete.
 834                  */
 835                 ahc_freeze_devq(ahc, scb);
 836                 if ((scb->flags & SCB_SENSE) == 0) {
 837                         ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
 838                 } else {
 839                         scb->flags &= ~SCB_SENSE;
 840                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
 841                 }
 842                 ahc_freeze_scb(scb);
 843
 844                 if ((ahc->features & AHC_ULTRA2) != 0) {
 845                         /*
 846                          * Clear the channel in case we return
 847                          * to data phase later.
 848                          */
 849                         ahc_outb(ahc, SXFRCTL0,
 850                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
 851                         ahc_outb(ahc, SXFRCTL0,
 852                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
 853                 }
 854                 if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
 855                         u_int dscommand1;
 856
 857                         /* Ensure HHADDR is 0 for future DMA operations. */
 858                         dscommand1 = ahc_inb(ahc, DSCOMMAND1);
 859                         ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
 860                         ahc_outb(ahc, HADDR, 0);
 861                         ahc_outb(ahc, DSCOMMAND1, dscommand1);
 862                 }
 863                 break;
 864         }
 865         case MKMSG_FAILED:
 866         {
 867                 u_int scbindex;
 868
 869                 kprintf("%s:%c:%d:%d: Attempt to issue message failed\n",
 870                        ahc_name(ahc), devinfo.channel, devinfo.target,
 871                        devinfo.lun);
 872                 scbindex = ahc_inb(ahc, SCB_TAG);
 873                 scb = ahc_lookup_scb(ahc, scbindex);
 874                 if (scb != NULL
 875                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
 876                         /*
 877                          * Ensure that we didn't put a second instance of this
 878                          * SCB into the QINFIFO.
 879                          */
 880                         ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
 881                                            SCB_GET_CHANNEL(ahc, scb),
 882                                            SCB_GET_LUN(scb), scb->hscb->tag,
 883                                            ROLE_INITIATOR, /*status*/0,
 884                                            SEARCH_REMOVE);
 885                 break;
 886         }
 887         case NO_FREE_SCB:
 888         {
 889                 kprintf("%s: No free or disconnected SCBs\n", ahc_name(ahc));
 890                 ahc_dump_card_state(ahc);
 891                 panic("for safety");
 892                 break;
 893         }
 894         case SCB_MISMATCH:
 895         {
 896                 u_int scbptr;
 897
 898                 scbptr = ahc_inb(ahc, SCBPTR);
 899                 kprintf("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
 900                        scbptr, ahc_inb(ahc, ARG_1),
 901                        ahc->scb_data->hscbs[scbptr].tag);
 902                 ahc_dump_card_state(ahc);
 903                 panic("for saftey");
 904                 break;
 905         }
 906         case OUT_OF_RANGE:
 907         {
 908                 kprintf("%s: BTT calculation out of range\n", ahc_name(ahc));
 909                 kprintf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
 910                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
 911                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
 912                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
 913                 kprintf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
 914                        "SINDEX == 0x%x\n, A == 0x%x\n",
 915                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
 916                        ahc_index_busy_tcl(ahc,
 917                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
 918                                       ahc_inb(ahc, SAVED_LUN))),
 919                        ahc_inb(ahc, SINDEX),
 920                        ahc_inb(ahc, ACCUM));
 921                 kprintf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
 922                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
 923                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
 924                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
 925                        ahc_inb(ahc, SCB_CONTROL));
 926                 kprintf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
 927                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
 928                 ahc_dump_card_state(ahc);
 929                 panic("for safety");
 930                 break;
 931         }
 932         default:
 933                 kprintf("ahc_intr: seqint, "
 934                        "intstat == 0x%x, scsisigi = 0x%x\n",
 935                        intstat, ahc_inb(ahc, SCSISIGI));
 936                 break;
 937         }
 938 unpause:
 939         /*
 940          *  The sequencer is paused immediately on
 941          *  a SEQINT, so we should restart it when
 942          *  we're done.
 943          */
 944         ahc_unpause(ahc);
 945 }
 946
 947 void
 948 ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
 949 {
 950         u_int   scb_index;
 951         u_int   status0;
 952         u_int   status;
 953         struct  scb *scb;
 954         char    cur_channel;
 955         char    intr_channel;
 956
 957         if ((ahc->features & AHC_TWIN) != 0
 958          && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
 959                 cur_channel = 'B';
 960         else
 961                 cur_channel = 'A';
 962         intr_channel = cur_channel;
 963
 964         if ((ahc->features & AHC_ULTRA2) != 0)
 965                 status0 = ahc_inb(ahc, SSTAT0) & IOERR;
 966         else
 967                 status0 = 0;
 968         status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
 969         if (status == 0 && status0 == 0) {
 970                 if ((ahc->features & AHC_TWIN) != 0) {
 971                         /* Try the other channel */
 972                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
 973                         status = ahc_inb(ahc, SSTAT1)
 974                                & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
 975                         intr_channel = (cur_channel == 'A') ? 'B' : 'A';
 976                 }
 977                 if (status == 0) {
 978                         kprintf("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
 979                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
 980                         ahc_unpause(ahc);
 981                         return;
 982                 }
 983         }
 984
 985         /* Make sure the sequencer is in a safe location. */
 986         ahc_clear_critical_section(ahc);
 987
 988         scb_index = ahc_inb(ahc, SCB_TAG);
 989         scb = ahc_lookup_scb(ahc, scb_index);
 990         if (scb != NULL
 991          && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
 992                 scb = NULL;
 993
 994         if ((ahc->features & AHC_ULTRA2) != 0
 995          && (status0 & IOERR) != 0) {
 996                 int now_lvd;
 997
 998                 now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
 999                 kprintf("%s: Transceiver State Has Changed to %s mode\n",
1000                        ahc_name(ahc), now_lvd ? "LVD" : "SE");
1001                 ahc_outb(ahc, CLRSINT0, CLRIOERR);
1002                 /*
1003                  * When transitioning to SE mode, the reset line
1004                  * glitches, triggering an arbitration bug in some
1005                  * Ultra2 controllers.  This bug is cleared when we
1006                  * assert the reset line.  Since a reset glitch has
1007                  * already occurred with this transition and a
1008                  * transceiver state change is handled just like
1009                  * a bus reset anyway, asserting the reset line
1010                  * ourselves is safe.
1011                  */
1012                 ahc_reset_channel(ahc, intr_channel,
1013                                  /*Initiate Reset*/now_lvd == 0);
1014         } else if ((status & SCSIRSTI) != 0) {
1015                 kprintf("%s: Someone reset channel %c\n",
1016                         ahc_name(ahc), intr_channel);
1017                 if (intr_channel != cur_channel)
1018                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1019                 ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1020         } else if ((status & SCSIPERR) != 0) {
1021                 /*
1022                  * Determine the bus phase and queue an appropriate message.
1023                  * SCSIPERR is latched true as soon as a parity error
1024                  * occurs.  If the sequencer acked the transfer that
1025                  * caused the parity error and the currently presented
1026                  * transfer on the bus has correct parity, SCSIPERR will
1027                  * be cleared by CLRSCSIPERR.  Use this to determine if
1028                  * we should look at the last phase the sequencer recorded,
1029                  * or the current phase presented on the bus.
1030                  */
1031                 struct  ahc_devinfo devinfo;
1032                 u_int   mesg_out;
1033                 u_int   curphase;
1034                 u_int   errorphase;
1035                 u_int   lastphase;
1036                 u_int   scsirate;
1037                 u_int   i;
1038                 u_int   sstat2;
1039                 int     silent;
1040
1041                 lastphase = ahc_inb(ahc, LASTPHASE);
1042                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1043                 sstat2 = ahc_inb(ahc, SSTAT2);
1044                 ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1045                 /*
1046                  * For all phases save DATA, the sequencer won't
1047                  * automatically ack a byte that has a parity error
1048                  * in it.  So the only way that the current phase
1049                  * could be 'data-in' is if the parity error is for
1050                  * an already acked byte in the data phase.  During
1051                  * synchronous data-in transfers, we may actually
1052                  * ack bytes before latching the current phase in
1053                  * LASTPHASE, leading to the discrepancy between
1054                  * curphase and lastphase.
1055                  */
1056                 if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1057                  || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1058                         errorphase = curphase;
1059                 else
1060                         errorphase = lastphase;
1061
1062                 for (i = 0; i < num_phases; i++) {
1063                         if (errorphase == ahc_phase_table[i].phase)
1064                                 break;
1065                 }
1066                 mesg_out = ahc_phase_table[i].mesg_out;
1067                 silent = FALSE;
1068                 if (scb != NULL) {
1069                         if (SCB_IS_SILENT(scb))
1070                                 silent = TRUE;
1071                         else
1072                                 ahc_print_path(ahc, scb);
1073                         scb->flags |= SCB_TRANSMISSION_ERROR;
1074                 } else
1075                         kprintf("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1076                                SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1077                 scsirate = ahc_inb(ahc, SCSIRATE);
1078                 if (silent == FALSE) {
1079                         kprintf("parity error detected %s. "
1080                                "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1081                                ahc_phase_table[i].phasemsg,
1082                                ahc_inw(ahc, SEQADDR0),
1083                                scsirate);
1084                         if ((ahc->features & AHC_DT) != 0) {
1085                                 if ((sstat2 & CRCVALERR) != 0)
1086                                         kprintf("\tCRC Value Mismatch\n");
1087                                 if ((sstat2 & CRCENDERR) != 0)
1088                                         kprintf("\tNo terminal CRC packet "
1089                                                "received\n");
1090                                 if ((sstat2 & CRCREQERR) != 0)
1091                                         kprintf("\tIllegal CRC packet "
1092                                                "request\n");
1093                                 if ((sstat2 & DUAL_EDGE_ERR) != 0)
1094                                         kprintf("\tUnexpected %sDT Data Phase\n",
1095                                                (scsirate & SINGLE_EDGE)
1096                                              ? "" : "non-");
1097                         }
1098                 }
1099
1100                 if ((ahc->features & AHC_DT) != 0
1101                  && (sstat2 & DUAL_EDGE_ERR) != 0) {
1102                         /*
1103                          * This error applies regardless of
1104                          * data direction, so ignore the value
1105                          * in the phase table.
1106                          */
1107                         mesg_out = MSG_INITIATOR_DET_ERR;
1108                 }
1109
1110                 /*
1111                  * We've set the hardware to assert ATN if we
1112                  * get a parity error on "in" phases, so all we
1113                  * need to do is stuff the message buffer with
1114                  * the appropriate message.  "In" phases have set
1115                  * mesg_out to something other than MSG_NOP.
1116                  */
1117                 if (mesg_out != MSG_NOOP) {
1118                         if (ahc->msg_type != MSG_TYPE_NONE)
1119                                 ahc->send_msg_perror = TRUE;
1120                         else
1121                                 ahc_outb(ahc, MSG_OUT, mesg_out);
1122                 }
1123                 /*
1124                  * Force a renegotiation with this target just in
1125                  * case we are out of sync for some external reason
1126                  * unknown (or unreported) by the target.
1127                  */
1128                 ahc_fetch_devinfo(ahc, &devinfo);
1129                 ahc_force_renegotiation(ahc, &devinfo);
1130
1131                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1132                 ahc_unpause(ahc);
1133         } else if ((status & SELTO) != 0) {
1134                 u_int   scbptr;
1135
1136                 /* Stop the selection */
1137                 ahc_outb(ahc, SCSISEQ, 0);
1138
1139                 /* No more pending messages */
1140                 ahc_clear_msg_state(ahc);
1141
1142                 /* Clear interrupt state */
1143                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1144                 ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1145
1146                 /*
1147                  * Although the driver does not care about the
1148                  * 'Selection in Progress' status bit, the busy
1149                  * LED does.  SELINGO is only cleared by a successful
1150                  * selection, so we must manually clear it to insure
1151                  * the LED turns off just in case no future successful
1152                  * selections occur (e.g. no devices on the bus).
1153                  */
1154                 ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1155
1156                 scbptr = ahc_inb(ahc, WAITING_SCBH);
1157                 ahc_outb(ahc, SCBPTR, scbptr);
1158                 scb_index = ahc_inb(ahc, SCB_TAG);
1159
1160                 scb = ahc_lookup_scb(ahc, scb_index);
1161                 if (scb == NULL) {
1162                         kprintf("%s: ahc_intr - referenced scb not "
1163                                "valid during SELTO scb(%d, %d)\n",
1164                                ahc_name(ahc), scbptr, scb_index);
1165                         ahc_dump_card_state(ahc);
1166                 } else {
1167                         struct ahc_devinfo devinfo;
1168 #ifdef AHC_DEBUG
1169                         if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1170                                 ahc_print_path(ahc, scb);
1171                                 kprintf("Saw Selection Timeout for SCB 0x%x\n",
1172                                        scb_index);
1173                         }
1174 #endif
1175                         /*
1176                          * Force a renegotiation with this target just in
1177                          * case the cable was pulled and will later be
1178                          * re-attached.  The target may forget its negotiation
1179                          * settings with us should it attempt to reselect
1180                          * during the interruption.  The target will not issue
1181                          * a unit attention in this case, so we must always
1182                          * renegotiate.
1183                          */
1184                         ahc_scb_devinfo(ahc, &devinfo, scb);
1185                         ahc_force_renegotiation(ahc, &devinfo);
1186                         ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1187                         ahc_freeze_devq(ahc, scb);
1188                 }
1189                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1190                 ahc_restart(ahc);
1191         } else if ((status & BUSFREE) != 0
1192                 && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1193                 struct  ahc_devinfo devinfo;
1194                 u_int   lastphase;
1195                 u_int   saved_scsiid;
1196                 u_int   saved_lun;
1197                 u_int   target;
1198                 u_int   initiator_role_id;
1199                 char    channel;
1200                 int     printerror;
1201
1202                 /*
1203                  * Clear our selection hardware as soon as possible.
1204                  * We may have an entry in the waiting Q for this target,
1205                  * that is affected by this busfree and we don't want to
1206                  * go about selecting the target while we handle the event.
1207                  */
1208                 ahc_outb(ahc, SCSISEQ,
1209                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1210
1211                 /*
1212                  * Disable busfree interrupts and clear the busfree
1213                  * interrupt status.  We do this here so that several
1214                  * bus transactions occur prior to clearing the SCSIINT
1215                  * latch.  It can take a bit for the clearing to take effect.
1216                  */
1217                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1218                 ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1219
1220                 /*
1221                  * Look at what phase we were last in.
1222                  * If its message out, chances are pretty good
1223                  * that the busfree was in response to one of
1224                  * our abort requests.
1225                  */
1226                 lastphase = ahc_inb(ahc, LASTPHASE);
1227                 saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1228                 saved_lun = ahc_inb(ahc, SAVED_LUN);
1229                 target = SCSIID_TARGET(ahc, saved_scsiid);
1230                 initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1231                 channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1232                 ahc_compile_devinfo(&devinfo, initiator_role_id,
1233                                     target, saved_lun, channel, ROLE_INITIATOR);
1234                 printerror = 1;
1235
1236                 if (lastphase == P_MESGOUT) {
1237                         u_int tag;
1238
1239                         tag = SCB_LIST_NULL;
1240                         if (ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT_TAG, TRUE)
1241                          || ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT, TRUE)) {
1242                                 if (ahc->msgout_buf[ahc->msgout_index - 1]
1243                                  == MSG_ABORT_TAG)
1244                                         tag = scb->hscb->tag;
1245                                 ahc_print_path(ahc, scb);
1246                                 kprintf("SCB %d - Abort%s Completed.\n",
1247                                        scb->hscb->tag, tag == SCB_LIST_NULL ?
1248                                        "" : " Tag");
1249                                 ahc_abort_scbs(ahc, target, channel,
1250                                                saved_lun, tag,
1251                                                ROLE_INITIATOR,
1252                                                CAM_REQ_ABORTED);
1253                                 printerror = 0;
1254                         } else if (ahc_sent_msg(ahc, AHCMSG_1B,
1255                                                 MSG_BUS_DEV_RESET, TRUE)) {
1256 #if defined(__DragonFly__) || defined(__FreeBSD__)
1257                                 /*
1258                                  * Don't mark the user's request for this BDR
1259                                  * as completing with CAM_BDR_SENT.  CAM3
1260                                  * specifies CAM_REQ_CMP.
1261                                  */
1262                                 if (scb != NULL
1263                                  && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
1264                                  && ahc_match_scb(ahc, scb, target, channel,
1265                                                   CAM_LUN_WILDCARD,
1266                                                   SCB_LIST_NULL,
1267                                                   ROLE_INITIATOR)) {
1268                                         ahc_set_transaction_status(scb, CAM_REQ_CMP);
1269                                 }
1270 #endif
1271                                 ahc_compile_devinfo(&devinfo,
1272                                                     initiator_role_id,
1273                                                     target,
1274                                                     CAM_LUN_WILDCARD,
1275                                                     channel,
1276                                                     ROLE_INITIATOR);
1277                                 ahc_handle_devreset(ahc, &devinfo,
1278                                                     CAM_BDR_SENT,
1279                                                     "Bus Device Reset",
1280                                                     /*verbose_level*/0);
1281                                 printerror = 0;
1282                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1283                                                 MSG_EXT_PPR, FALSE)) {
1284                                 struct ahc_initiator_tinfo *tinfo;
1285                                 struct ahc_tmode_tstate *tstate;
1286
1287                                 /*
1288                                  * PPR Rejected.  Try non-ppr negotiation
1289                                  * and retry command.
1290                                  */
1291                                 tinfo = ahc_fetch_transinfo(ahc,
1292                                                             devinfo.channel,
1293                                                             devinfo.our_scsiid,
1294                                                             devinfo.target,
1295                                                             &tstate);
1296                                 tinfo->curr.transport_version = 2;
1297                                 tinfo->goal.transport_version = 2;
1298                                 tinfo->goal.ppr_options = 0;
1299                                 ahc_qinfifo_requeue_tail(ahc, scb);
1300                                 printerror = 0;
1301                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1302                                                 MSG_EXT_WDTR, FALSE)) {
1303                                 /*
1304                                  * Negotiation Rejected.  Go-narrow and
1305                                  * retry command.
1306                                  */
1307                                 ahc_set_width(ahc, &devinfo,
1308                                               MSG_EXT_WDTR_BUS_8_BIT,
1309                                               AHC_TRANS_CUR|AHC_TRANS_GOAL,
1310                                               /*paused*/TRUE);
1311                                 ahc_qinfifo_requeue_tail(ahc, scb);
1312                                 printerror = 0;
1313                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1314                                                 MSG_EXT_SDTR, FALSE)) {
1315                                 /*
1316                                  * Negotiation Rejected.  Go-async and
1317                                  * retry command.
1318                                  */
1319                                 ahc_set_syncrate(ahc, &devinfo,
1320                                                 /*syncrate*/NULL,
1321                                                 /*period*/0, /*offset*/0,
1322                                                 /*ppr_options*/0,
1323                                                 AHC_TRANS_CUR|AHC_TRANS_GOAL,
1324                                                 /*paused*/TRUE);
1325                                 ahc_qinfifo_requeue_tail(ahc, scb);
1326                                 printerror = 0;
1327                         }
1328                 }
1329                 if (printerror != 0) {
1330                         u_int i;
1331
1332                         if (scb != NULL) {
1333                                 u_int tag;
1334
1335                                 if ((scb->hscb->control & TAG_ENB) != 0)
1336                                         tag = scb->hscb->tag;
1337                                 else
1338                                         tag = SCB_LIST_NULL;
1339                                 ahc_print_path(ahc, scb);
1340                                 ahc_abort_scbs(ahc, target, channel,
1341                                                SCB_GET_LUN(scb), tag,
1342                                                ROLE_INITIATOR,
1343                                                CAM_UNEXP_BUSFREE);
1344                         } else {
1345                                 /*
1346                                  * We had not fully identified this connection,
1347                                  * so we cannot abort anything.
1348                                  */
1349                                 kprintf("%s: ", ahc_name(ahc));
1350                         }
1351                         for (i = 0; i < num_phases; i++) {
1352                                 if (lastphase == ahc_phase_table[i].phase)
1353                                         break;
1354                         }
1355                         if (lastphase != P_BUSFREE) {
1356                                 /*
1357                                  * Renegotiate with this device at the
1358                                  * next oportunity just in case this busfree
1359                                  * is due to a negotiation mismatch with the
1360                                  * device.
1361                                  */
1362                                 ahc_force_renegotiation(ahc, &devinfo);
1363                         }
1364                         kprintf("Unexpected busfree %s\n"
1365                                "SEQADDR == 0x%x\n",
1366                                ahc_phase_table[i].phasemsg,
1367                                ahc_inb(ahc, SEQADDR0)
1368                                 | (ahc_inb(ahc, SEQADDR1) << 8));
1369                 }
1370                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1371                 ahc_restart(ahc);
1372         } else {
1373                 kprintf("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1374                        ahc_name(ahc), status);
1375                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1376         }
1377 }
1378
1379 /*
1380  * Force renegotiation to occur the next time we initiate
1381  * a command to the current device.
1382  */
1383 static void
1384 ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1385 {
1386         struct  ahc_initiator_tinfo *targ_info;
1387         struct  ahc_tmode_tstate *tstate;
1388
1389         targ_info = ahc_fetch_transinfo(ahc,
1390                                         devinfo->channel,
1391                                         devinfo->our_scsiid,
1392                                         devinfo->target,
1393                                         &tstate);
1394         ahc_update_neg_request(ahc, devinfo, tstate,
1395                                targ_info, AHC_NEG_IF_NON_ASYNC);
1396 }
1397
1398 #define AHC_MAX_STEPS 2000
1399 void
1400 ahc_clear_critical_section(struct ahc_softc *ahc)
1401 {
1402         int     stepping;
1403         int     steps;
1404         u_int   simode0;
1405         u_int   simode1;
1406
1407         if (ahc->num_critical_sections == 0)
1408                 return;
1409
1410         stepping = FALSE;
1411         steps = 0;
1412         simode0 = 0;
1413         simode1 = 0;
1414         for (;;) {
1415                 struct  cs *cs;
1416                 u_int   seqaddr;
1417                 u_int   i;
1418
1419                 seqaddr = ahc_inb(ahc, SEQADDR0)
1420                         | (ahc_inb(ahc, SEQADDR1) << 8);
1421
1422                 /*
1423                  * Seqaddr represents the next instruction to execute,
1424                  * so we are really executing the instruction just
1425                  * before it.
1426                  */
1427                 if (seqaddr != 0)
1428                         seqaddr -= 1;
1429                 cs = ahc->critical_sections;
1430                 for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
1431
1432                         if (cs->begin < seqaddr && cs->end >= seqaddr)
1433                                 break;
1434                 }
1435
1436                 if (i == ahc->num_critical_sections)
1437                         break;
1438
1439                 if (steps > AHC_MAX_STEPS) {
1440                         kprintf("%s: Infinite loop in critical section\n",
1441                                ahc_name(ahc));
1442                         ahc_dump_card_state(ahc);
1443                         panic("critical section loop");
1444                 }
1445
1446                 steps++;
1447                 if (stepping == FALSE) {
1448
1449                         /*
1450                          * Disable all interrupt sources so that the
1451                          * sequencer will not be stuck by a pausing
1452                          * interrupt condition while we attempt to
1453                          * leave a critical section.
1454                          */
1455                         simode0 = ahc_inb(ahc, SIMODE0);
1456                         ahc_outb(ahc, SIMODE0, 0);
1457                         simode1 = ahc_inb(ahc, SIMODE1);
1458                         if ((ahc->features & AHC_DT) != 0)
1459                                 /*
1460                                  * On DT class controllers, we
1461                                  * use the enhanced busfree logic.
1462                                  * Unfortunately we cannot re-enable
1463                                  * busfree detection within the
1464                                  * current connection, so we must
1465                                  * leave it on while single stepping.
1466                                  */
1467                                 ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
1468                         else
1469                                 ahc_outb(ahc, SIMODE1, 0);
1470                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1471                         ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
1472                         stepping = TRUE;
1473                 }
1474                 if ((ahc->features & AHC_DT) != 0) {
1475                         ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
1476                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1477                 }
1478                 ahc_outb(ahc, HCNTRL, ahc->unpause);
1479                 while (!ahc_is_paused(ahc))
1480                         ahc_delay(200);
1481         }
1482         if (stepping) {
1483                 ahc_outb(ahc, SIMODE0, simode0);
1484                 ahc_outb(ahc, SIMODE1, simode1);
1485                 ahc_outb(ahc, SEQCTL, ahc->seqctl);
1486         }
1487 }
1488
1489 /*
1490  * Clear any pending interrupt status.
1491  */
1492 void
1493 ahc_clear_intstat(struct ahc_softc *ahc)
1494 {
1495         /* Clear any interrupt conditions this may have caused */
1496         ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
1497                                 |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
1498                                 CLRREQINIT);
1499         ahc_flush_device_writes(ahc);
1500         ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
1501         ahc_flush_device_writes(ahc);
1502         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1503         ahc_flush_device_writes(ahc);
1504 }
1505
1506 /**************************** Debugging Routines ******************************/
1507 #ifdef AHC_DEBUG
1508 uint32_t ahc_debug = AHC_DEBUG_OPTS;
1509 #endif
1510
1511 void
1512 ahc_print_scb(struct scb *scb)
1513 {
1514         int i;
1515
1516         struct hardware_scb *hscb = scb->hscb;
1517
1518         kprintf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
1519                (void *)scb,
1520                hscb->control,
1521                hscb->scsiid,
1522                hscb->lun,
1523                hscb->cdb_len);
1524         kprintf("Shared Data: ");
1525         for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
1526                 kprintf("%#02x", hscb->shared_data.cdb[i]);
1527         kprintf("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
1528                 ahc_le32toh(hscb->dataptr),
1529                 ahc_le32toh(hscb->datacnt),
1530                 ahc_le32toh(hscb->sgptr),
1531                 hscb->tag);
1532         if (scb->sg_count > 0) {
1533                 for (i = 0; i < scb->sg_count; i++) {
1534                         kprintf("sg[%d] - Addr 0x%x%x : Length %d\n",
1535                                i,
1536                                (ahc_le32toh(scb->sg_list[i].len) >> 24
1537                                 & SG_HIGH_ADDR_BITS),
1538                                ahc_le32toh(scb->sg_list[i].addr),
1539                                ahc_le32toh(scb->sg_list[i].len));
1540                 }
1541         }
1542 }
1543
1544 /************************* Transfer Negotiation *******************************/
1545 /*
1546  * Allocate per target mode instance (ID we respond to as a target)
1547  * transfer negotiation data structures.
1548  */
1549 static struct ahc_tmode_tstate *
1550 ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
1551 {
1552         struct ahc_tmode_tstate *master_tstate;
1553         struct ahc_tmode_tstate *tstate;
1554         int i;
1555
1556         master_tstate = ahc->enabled_targets[ahc->our_id];
1557         if (channel == 'B') {
1558                 scsi_id += 8;
1559                 master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
1560         }
1561         if (ahc->enabled_targets[scsi_id] != NULL
1562          && ahc->enabled_targets[scsi_id] != master_tstate)
1563                 panic("%s: ahc_alloc_tstate - Target already allocated",
1564                       ahc_name(ahc));
1565         tstate = kmalloc(sizeof(*tstate), M_DEVBUF, M_INTWAIT);
1566
1567         /*
1568          * If we have allocated a master tstate, copy user settings from
1569          * the master tstate (taken from SRAM or the EEPROM) for this
1570          * channel, but reset our current and goal settings to async/narrow
1571          * until an initiator talks to us.
1572          */
1573         if (master_tstate != NULL) {
1574                 memcpy(tstate, master_tstate, sizeof(*tstate));
1575                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
1576                 tstate->ultraenb = 0;
1577                 for (i = 0; i < AHC_NUM_TARGETS; i++) {
1578                         memset(&tstate->transinfo[i].curr, 0,
1579                               sizeof(tstate->transinfo[i].curr));
1580                         memset(&tstate->transinfo[i].goal, 0,
1581                               sizeof(tstate->transinfo[i].goal));
1582                 }
1583         } else
1584                 memset(tstate, 0, sizeof(*tstate));
1585         ahc->enabled_targets[scsi_id] = tstate;
1586         return (tstate);
1587 }
1588
1589 #ifdef AHC_TARGET_MODE
1590 /*
1591  * Free per target mode instance (ID we respond to as a target)
1592  * transfer negotiation data structures.
1593  */
1594 static void
1595 ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
1596 {
1597         struct ahc_tmode_tstate *tstate;
1598
1599         /*
1600          * Don't clean up our "master" tstate.
1601          * It has our default user settings.
1602          */
1603         if (((channel == 'B' && scsi_id == ahc->our_id_b)
1604           || (channel == 'A' && scsi_id == ahc->our_id))
1605          && force == FALSE)
1606                 return;
1607
1608         if (channel == 'B')
1609                 scsi_id += 8;
1610         tstate = ahc->enabled_targets[scsi_id];
1611         if (tstate != NULL)
1612                 kfree(tstate, M_DEVBUF);
1613         ahc->enabled_targets[scsi_id] = NULL;
1614 }
1615 #endif
1616
1617 /*
1618  * Called when we have an active connection to a target on the bus,
1619  * this function finds the nearest syncrate to the input period limited
1620  * by the capabilities of the bus connectivity of and sync settings for
1621  * the target.
1622  */
1623 struct ahc_syncrate *
1624 ahc_devlimited_syncrate(struct ahc_softc *ahc,
1625                         struct ahc_initiator_tinfo *tinfo,
1626                         u_int *period, u_int *ppr_options, role_t role)
1627 {
1628         struct  ahc_transinfo *transinfo;
1629         u_int   maxsync;
1630
1631         if ((ahc->features & AHC_ULTRA2) != 0) {
1632                 if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
1633                  && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
1634                         maxsync = AHC_SYNCRATE_DT;
1635                 } else {
1636                         maxsync = AHC_SYNCRATE_ULTRA;
1637                         /* Can't do DT on an SE bus */
1638                         *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1639                 }
1640         } else if ((ahc->features & AHC_ULTRA) != 0) {
1641                 maxsync = AHC_SYNCRATE_ULTRA;
1642         } else {
1643                 maxsync = AHC_SYNCRATE_FAST;
1644         }
1645         /*
1646          * Never allow a value higher than our current goal
1647          * period otherwise we may allow a target initiated
1648          * negotiation to go above the limit as set by the
1649          * user.  In the case of an initiator initiated
1650          * sync negotiation, we limit based on the user
1651          * setting.  This allows the system to still accept
1652          * incoming negotiations even if target initiated
1653          * negotiation is not performed.
1654          */
1655         if (role == ROLE_TARGET)
1656                 transinfo = &tinfo->user;
1657         else
1658                 transinfo = &tinfo->goal;
1659         *ppr_options &= transinfo->ppr_options;
1660         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
1661                 maxsync = MAX(maxsync, AHC_SYNCRATE_ULTRA2);
1662                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1663         }
1664         if (transinfo->period == 0) {
1665                 *period = 0;
1666                 *ppr_options = 0;
1667                 return (NULL);
1668         }
1669         *period = MAX(*period, transinfo->period);
1670         return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
1671 }
1672
1673 /*
1674  * Look up the valid period to SCSIRATE conversion in our table.
1675  * Return the period and offset that should be sent to the target
1676  * if this was the beginning of an SDTR.
1677  */
1678 struct ahc_syncrate *
1679 ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
1680                   u_int *ppr_options, u_int maxsync)
1681 {
1682         struct ahc_syncrate *syncrate;
1683
1684         if ((ahc->features & AHC_DT) == 0)
1685                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1686
1687         /* Skip all DT only entries if DT is not available */
1688         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
1689          && maxsync < AHC_SYNCRATE_ULTRA2)
1690                 maxsync = AHC_SYNCRATE_ULTRA2;
1691
1692         for (syncrate = &ahc_syncrates[maxsync];
1693              syncrate->rate != NULL;
1694              syncrate++) {
1695
1696                 /*
1697                  * The Ultra2 table doesn't go as low
1698                  * as for the Fast/Ultra cards.
1699                  */
1700                 if ((ahc->features & AHC_ULTRA2) != 0
1701                  && (syncrate->sxfr_u2 == 0))
1702                         break;
1703
1704                 if (*period <= syncrate->period) {
1705                         /*
1706                          * When responding to a target that requests
1707                          * sync, the requested rate may fall between
1708                          * two rates that we can output, but still be
1709                          * a rate that we can receive.  Because of this,
1710                          * we want to respond to the target with
1711                          * the same rate that it sent to us even
1712                          * if the period we use to send data to it
1713                          * is lower.  Only lower the response period
1714                          * if we must.
1715                          */
1716                         if (syncrate == &ahc_syncrates[maxsync])
1717                                 *period = syncrate->period;
1718
1719                         /*
1720                          * At some speeds, we only support
1721                          * ST transfers.
1722                          */
1723                         if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
1724                                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1725                         break;
1726                 }
1727         }
1728
1729         if ((*period == 0)
1730          || (syncrate->rate == NULL)
1731          || ((ahc->features & AHC_ULTRA2) != 0
1732           && (syncrate->sxfr_u2 == 0))) {
1733                 /* Use asynchronous transfers. */
1734                 *period = 0;
1735                 syncrate = NULL;
1736                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
1737         }
1738         return (syncrate);
1739 }
1740
1741 /*
1742  * Convert from an entry in our syncrate table to the SCSI equivalent
1743  * sync "period" factor.
1744  */
1745 u_int
1746 ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
1747 {
1748         struct ahc_syncrate *syncrate;
1749
1750         if ((ahc->features & AHC_ULTRA2) != 0)
1751                 scsirate &= SXFR_ULTRA2;
1752         else
1753                 scsirate &= SXFR;
1754
1755         syncrate = &ahc_syncrates[maxsync];
1756         while (syncrate->rate != NULL) {
1757
1758                 if ((ahc->features & AHC_ULTRA2) != 0) {
1759                         if (syncrate->sxfr_u2 == 0)
1760                                 break;
1761                         else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
1762                                 return (syncrate->period);
1763                 } else if (scsirate == (syncrate->sxfr & SXFR)) {
1764                                 return (syncrate->period);
1765                 }
1766                 syncrate++;
1767         }
1768         return (0); /* async */
1769 }
1770
1771 /*
1772  * Truncate the given synchronous offset to a value the
1773  * current adapter type and syncrate are capable of.
1774  */
1775 void
1776 ahc_validate_offset(struct ahc_softc *ahc,
1777                     struct ahc_initiator_tinfo *tinfo,
1778                     struct ahc_syncrate *syncrate,
1779                     u_int *offset, int wide, role_t role)
1780 {
1781         u_int maxoffset;
1782
1783         /* Limit offset to what we can do */
1784         if (syncrate == NULL) {
1785                 maxoffset = 0;
1786         } else if ((ahc->features & AHC_ULTRA2) != 0) {
1787                 maxoffset = MAX_OFFSET_ULTRA2;
1788         } else {
1789                 if (wide)
1790                         maxoffset = MAX_OFFSET_16BIT;
1791                 else
1792                         maxoffset = MAX_OFFSET_8BIT;
1793         }
1794         *offset = MIN(*offset, maxoffset);
1795         if (tinfo != NULL) {
1796                 if (role == ROLE_TARGET)
1797                         *offset = MIN(*offset, tinfo->user.offset);
1798                 else
1799                         *offset = MIN(*offset, tinfo->goal.offset);
1800         }
1801 }
1802
1803 /*
1804  * Truncate the given transfer width parameter to a value the
1805  * current adapter type is capable of.
1806  */
1807 void
1808 ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
1809                    u_int *bus_width, role_t role)
1810 {
1811         switch (*bus_width) {
1812         default:
1813                 if (ahc->features & AHC_WIDE) {
1814                         /* Respond Wide */
1815                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
1816                         break;
1817                 }
1818                 /* FALLTHROUGH */
1819         case MSG_EXT_WDTR_BUS_8_BIT:
1820                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
1821                 break;
1822         }
1823         if (tinfo != NULL) {
1824                 if (role == ROLE_TARGET)
1825                         *bus_width = MIN(tinfo->user.width, *bus_width);
1826                 else
1827                         *bus_width = MIN(tinfo->goal.width, *bus_width);
1828         }
1829 }
1830
1831 /*
1832  * Update the bitmask of targets for which the controller should
1833  * negotiate with at the next convenient oportunity.  This currently
1834  * means the next time we send the initial identify messages for
1835  * a new transaction.
1836  */
1837 int
1838 ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
1839                        struct ahc_tmode_tstate *tstate,
1840                        struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
1841 {
1842         u_int auto_negotiate_orig;
1843
1844         auto_negotiate_orig = tstate->auto_negotiate;
1845         if (neg_type == AHC_NEG_ALWAYS) {
1846                 /*
1847                  * Force our "current" settings to be
1848                  * unknown so that unless a bus reset
1849                  * occurs the need to renegotiate is
1850                  * recorded persistently.
1851                  */
1852                 if ((ahc->features & AHC_WIDE) != 0)
1853                         tinfo->curr.width = AHC_WIDTH_UNKNOWN;
1854                 tinfo->curr.period = AHC_PERIOD_UNKNOWN;
1855                 tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
1856         }
1857         if (tinfo->curr.period != tinfo->goal.period
1858          || tinfo->curr.width != tinfo->goal.width
1859          || tinfo->curr.offset != tinfo->goal.offset
1860          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
1861          || (neg_type == AHC_NEG_IF_NON_ASYNC
1862           && (tinfo->goal.offset != 0
1863            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
1864            || tinfo->goal.ppr_options != 0)))
1865                 tstate->auto_negotiate |= devinfo->target_mask;
1866         else
1867                 tstate->auto_negotiate &= ~devinfo->target_mask;
1868
1869         return (auto_negotiate_orig != tstate->auto_negotiate);
1870 }
1871
1872 /*
1873  * Update the user/goal/curr tables of synchronous negotiation
1874  * parameters as well as, in the case of a current or active update,
1875  * any data structures on the host controller.  In the case of an
1876  * active update, the specified target is currently talking to us on
1877  * the bus, so the transfer parameter update must take effect
1878  * immediately.
1879  */
1880 void
1881 ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
1882                  struct ahc_syncrate *syncrate, u_int period,
1883                  u_int offset, u_int ppr_options, u_int type, int paused)
1884 {
1885         struct  ahc_initiator_tinfo *tinfo;
1886         struct  ahc_tmode_tstate *tstate;
1887         u_int   old_period;
1888         u_int   old_offset;
1889         u_int   old_ppr;
1890         int     active;
1891         int     update_needed;
1892
1893         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
1894         update_needed = 0;
1895
1896         if (syncrate == NULL) {
1897                 period = 0;
1898                 offset = 0;
1899         }
1900
1901         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
1902                                     devinfo->target, &tstate);
1903
1904         if ((type & AHC_TRANS_USER) != 0) {
1905                 tinfo->user.period = period;
1906                 tinfo->user.offset = offset;
1907                 tinfo->user.ppr_options = ppr_options;
1908         }
1909
1910         if ((type & AHC_TRANS_GOAL) != 0) {
1911                 tinfo->goal.period = period;
1912                 tinfo->goal.offset = offset;
1913                 tinfo->goal.ppr_options = ppr_options;
1914         }
1915
1916         old_period = tinfo->curr.period;
1917         old_offset = tinfo->curr.offset;
1918         old_ppr    = tinfo->curr.ppr_options;
1919
1920         if ((type & AHC_TRANS_CUR) != 0
1921          && (old_period != period
1922           || old_offset != offset
1923           || old_ppr != ppr_options)) {
1924                 u_int   scsirate;
1925
1926                 update_needed++;
1927                 scsirate = tinfo->scsirate;
1928                 if ((ahc->features & AHC_ULTRA2) != 0) {
1929
1930                         scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
1931                         if (syncrate != NULL) {
1932                                 scsirate |= syncrate->sxfr_u2;
1933                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
1934                                         scsirate |= ENABLE_CRC;
1935                                 else
1936                                         scsirate |= SINGLE_EDGE;
1937                         }
1938                 } else {
1939
1940                         scsirate &= ~(SXFR|SOFS);
1941                         /*
1942                          * Ensure Ultra mode is set properly for
1943                          * this target.
1944                          */
1945                         tstate->ultraenb &= ~devinfo->target_mask;
1946                         if (syncrate != NULL) {
1947                                 if (syncrate->sxfr & ULTRA_SXFR) {
1948                                         tstate->ultraenb |=
1949                                                 devinfo->target_mask;
1950                                 }
1951                                 scsirate |= syncrate->sxfr & SXFR;
1952                                 scsirate |= offset & SOFS;
1953                         }
1954                         if (active) {
1955                                 u_int sxfrctl0;
1956
1957                                 sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
1958                                 sxfrctl0 &= ~FAST20;
1959                                 if (tstate->ultraenb & devinfo->target_mask)
1960                                         sxfrctl0 |= FAST20;
1961                                 ahc_outb(ahc, SXFRCTL0, sxfrctl0);
1962                         }
1963                 }
1964                 if (active) {
1965                         ahc_outb(ahc, SCSIRATE, scsirate);
1966                         if ((ahc->features & AHC_ULTRA2) != 0)
1967                                 ahc_outb(ahc, SCSIOFFSET, offset);
1968                 }
1969
1970                 tinfo->scsirate = scsirate;
1971                 tinfo->curr.period = period;
1972                 tinfo->curr.offset = offset;
1973                 tinfo->curr.ppr_options = ppr_options;
1974
1975                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
1976                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
1977                 if (bootverbose) {
1978                         if (offset != 0) {
1979                                 kprintf("%s: target %d synchronous at %sMHz%s, "
1980                                        "offset = 0x%x\n", ahc_name(ahc),
1981                                        devinfo->target, syncrate->rate,
1982                                        (ppr_options & MSG_EXT_PPR_DT_REQ)
1983                                        ? " DT" : "", offset);
1984                         } else {
1985                                 kprintf("%s: target %d using "
1986                                        "asynchronous transfers\n",
1987                                        ahc_name(ahc), devinfo->target);
1988                         }
1989                 }
1990         }
1991
1992         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
1993                                                 tinfo, AHC_NEG_TO_GOAL);
1994
1995         if (update_needed)
1996                 ahc_update_pending_scbs(ahc);
1997 }
1998
1999 /*
2000  * Update the user/goal/curr tables of wide negotiation
2001  * parameters as well as, in the case of a current or active update,
2002  * any data structures on the host controller.  In the case of an
2003  * active update, the specified target is currently talking to us on
2004  * the bus, so the transfer parameter update must take effect
2005  * immediately.
2006  */
2007 void
2008 ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2009               u_int width, u_int type, int paused)
2010 {
2011         struct  ahc_initiator_tinfo *tinfo;
2012         struct  ahc_tmode_tstate *tstate;
2013         u_int   oldwidth;
2014         int     active;
2015         int     update_needed;
2016
2017         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2018         update_needed = 0;
2019         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2020                                     devinfo->target, &tstate);
2021
2022         if ((type & AHC_TRANS_USER) != 0)
2023                 tinfo->user.width = width;
2024
2025         if ((type & AHC_TRANS_GOAL) != 0)
2026                 tinfo->goal.width = width;
2027
2028         oldwidth = tinfo->curr.width;
2029         if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2030                 u_int   scsirate;
2031
2032                 update_needed++;
2033                 scsirate =  tinfo->scsirate;
2034                 scsirate &= ~WIDEXFER;
2035                 if (width == MSG_EXT_WDTR_BUS_16_BIT)
2036                         scsirate |= WIDEXFER;
2037
2038                 tinfo->scsirate = scsirate;
2039
2040                 if (active)
2041                         ahc_outb(ahc, SCSIRATE, scsirate);
2042
2043                 tinfo->curr.width = width;
2044
2045                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2046                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG, NULL);
2047                 if (bootverbose) {
2048                         kprintf("%s: target %d using %dbit transfers\n",
2049                                ahc_name(ahc), devinfo->target,
2050                                8 * (0x01 << width));
2051                 }
2052         }
2053
2054         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2055                                                 tinfo, AHC_NEG_TO_GOAL);
2056         if (update_needed)
2057                 ahc_update_pending_scbs(ahc);
2058 }
2059
2060 /*
2061  * Update the current state of tagged queuing for a given target.
2062  */
2063 void
2064 ahc_set_tags(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2065              ahc_queue_alg alg)
2066 {
2067         ahc_platform_set_tags(ahc, devinfo, alg);
2068         ahc_send_async(ahc, devinfo->channel, devinfo->target,
2069                        devinfo->lun, AC_TRANSFER_NEG, &alg);
2070 }
2071
2072 /*
2073  * When the transfer settings for a connection change, update any
2074  * in-transit SCBs to contain the new data so the hardware will
2075  * be set correctly during future (re)selections.
2076  */
2077 static void
2078 ahc_update_pending_scbs(struct ahc_softc *ahc)
2079 {
2080         struct  scb *pending_scb;
2081         int     pending_scb_count;
2082         int     i;
2083         int     paused;
2084         u_int   saved_scbptr;
2085
2086         /*
2087          * Traverse the pending SCB list and ensure that all of the
2088          * SCBs there have the proper settings.
2089          */
2090         pending_scb_count = 0;
2091         LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2092                 struct ahc_devinfo devinfo;
2093                 struct hardware_scb *pending_hscb;
2094                 struct ahc_initiator_tinfo *tinfo;
2095                 struct ahc_tmode_tstate *tstate;
2096
2097                 ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2098                 tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2099                                             devinfo.our_scsiid,
2100                                             devinfo.target, &tstate);
2101                 pending_hscb = pending_scb->hscb;
2102                 pending_hscb->control &= ~ULTRAENB;
2103                 if ((tstate->ultraenb & devinfo.target_mask) != 0)
2104                         pending_hscb->control |= ULTRAENB;
2105                 pending_hscb->scsirate = tinfo->scsirate;
2106                 pending_hscb->scsioffset = tinfo->curr.offset;
2107                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2108                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2109                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2110                         pending_hscb->control &= ~MK_MESSAGE;
2111                 }
2112                 ahc_sync_scb(ahc, pending_scb,
2113                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2114                 pending_scb_count++;
2115         }
2116
2117         if (pending_scb_count == 0)
2118                 return;
2119
2120         if (ahc_is_paused(ahc)) {
2121                 paused = 1;
2122         } else {
2123                 paused = 0;
2124                 ahc_pause(ahc);
2125         }
2126
2127         saved_scbptr = ahc_inb(ahc, SCBPTR);
2128         /* Ensure that the hscbs down on the card match the new information */
2129         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2130                 struct  hardware_scb *pending_hscb;
2131                 u_int   control;
2132                 u_int   scb_tag;
2133
2134                 ahc_outb(ahc, SCBPTR, i);
2135                 scb_tag = ahc_inb(ahc, SCB_TAG);
2136                 pending_scb = ahc_lookup_scb(ahc, scb_tag);
2137                 if (pending_scb == NULL)
2138                         continue;
2139
2140                 pending_hscb = pending_scb->hscb;
2141                 control = ahc_inb(ahc, SCB_CONTROL);
2142                 control &= ~(ULTRAENB|MK_MESSAGE);
2143                 control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2144                 ahc_outb(ahc, SCB_CONTROL, control);
2145                 ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2146                 ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2147         }
2148         ahc_outb(ahc, SCBPTR, saved_scbptr);
2149
2150         if (paused == 0)
2151                 ahc_unpause(ahc);
2152 }
2153
2154 /**************************** Pathing Information *****************************/
2155 static void
2156 ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2157 {
2158         u_int   saved_scsiid;
2159         role_t  role;
2160         int     our_id;
2161
2162         if (ahc_inb(ahc, SSTAT0) & TARGET)
2163                 role = ROLE_TARGET;
2164         else
2165                 role = ROLE_INITIATOR;
2166
2167         if (role == ROLE_TARGET
2168          && (ahc->features & AHC_MULTI_TID) != 0
2169          && (ahc_inb(ahc, SEQ_FLAGS)
2170            & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2171                 /* We were selected, so pull our id from TARGIDIN */
2172                 our_id = ahc_inb(ahc, TARGIDIN) & OID;
2173         } else if ((ahc->features & AHC_ULTRA2) != 0)
2174                 our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2175         else
2176                 our_id = ahc_inb(ahc, SCSIID) & OID;
2177
2178         saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2179         ahc_compile_devinfo(devinfo,
2180                             our_id,
2181                             SCSIID_TARGET(ahc, saved_scsiid),
2182                             ahc_inb(ahc, SAVED_LUN),
2183                             SCSIID_CHANNEL(ahc, saved_scsiid),
2184                             role);
2185 }
2186
2187 struct ahc_phase_table_entry*
2188 ahc_lookup_phase_entry(int phase)
2189 {
2190         struct ahc_phase_table_entry *entry;
2191         struct ahc_phase_table_entry *last_entry;
2192
2193         /*
2194          * num_phases doesn't include the default entry which
2195          * will be returned if the phase doesn't match.
2196          */
2197         last_entry = &ahc_phase_table[num_phases];
2198         for (entry = ahc_phase_table; entry < last_entry; entry++) {
2199                 if (phase == entry->phase)
2200                         break;
2201         }
2202         return (entry);
2203 }
2204
2205 void
2206 ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2207                     u_int lun, char channel, role_t role)
2208 {
2209         devinfo->our_scsiid = our_id;
2210         devinfo->target = target;
2211         devinfo->lun = lun;
2212         devinfo->target_offset = target;
2213         devinfo->channel = channel;
2214         devinfo->role = role;
2215         if (channel == 'B')
2216                 devinfo->target_offset += 8;
2217         devinfo->target_mask = (0x01 << devinfo->target_offset);
2218 }
2219
2220 void
2221 ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2222 {
2223         kprintf("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2224                devinfo->target, devinfo->lun);
2225 }
2226
2227 static void
2228 ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2229                 struct scb *scb)
2230 {
2231         role_t  role;
2232         int     our_id;
2233
2234         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2235         role = ROLE_INITIATOR;
2236         if ((scb->flags & SCB_TARGET_SCB) != 0)
2237                 role = ROLE_TARGET;
2238         ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2239                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2240 }
2241
2242
2243 /************************ Message Phase Processing ****************************/
2244 static void
2245 ahc_assert_atn(struct ahc_softc *ahc)
2246 {
2247         u_int scsisigo;
2248
2249         scsisigo = ATNO;
2250         if ((ahc->features & AHC_DT) == 0)
2251                 scsisigo |= ahc_inb(ahc, SCSISIGI);
2252         ahc_outb(ahc, SCSISIGO, scsisigo);
2253 }
2254
2255 /*
2256  * When an initiator transaction with the MK_MESSAGE flag either reconnects
2257  * or enters the initial message out phase, we are interrupted.  Fill our
2258  * outgoing message buffer with the appropriate message and beging handing
2259  * the message phase(s) manually.
2260  */
2261 static void
2262 ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2263                            struct scb *scb)
2264 {
2265         /*
2266          * To facilitate adding multiple messages together,
2267          * each routine should increment the index and len
2268          * variables instead of setting them explicitly.
2269          */
2270         ahc->msgout_index = 0;
2271         ahc->msgout_len = 0;
2272
2273         if ((scb->flags & SCB_DEVICE_RESET) == 0
2274          && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2275                 u_int identify_msg;
2276
2277                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2278                 if ((scb->hscb->control & DISCENB) != 0)
2279                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
2280                 ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2281                 ahc->msgout_len++;
2282
2283                 if ((scb->hscb->control & TAG_ENB) != 0) {
2284                         ahc->msgout_buf[ahc->msgout_index++] =
2285                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2286                         ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2287                         ahc->msgout_len += 2;
2288                 }
2289         }
2290
2291         if (scb->flags & SCB_DEVICE_RESET) {
2292                 ahc->msgout_buf[ahc->msgout_index++] = MSG_BUS_DEV_RESET;
2293                 ahc->msgout_len++;
2294                 ahc_print_path(ahc, scb);
2295                 kprintf("Bus Device Reset Message Sent\n");
2296                 /*
2297                  * Clear our selection hardware in advance of
2298                  * the busfree.  We may have an entry in the waiting
2299                  * Q for this target, and we don't want to go about
2300                  * selecting while we handle the busfree and blow it
2301                  * away.
2302                  */
2303                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2304         } else if ((scb->flags & SCB_ABORT) != 0) {
2305                 if ((scb->hscb->control & TAG_ENB) != 0)
2306                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT_TAG;
2307                 else
2308                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT;
2309                 ahc->msgout_len++;
2310                 ahc_print_path(ahc, scb);
2311                 kprintf("Abort%s Message Sent\n",
2312                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2313                 /*
2314                  * Clear our selection hardware in advance of
2315                  * the busfree.  We may have an entry in the waiting
2316                  * Q for this target, and we don't want to go about
2317                  * selecting while we handle the busfree and blow it
2318                  * away.
2319                  */
2320                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2321         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2322                 ahc_build_transfer_msg(ahc, devinfo);
2323         } else {
2324                 kprintf("ahc_intr: AWAITING_MSG for an SCB that "
2325                        "does not have a waiting message\n");
2326                 kprintf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2327                        devinfo->target_mask);
2328                 panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2329                       "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2330                       ahc_inb(ahc, MSG_OUT), scb->flags);
2331         }
2332
2333         /*
2334          * Clear the MK_MESSAGE flag from the SCB so we aren't
2335          * asked to send this message again.
2336          */
2337         ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2338         scb->hscb->control &= ~MK_MESSAGE;
2339         ahc->msgout_index = 0;
2340         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2341 }
2342
2343 /*
2344  * Build an appropriate transfer negotiation message for the
2345  * currently active target.
2346  */
2347 static void
2348 ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2349 {
2350         /*
2351          * We need to initiate transfer negotiations.
2352          * If our current and goal settings are identical,
2353          * we want to renegotiate due to a check condition.
2354          */
2355         struct  ahc_initiator_tinfo *tinfo;
2356         struct  ahc_tmode_tstate *tstate;
2357         struct  ahc_syncrate *rate;
2358         int     dowide;
2359         int     dosync;
2360         int     doppr;
2361         u_int   period;
2362         u_int   ppr_options;
2363         u_int   offset;
2364
2365         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2366                                     devinfo->target, &tstate);
2367         /*
2368          * Filter our period based on the current connection.
2369          * If we can't perform DT transfers on this segment (not in LVD
2370          * mode for instance), then our decision to issue a PPR message
2371          * may change.
2372          */
2373         period = tinfo->goal.period;
2374         offset = tinfo->goal.offset;
2375         ppr_options = tinfo->goal.ppr_options;
2376         /* Target initiated PPR is not allowed in the SCSI spec */
2377         if (devinfo->role == ROLE_TARGET)
2378                 ppr_options = 0;
2379         rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2380                                        &ppr_options, devinfo->role);
2381         dowide = tinfo->curr.width != tinfo->goal.width;
2382         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2383         /*
2384          * Only use PPR if we have options that need it, even if the device
2385          * claims to support it.  There might be an expander in the way
2386          * that doesn't.
2387          */
2388         doppr = ppr_options != 0;
2389
2390         if (!dowide && !dosync && !doppr) {
2391                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
2392                 dosync = tinfo->goal.offset != 0;
2393         }
2394
2395         if (!dowide && !dosync && !doppr) {
2396                 /*
2397                  * Force async with a WDTR message if we have a wide bus,
2398                  * or just issue an SDTR with a 0 offset.
2399                  */
2400                 if ((ahc->features & AHC_WIDE) != 0)
2401                         dowide = 1;
2402                 else
2403                         dosync = 1;
2404
2405                 if (bootverbose) {
2406                         ahc_print_devinfo(ahc, devinfo);
2407                         kprintf("Ensuring async\n");
2408                 }
2409         }
2410
2411         /* Target initiated PPR is not allowed in the SCSI spec */
2412         if (devinfo->role == ROLE_TARGET)
2413                 doppr = 0;
2414
2415         /*
2416          * Both the PPR message and SDTR message require the
2417          * goal syncrate to be limited to what the target device
2418          * is capable of handling (based on whether an LVD->SE
2419          * expander is on the bus), so combine these two cases.
2420          * Regardless, guarantee that if we are using WDTR and SDTR
2421          * messages that WDTR comes first.
2422          */
2423         if (doppr || (dosync && !dowide)) {
2424
2425                 offset = tinfo->goal.offset;
2426                 ahc_validate_offset(ahc, tinfo, rate, &offset,
2427                                     doppr ? tinfo->goal.width
2428                                           : tinfo->curr.width,
2429                                     devinfo->role);
2430                 if (doppr) {
2431                         ahc_construct_ppr(ahc, devinfo, period, offset,
2432                                           tinfo->goal.width, ppr_options);
2433                 } else {
2434                         ahc_construct_sdtr(ahc, devinfo, period, offset);
2435                 }
2436         } else {
2437                 ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
2438         }
2439 }
2440
2441 /*
2442  * Build a synchronous negotiation message in our message
2443  * buffer based on the input parameters.
2444  */
2445 static void
2446 ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2447                    u_int period, u_int offset)
2448 {
2449         if (offset == 0)
2450                 period = AHC_ASYNC_XFER_PERIOD;
2451         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXTENDED;
2452         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_SDTR_LEN;
2453         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_SDTR;
2454         ahc->msgout_buf[ahc->msgout_index++] = period;
2455         ahc->msgout_buf[ahc->msgout_index++] = offset;
2456         ahc->msgout_len += 5;
2457         if (bootverbose) {
2458                 kprintf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
2459                        ahc_name(ahc), devinfo->channel, devinfo->target,
2460                        devinfo->lun, period, offset);
2461         }
2462 }
2463
2464 /*
2465  * Build a wide negotiation message in our message
2466  * buffer based on the input parameters.
2467  */
2468 static void
2469 ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2470                    u_int bus_width)
2471 {
2472         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXTENDED;
2473         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_WDTR_LEN;
2474         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_WDTR;
2475         ahc->msgout_buf[ahc->msgout_index++] = bus_width;
2476         ahc->msgout_len += 4;
2477         if (bootverbose) {
2478                 kprintf("(%s:%c:%d:%d): Sending WDTR %x\n",
2479                        ahc_name(ahc), devinfo->channel, devinfo->target,
2480                        devinfo->lun, bus_width);
2481         }
2482 }
2483
2484 /*
2485  * Build a parallel protocol request message in our message
2486  * buffer based on the input parameters.
2487  */
2488 static void
2489 ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2490                   u_int period, u_int offset, u_int bus_width,
2491                   u_int ppr_options)
2492 {
2493         if (offset == 0)
2494                 period = AHC_ASYNC_XFER_PERIOD;
2495         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXTENDED;
2496         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_PPR_LEN;
2497         ahc->msgout_buf[ahc->msgout_index++] = MSG_EXT_PPR;
2498         ahc->msgout_buf[ahc->msgout_index++] = period;
2499         ahc->msgout_buf[ahc->msgout_index++] = 0;
2500         ahc->msgout_buf[ahc->msgout_index++] = offset;
2501         ahc->msgout_buf[ahc->msgout_index++] = bus_width;
2502         ahc->msgout_buf[ahc->msgout_index++] = ppr_options;
2503         ahc->msgout_len += 8;
2504         if (bootverbose) {
2505                 kprintf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
2506                        "offset %x, ppr_options %x\n", ahc_name(ahc),
2507                        devinfo->channel, devinfo->target, devinfo->lun,
2508                        bus_width, period, offset, ppr_options);
2509         }
2510 }
2511
2512 /*
2513  * Clear any active message state.
2514  */
2515 static void
2516 ahc_clear_msg_state(struct ahc_softc *ahc)
2517 {
2518         ahc->msgout_len = 0;
2519         ahc->msgin_index = 0;
2520         ahc->msg_type = MSG_TYPE_NONE;
2521         if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
2522                 /*
2523                  * The target didn't care to respond to our
2524                  * message request, so clear ATN.
2525                  */
2526                 ahc_outb(ahc, CLRSINT1, CLRATNO);
2527         }
2528         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
2529         ahc_outb(ahc, SEQ_FLAGS2,
2530                  ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
2531 }
2532
2533 static void
2534 ahc_handle_proto_violation(struct ahc_softc *ahc)
2535 {
2536         struct  ahc_devinfo devinfo;
2537         struct  scb *scb;
2538         u_int   scbid;
2539         u_int   seq_flags;
2540         u_int   curphase;
2541         u_int   lastphase;
2542         int     found;
2543
2544         ahc_fetch_devinfo(ahc, &devinfo);
2545         scbid = ahc_inb(ahc, SCB_TAG);
2546         scb = ahc_lookup_scb(ahc, scbid);
2547         seq_flags = ahc_inb(ahc, SEQ_FLAGS);
2548         curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
2549         lastphase = ahc_inb(ahc, LASTPHASE);
2550         if ((seq_flags & NOT_IDENTIFIED) != 0) {
2551
2552                 /*
2553                  * The reconnecting target either did not send an
2554                  * identify message, or did, but we didn't find an SCB
2555                  * to match.
2556                  */
2557                 ahc_print_devinfo(ahc, &devinfo);
2558                 kprintf("Target did not send an IDENTIFY message. "
2559                        "LASTPHASE = 0x%x.\n", lastphase);
2560                 scb = NULL;
2561         } else if (scb == NULL) {
2562                 /*
2563                  * We don't seem to have an SCB active for this
2564                  * transaction.  Print an error and reset the bus.
2565                  */
2566                 ahc_print_devinfo(ahc, &devinfo);
2567                 kprintf("No SCB found during protocol violation\n");
2568                 goto proto_violation_reset;
2569         } else {
2570                 ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
2571                 if ((seq_flags & NO_CDB_SENT) != 0) {
2572                         ahc_print_path(ahc, scb);
2573                         kprintf("No or incomplete CDB sent to device.\n");
2574                 } else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
2575                         /*
2576                          * The target never bothered to provide status to
2577                          * us prior to completing the command.  Since we don't
2578                          * know the disposition of this command, we must attempt
2579                          * to abort it.  Assert ATN and prepare to send an abort
2580                          * message.
2581                          */
2582                         ahc_print_path(ahc, scb);
2583                         kprintf("Completed command without status.\n");
2584                 } else {
2585                         ahc_print_path(ahc, scb);
2586                         kprintf("Unknown protocol violation.\n");
2587                         ahc_dump_card_state(ahc);
2588                 }
2589         }
2590         if ((lastphase & ~P_DATAIN_DT) == 0
2591          || lastphase == P_COMMAND) {
2592 proto_violation_reset:
2593                 /*
2594                  * Target either went directly to data/command
2595                  * phase or didn't respond to our ATN.
2596                  * The only safe thing to do is to blow
2597                  * it away with a bus reset.
2598                  */
2599                 found = ahc_reset_channel(ahc, 'A', TRUE);
2600                 kprintf("%s: Issued Channel %c Bus Reset. "
2601                        "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
2602         } else {
2603                 /*
2604                  * Leave the selection hardware off in case
2605                  * this abort attempt will affect yet to
2606                  * be sent commands.
2607                  */
2608                 ahc_outb(ahc, SCSISEQ,
2609                          ahc_inb(ahc, SCSISEQ) & ~ENSELO);
2610                 ahc_assert_atn(ahc);
2611                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
2612                 if (scb == NULL) {
2613                         ahc_print_devinfo(ahc, &devinfo);
2614                         ahc->msgout_buf[0] = MSG_ABORT_TASK;
2615                         ahc->msgout_len = 1;
2616                         ahc->msgout_index = 0;
2617                         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2618                 } else {
2619                         ahc_print_path(ahc, scb);
2620                         scb->flags |= SCB_ABORT;
2621                 }
2622                 kprintf("Protocol violation %s.  Attempting to abort.\n",
2623                        ahc_lookup_phase_entry(curphase)->phasemsg);
2624         }
2625 }
2626
2627 /*
2628  * Manual message loop handler.
2629  */
2630 static void
2631 ahc_handle_message_phase(struct ahc_softc *ahc)
2632 {
2633         struct  ahc_devinfo devinfo;
2634         u_int   bus_phase;
2635         int     end_session;
2636
2637         ahc_fetch_devinfo(ahc, &devinfo);
2638         end_session = FALSE;
2639         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
2640
2641 reswitch:
2642         switch (ahc->msg_type) {
2643         case MSG_TYPE_INITIATOR_MSGOUT:
2644         {
2645                 int lastbyte;
2646                 int phasemis;
2647                 int msgdone;
2648
2649                 if (ahc->msgout_len == 0)
2650                         panic("HOST_MSG_LOOP interrupt with no active message");
2651
2652 #ifdef AHC_DEBUG
2653                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2654                         ahc_print_devinfo(ahc, &devinfo);
2655                         kprintf("INITIATOR_MSG_OUT");
2656                 }
2657 #endif
2658                 phasemis = bus_phase != P_MESGOUT;
2659                 if (phasemis) {
2660 #ifdef AHC_DEBUG
2661                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2662                                 kprintf(" PHASEMIS %s\n",
2663                                        ahc_lookup_phase_entry(bus_phase)
2664                                                              ->phasemsg);
2665                         }
2666 #endif
2667                         if (bus_phase == P_MESGIN) {
2668                                 /*
2669                                  * Change gears and see if
2670                                  * this messages is of interest to
2671                                  * us or should be passed back to
2672                                  * the sequencer.
2673                                  */
2674                                 ahc_outb(ahc, CLRSINT1, CLRATNO);
2675                                 ahc->send_msg_perror = FALSE;
2676                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
2677                                 ahc->msgin_index = 0;
2678                                 goto reswitch;
2679                         }
2680                         end_session = TRUE;
2681                         break;
2682                 }
2683
2684                 if (ahc->send_msg_perror) {
2685                         ahc_outb(ahc, CLRSINT1, CLRATNO);
2686                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2687 #ifdef AHC_DEBUG
2688                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2689                                 kprintf(" byte 0x%x\n", ahc->send_msg_perror);
2690 #endif
2691                         ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
2692                         break;
2693                 }
2694
2695                 msgdone = ahc->msgout_index == ahc->msgout_len;
2696                 if (msgdone) {
2697                         /*
2698                          * The target has requested a retry.
2699                          * Re-assert ATN, reset our message index to
2700                          * 0, and try again.
2701                          */
2702                         ahc->msgout_index = 0;
2703                         ahc_assert_atn(ahc);
2704                 }
2705
2706                 lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
2707                 if (lastbyte) {
2708                         /* Last byte is signified by dropping ATN */
2709                         ahc_outb(ahc, CLRSINT1, CLRATNO);
2710                 }
2711
2712                 /*
2713                  * Clear our interrupt status and present
2714                  * the next byte on the bus.
2715                  */
2716                 ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2717 #ifdef AHC_DEBUG
2718                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2719                         kprintf(" byte 0x%x\n",
2720                                ahc->msgout_buf[ahc->msgout_index]);
2721 #endif
2722                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
2723                 break;
2724         }
2725         case MSG_TYPE_INITIATOR_MSGIN:
2726         {
2727                 int phasemis;
2728                 int message_done;
2729
2730 #ifdef AHC_DEBUG
2731                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2732                         ahc_print_devinfo(ahc, &devinfo);
2733                         kprintf("INITIATOR_MSG_IN");
2734                 }
2735 #endif
2736                 phasemis = bus_phase != P_MESGIN;
2737                 if (phasemis) {
2738 #ifdef AHC_DEBUG
2739                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2740                                 kprintf(" PHASEMIS %s\n",
2741                                        ahc_lookup_phase_entry(bus_phase)
2742                                                              ->phasemsg);
2743                         }
2744 #endif
2745                         ahc->msgin_index = 0;
2746                         if (bus_phase == P_MESGOUT
2747                          && (ahc->send_msg_perror == TRUE
2748                           || (ahc->msgout_len != 0
2749                            && ahc->msgout_index == 0))) {
2750                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2751                                 goto reswitch;
2752                         }
2753                         end_session = TRUE;
2754                         break;
2755                 }
2756
2757                 /* Pull the byte in without acking it */
2758                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
2759 #ifdef AHC_DEBUG
2760                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
2761                         kprintf(" byte 0x%x\n",
2762                                ahc->msgin_buf[ahc->msgin_index]);
2763 #endif
2764
2765                 message_done = ahc_parse_msg(ahc, &devinfo);
2766
2767                 if (message_done) {
2768                         /*
2769                          * Clear our incoming message buffer in case there
2770                          * is another message following this one.
2771                          */
2772                         ahc->msgin_index = 0;
2773
2774                         /*
2775                          * If this message illicited a response,
2776                          * assert ATN so the target takes us to the
2777                          * message out phase.
2778                          */
2779                         if (ahc->msgout_len != 0) {
2780 #ifdef AHC_DEBUG
2781                                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
2782                                         ahc_print_devinfo(ahc, &devinfo);
2783                                         kprintf("Asserting ATN for response\n");
2784                                 }
2785 #endif
2786                                 ahc_assert_atn(ahc);
2787                         }
2788                 } else
2789                         ahc->msgin_index++;
2790
2791                 if (message_done == MSGLOOP_TERMINATED) {
2792                         end_session = TRUE;
2793                 } else {
2794                         /* Ack the byte */
2795                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
2796                         ahc_inb(ahc, SCSIDATL);
2797                 }
2798                 break;
2799         }
2800         case MSG_TYPE_TARGET_MSGIN:
2801         {
2802                 int msgdone;
2803                 int msgout_request;
2804
2805                 if (ahc->msgout_len == 0)
2806                         panic("Target MSGIN with no active message");
2807
2808                 /*
2809                  * If we interrupted a mesgout session, the initiator
2810                  * will not know this until our first REQ.  So, we
2811                  * only honor mesgout requests after we've sent our
2812                  * first byte.
2813                  */
2814                 if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
2815                  && ahc->msgout_index > 0)
2816                         msgout_request = TRUE;
2817                 else
2818                         msgout_request = FALSE;
2819
2820                 if (msgout_request) {
2821
2822                         /*
2823                          * Change gears and see if
2824                          * this messages is of interest to
2825                          * us or should be passed back to
2826                          * the sequencer.
2827                          */
2828                         ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
2829                         ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
2830                         ahc->msgin_index = 0;
2831                         /* Dummy read to REQ for first byte */
2832                         ahc_inb(ahc, SCSIDATL);
2833                         ahc_outb(ahc, SXFRCTL0,
2834                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2835                         break;
2836                 }
2837
2838                 msgdone = ahc->msgout_index == ahc->msgout_len;
2839                 if (msgdone) {
2840                         ahc_outb(ahc, SXFRCTL0,
2841                                  ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
2842                         end_session = TRUE;
2843                         break;
2844                 }
2845
2846                 /*
2847                  * Present the next byte on the bus.
2848                  */
2849                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2850                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
2851                 break;
2852         }
2853         case MSG_TYPE_TARGET_MSGOUT:
2854         {
2855                 int lastbyte;
2856                 int msgdone;
2857
2858                 /*
2859                  * The initiator signals that this is
2860                  * the last byte by dropping ATN.
2861                  */
2862                 lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
2863
2864                 /*
2865                  * Read the latched byte, but turn off SPIOEN first
2866                  * so that we don't inadvertently cause a REQ for the
2867                  * next byte.
2868                  */
2869                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
2870                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
2871                 msgdone = ahc_parse_msg(ahc, &devinfo);
2872                 if (msgdone == MSGLOOP_TERMINATED) {
2873                         /*
2874                          * The message is *really* done in that it caused
2875                          * us to go to bus free.  The sequencer has already
2876                          * been reset at this point, so pull the ejection
2877                          * handle.
2878                          */
2879                         return;
2880                 }
2881
2882                 ahc->msgin_index++;
2883
2884                 /*
2885                  * XXX Read spec about initiator dropping ATN too soon
2886                  *     and use msgdone to detect it.
2887                  */
2888                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
2889                         ahc->msgin_index = 0;
2890
2891                         /*
2892                          * If this message illicited a response, transition
2893                          * to the Message in phase and send it.
2894                          */
2895                         if (ahc->msgout_len != 0) {
2896                                 ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
2897                                 ahc_outb(ahc, SXFRCTL0,
2898                                          ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2899                                 ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
2900                                 ahc->msgin_index = 0;
2901                                 break;
2902                         }
2903                 }
2904
2905                 if (lastbyte)
2906                         end_session = TRUE;
2907                 else {
2908                         /* Ask for the next byte. */
2909                         ahc_outb(ahc, SXFRCTL0,
2910                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
2911                 }
2912
2913                 break;
2914         }
2915         default:
2916                 panic("Unknown REQINIT message type");
2917         }
2918
2919         if (end_session) {
2920                 ahc_clear_msg_state(ahc);
2921                 ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
2922         } else
2923                 ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
2924 }
2925
2926 /*
2927  * See if we sent a particular extended message to the target.
2928  * If "full" is true, return true only if the target saw the full
2929  * message.  If "full" is false, return true if the target saw at
2930  * least the first byte of the message.
2931  */
2932 static int
2933 ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
2934 {
2935         int found;
2936         u_int index;
2937
2938         found = FALSE;
2939         index = 0;
2940
2941         while (index < ahc->msgout_len) {
2942                 if (ahc->msgout_buf[index] == MSG_EXTENDED) {
2943                         u_int end_index;
2944
2945                         end_index = index + 1 + ahc->msgout_buf[index + 1];
2946                         if (ahc->msgout_buf[index+2] == msgval
2947                          && type == AHCMSG_EXT) {
2948
2949                                 if (full) {
2950                                         if (ahc->msgout_index > end_index)
2951                                                 found = TRUE;
2952                                 } else if (ahc->msgout_index > index)
2953                                         found = TRUE;
2954                         }
2955                         index = end_index;
2956                 } else if (ahc->msgout_buf[index] >= MSG_SIMPLE_TASK
2957                         && ahc->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
2958
2959                         /* Skip tag type and tag id or residue param*/
2960                         index += 2;
2961                 } else {
2962                         /* Single byte message */
2963                         if (type == AHCMSG_1B
2964                          && ahc->msgout_buf[index] == msgval
2965                          && ahc->msgout_index > index)
2966                                 found = TRUE;
2967                         index++;
2968                 }
2969
2970                 if (found)
2971                         break;
2972         }
2973         return (found);
2974 }
2975
2976 /*
2977  * Wait for a complete incoming message, parse it, and respond accordingly.
2978  */
2979 static int
2980 ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2981 {
2982         struct  ahc_initiator_tinfo *tinfo;
2983         struct  ahc_tmode_tstate *tstate;
2984         int     reject;
2985         int     done;
2986         int     response;
2987         u_int   targ_scsirate;
2988
2989         done = MSGLOOP_IN_PROG;
2990         response = FALSE;
2991         reject = FALSE;
2992         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2993                                     devinfo->target, &tstate);
2994         targ_scsirate = tinfo->scsirate;
2995
2996         /*
2997          * Parse as much of the message as is available,
2998          * rejecting it if we don't support it.  When
2999          * the entire message is available and has been
3000          * handled, return MSGLOOP_MSGCOMPLETE, indicating
3001          * that we have parsed an entire message.
3002          *
3003          * In the case of extended messages, we accept the length
3004          * byte outright and perform more checking once we know the
3005          * extended message type.
3006          */
3007         switch (ahc->msgin_buf[0]) {
3008         case MSG_DISCONNECT:
3009         case MSG_SAVEDATAPOINTER:
3010         case MSG_CMDCOMPLETE:
3011         case MSG_RESTOREPOINTERS:
3012         case MSG_IGN_WIDE_RESIDUE:
3013                 /*
3014                  * End our message loop as these are messages
3015                  * the sequencer handles on its own.
3016                  */
3017                 done = MSGLOOP_TERMINATED;
3018                 break;
3019         case MSG_MESSAGE_REJECT:
3020                 response = ahc_handle_msg_reject(ahc, devinfo);
3021                 /* FALLTHROUGH */
3022         case MSG_NOOP:
3023                 done = MSGLOOP_MSGCOMPLETE;
3024                 break;
3025         case MSG_EXTENDED:
3026         {
3027                 /* Wait for enough of the message to begin validation */
3028                 if (ahc->msgin_index < 2)
3029                         break;
3030                 switch (ahc->msgin_buf[2]) {
3031                 case MSG_EXT_SDTR:
3032                 {
3033                         struct   ahc_syncrate *syncrate;
3034                         u_int    period;
3035                         u_int    ppr_options;
3036                         u_int    offset;
3037                         u_int    saved_offset;
3038
3039                         if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3040                                 reject = TRUE;
3041                                 break;
3042                         }
3043
3044                         /*
3045                          * Wait until we have both args before validating
3046                          * and acting on this message.
3047                          *
3048                          * Add one to MSG_EXT_SDTR_LEN to account for
3049                          * the extended message preamble.
3050                          */
3051                         if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3052                                 break;
3053
3054                         period = ahc->msgin_buf[3];
3055                         ppr_options = 0;
3056                         saved_offset = offset = ahc->msgin_buf[4];
3057                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3058                                                            &ppr_options,
3059                                                            devinfo->role);
3060                         ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3061                                             targ_scsirate & WIDEXFER,
3062                                             devinfo->role);
3063                         if (bootverbose) {
3064                                 kprintf("(%s:%c:%d:%d): Received "
3065                                        "SDTR period %x, offset %x\n\t"
3066                                        "Filtered to period %x, offset %x\n",
3067                                        ahc_name(ahc), devinfo->channel,
3068                                        devinfo->target, devinfo->lun,
3069                                        ahc->msgin_buf[3], saved_offset,
3070                                        period, offset);
3071                         }
3072                         ahc_set_syncrate(ahc, devinfo,
3073                                          syncrate, period,
3074                                          offset, ppr_options,
3075                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3076                                          /*paused*/TRUE);
3077
3078                         /*
3079                          * See if we initiated Sync Negotiation
3080                          * and didn't have to fall down to async
3081                          * transfers.
3082                          */
3083                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3084                                 /* We started it */
3085                                 if (saved_offset != offset) {
3086                                         /* Went too low - force async */
3087                                         reject = TRUE;
3088                                 }
3089                         } else {
3090                                 /*
3091                                  * Send our own SDTR in reply
3092                                  */
3093                                 if (bootverbose
3094                                  && devinfo->role == ROLE_INITIATOR) {
3095                                         kprintf("(%s:%c:%d:%d): Target "
3096                                                "Initiated SDTR\n",
3097                                                ahc_name(ahc), devinfo->channel,
3098                                                devinfo->target, devinfo->lun);
3099                                 }
3100                                 ahc->msgout_index = 0;
3101                                 ahc->msgout_len = 0;
3102                                 ahc_construct_sdtr(ahc, devinfo,
3103                                                    period, offset);
3104                                 ahc->msgout_index = 0;
3105                                 response = TRUE;
3106                         }
3107                         done = MSGLOOP_MSGCOMPLETE;
3108                         break;
3109                 }
3110                 case MSG_EXT_WDTR:
3111                 {
3112                         u_int bus_width;
3113                         u_int saved_width;
3114                         u_int sending_reply;
3115
3116                         sending_reply = FALSE;
3117                         if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3118                                 reject = TRUE;
3119                                 break;
3120                         }
3121
3122                         /*
3123                          * Wait until we have our arg before validating
3124                          * and acting on this message.
3125                          *
3126                          * Add one to MSG_EXT_WDTR_LEN to account for
3127                          * the extended message preamble.
3128                          */
3129                         if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3130                                 break;
3131
3132                         bus_width = ahc->msgin_buf[3];
3133                         saved_width = bus_width;
3134                         ahc_validate_width(ahc, tinfo, &bus_width,
3135                                            devinfo->role);
3136                         if (bootverbose) {
3137                                 kprintf("(%s:%c:%d:%d): Received WDTR "
3138                                        "%x filtered to %x\n",
3139                                        ahc_name(ahc), devinfo->channel,
3140                                        devinfo->target, devinfo->lun,
3141                                        saved_width, bus_width);
3142                         }
3143
3144                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3145                                 /*
3146                                  * Don't send a WDTR back to the
3147                                  * target, since we asked first.
3148                                  * If the width went higher than our
3149                                  * request, reject it.
3150                                  */
3151                                 if (saved_width > bus_width) {
3152                                         reject = TRUE;
3153                                         kprintf("(%s:%c:%d:%d): requested %dBit "
3154                                                "transfers.  Rejecting...\n",
3155                                                ahc_name(ahc), devinfo->channel,
3156                                                devinfo->target, devinfo->lun,
3157                                                8 * (0x01 << bus_width));
3158                                         bus_width = 0;
3159                                 }
3160                         } else {
3161                                 /*
3162                                  * Send our own WDTR in reply
3163                                  */
3164                                 if (bootverbose
3165                                  && devinfo->role == ROLE_INITIATOR) {
3166                                         kprintf("(%s:%c:%d:%d): Target "
3167                                                "Initiated WDTR\n",
3168                                                ahc_name(ahc), devinfo->channel,
3169                                                devinfo->target, devinfo->lun);
3170                                 }
3171                                 ahc->msgout_index = 0;
3172                                 ahc->msgout_len = 0;
3173                                 ahc_construct_wdtr(ahc, devinfo, bus_width);
3174                                 ahc->msgout_index = 0;
3175                                 response = TRUE;
3176                                 sending_reply = TRUE;
3177                         }
3178                         /*
3179                          * After a wide message, we are async, but
3180                          * some devices don't seem to honor this portion
3181                          * of the spec.  Force a renegotiation of the
3182                          * sync component of our transfer agreement even
3183                          * if our goal is async.  By updating our width
3184                          * after forcing the negotiation, we avoid
3185                          * renegotiating for width.
3186                          */
3187                         ahc_update_neg_request(ahc, devinfo, tstate,
3188                                                tinfo, AHC_NEG_ALWAYS);
3189                         ahc_set_width(ahc, devinfo, bus_width,
3190                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3191                                       /*paused*/TRUE);
3192                         if (sending_reply == FALSE && reject == FALSE) {
3193
3194                                 /*
3195                                  * We will always have an SDTR to send.
3196                                  */
3197                                 ahc->msgout_index = 0;
3198                                 ahc->msgout_len = 0;
3199                                 ahc_build_transfer_msg(ahc, devinfo);
3200                                 ahc->msgout_index = 0;
3201                                 response = TRUE;
3202                         }
3203                         done = MSGLOOP_MSGCOMPLETE;
3204                         break;
3205                 }
3206                 case MSG_EXT_PPR:
3207                 {
3208                         struct  ahc_syncrate *syncrate;
3209                         u_int   period;
3210                         u_int   offset;
3211                         u_int   bus_width;
3212                         u_int   ppr_options;
3213                         u_int   saved_width;
3214                         u_int   saved_offset;
3215                         u_int   saved_ppr_options;
3216
3217                         if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3218                                 reject = TRUE;
3219                                 break;
3220                         }
3221
3222                         /*
3223                          * Wait until we have all args before validating
3224                          * and acting on this message.
3225                          *
3226                          * Add one to MSG_EXT_PPR_LEN to account for
3227                          * the extended message preamble.
3228                          */
3229                         if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3230                                 break;
3231
3232                         period = ahc->msgin_buf[3];
3233                         offset = ahc->msgin_buf[5];
3234                         bus_width = ahc->msgin_buf[6];
3235                         saved_width = bus_width;
3236                         ppr_options = ahc->msgin_buf[7];
3237                         /*
3238                          * According to the spec, a DT only
3239                          * period factor with no DT option
3240                          * set implies async.
3241                          */
3242                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3243                          && period == 9)
3244                                 offset = 0;
3245                         saved_ppr_options = ppr_options;
3246                         saved_offset = offset;
3247
3248                         /*
3249                          * Mask out any options we don't support
3250                          * on any controller.  Transfer options are
3251                          * only available if we are negotiating wide.
3252                          */
3253                         ppr_options &= MSG_EXT_PPR_DT_REQ;
3254                         if (bus_width == 0)
3255                                 ppr_options = 0;
3256
3257                         ahc_validate_width(ahc, tinfo, &bus_width,
3258                                            devinfo->role);
3259                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3260                                                            &ppr_options,
3261                                                            devinfo->role);
3262                         ahc_validate_offset(ahc, tinfo, syncrate,
3263                                             &offset, bus_width,
3264                                             devinfo->role);
3265
3266                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, TRUE)) {
3267                                 /*
3268                                  * If we are unable to do any of the
3269                                  * requested options (we went too low),
3270                                  * then we'll have to reject the message.
3271                                  */
3272                                 if (saved_width > bus_width
3273                                  || saved_offset != offset
3274                                  || saved_ppr_options != ppr_options) {
3275                                         reject = TRUE;
3276                                         period = 0;
3277                                         offset = 0;
3278                                         bus_width = 0;
3279                                         ppr_options = 0;
3280                                         syncrate = NULL;
3281                                 }
3282                         } else {
3283                                 if (devinfo->role != ROLE_TARGET)
3284                                         kprintf("(%s:%c:%d:%d): Target "
3285                                                "Initiated PPR\n",
3286                                                ahc_name(ahc), devinfo->channel,
3287                                                devinfo->target, devinfo->lun);
3288                                 else
3289                                         kprintf("(%s:%c:%d:%d): Initiator "
3290                                                "Initiated PPR\n",
3291                                                ahc_name(ahc), devinfo->channel,
3292                                                devinfo->target, devinfo->lun);
3293                                 ahc->msgout_index = 0;
3294                                 ahc->msgout_len = 0;
3295                                 ahc_construct_ppr(ahc, devinfo, period, offset,
3296                                                   bus_width, ppr_options);
3297                                 ahc->msgout_index = 0;
3298                                 response = TRUE;
3299                         }
3300                         if (bootverbose) {
3301                                 kprintf("(%s:%c:%d:%d): Received PPR width %x, "
3302                                        "period %x, offset %x,options %x\n"
3303                                        "\tFiltered to width %x, period %x, "
3304                                        "offset %x, options %x\n",
3305                                        ahc_name(ahc), devinfo->channel,
3306                                        devinfo->target, devinfo->lun,
3307                                        saved_width, ahc->msgin_buf[3],
3308                                        saved_offset, saved_ppr_options,
3309                                        bus_width, period, offset, ppr_options);
3310                         }
3311                         ahc_set_width(ahc, devinfo, bus_width,
3312                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3313                                       /*paused*/TRUE);
3314                         ahc_set_syncrate(ahc, devinfo,
3315                                          syncrate, period,
3316                                          offset, ppr_options,
3317                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3318                                          /*paused*/TRUE);
3319                         done = MSGLOOP_MSGCOMPLETE;
3320                         break;
3321                 }
3322                 default:
3323                         /* Unknown extended message.  Reject it. */
3324                         reject = TRUE;
3325                         break;
3326                 }
3327                 break;
3328         }
3329 #ifdef AHC_TARGET_MODE
3330         case MSG_BUS_DEV_RESET:
3331                 ahc_handle_devreset(ahc, devinfo,
3332                                     CAM_BDR_SENT,
3333                                     "Bus Device Reset Received",
3334                                     /*verbose_level*/0);
3335                 ahc_restart(ahc);
3336                 done = MSGLOOP_TERMINATED;
3337                 break;
3338         case MSG_ABORT_TAG:
3339         case MSG_ABORT:
3340         case MSG_CLEAR_QUEUE:
3341         {
3342                 int tag;
3343
3344                 /* Target mode messages */
3345                 if (devinfo->role != ROLE_TARGET) {
3346                         reject = TRUE;
3347                         break;
3348                 }
3349                 tag = SCB_LIST_NULL;
3350                 if (ahc->msgin_buf[0] == MSG_ABORT_TAG)
3351                         tag = ahc_inb(ahc, INITIATOR_TAG);
3352                 ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3353                                devinfo->lun, tag, ROLE_TARGET,
3354                                CAM_REQ_ABORTED);
3355
3356                 tstate = ahc->enabled_targets[devinfo->our_scsiid];
3357                 if (tstate != NULL) {
3358                         struct ahc_tmode_lstate* lstate;
3359
3360                         lstate = tstate->enabled_luns[devinfo->lun];
3361                         if (lstate != NULL) {
3362                                 ahc_queue_lstate_event(ahc, lstate,
3363                                                        devinfo->our_scsiid,
3364                                                        ahc->msgin_buf[0],
3365                                                        /*arg*/tag);
3366                                 ahc_send_lstate_events(ahc, lstate);
3367                         }
3368                 }
3369                 ahc_restart(ahc);
3370                 done = MSGLOOP_TERMINATED;
3371                 break;
3372         }
3373 #endif
3374         case MSG_TERM_IO_PROC:
3375         default:
3376                 reject = TRUE;
3377                 break;
3378         }
3379
3380         if (reject) {
3381                 /*
3382                  * Setup to reject the message.
3383                  */
3384                 ahc->msgout_index = 0;
3385                 ahc->msgout_len = 1;
3386                 ahc->msgout_buf[0] = MSG_MESSAGE_REJECT;
3387                 done = MSGLOOP_MSGCOMPLETE;
3388                 response = TRUE;
3389         }
3390
3391         if (done != MSGLOOP_IN_PROG && !response)
3392                 /* Clear the outgoing message buffer */
3393                 ahc->msgout_len = 0;
3394
3395         return (done);
3396 }
3397
3398 /*
3399  * Process a message reject message.
3400  */
3401 static int
3402 ahc_handle_msg_reject(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3403 {
3404         /*
3405          * What we care about here is if we had an
3406          * outstanding SDTR or WDTR message for this
3407          * target.  If we did, this is a signal that
3408          * the target is refusing negotiation.
3409          */
3410         struct scb *scb;
3411         struct ahc_initiator_tinfo *tinfo;
3412         struct ahc_tmode_tstate *tstate;
3413         u_int scb_index;
3414         u_int last_msg;
3415         int   response = 0;
3416
3417         scb_index = ahc_inb(ahc, SCB_TAG);
3418         scb = ahc_lookup_scb(ahc, scb_index);
3419         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel,
3420                                     devinfo->our_scsiid,
3421                                     devinfo->target, &tstate);
3422         /* Might be necessary */
3423         last_msg = ahc_inb(ahc, LAST_MSG);
3424
3425         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
3426                 /*
3427                  * Target does not support the PPR message.
3428                  * Attempt to negotiate SPI-2 style.
3429                  */
3430                 if (bootverbose) {
3431                         kprintf("(%s:%c:%d:%d): PPR Rejected. "
3432                                "Trying WDTR/SDTR\n",
3433                                ahc_name(ahc), devinfo->channel,
3434                                devinfo->target, devinfo->lun);
3435                 }
3436                 tinfo->goal.ppr_options = 0;
3437                 tinfo->curr.transport_version = 2;
3438                 tinfo->goal.transport_version = 2;
3439                 ahc->msgout_index = 0;
3440                 ahc->msgout_len = 0;
3441                 ahc_build_transfer_msg(ahc, devinfo);
3442                 ahc->msgout_index = 0;
3443                 response = 1;
3444         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
3445
3446                 /* note 8bit xfers */
3447                 kprintf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
3448                        "8bit transfers\n", ahc_name(ahc),
3449                        devinfo->channel, devinfo->target, devinfo->lun);
3450                 ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3451                               AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3452                               /*paused*/TRUE);
3453                 /*
3454                  * No need to clear the sync rate.  If the target
3455                  * did not accept the command, our syncrate is
3456                  * unaffected.  If the target started the negotiation,
3457                  * but rejected our response, we already cleared the
3458                  * sync rate before sending our WDTR.
3459                  */
3460                 if (tinfo->goal.offset != tinfo->curr.offset) {
3461
3462                         /* Start the sync negotiation */
3463                         ahc->msgout_index = 0;
3464                         ahc->msgout_len = 0;
3465                         ahc_build_transfer_msg(ahc, devinfo);
3466                         ahc->msgout_index = 0;
3467                         response = 1;
3468                 }
3469         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
3470                 /* note asynch xfers and clear flag */
3471                 ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL, /*period*/0,
3472                                  /*offset*/0, /*ppr_options*/0,
3473                                  AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3474                                  /*paused*/TRUE);
3475                 kprintf("(%s:%c:%d:%d): refuses synchronous negotiation. "
3476                        "Using asynchronous transfers\n",
3477                        ahc_name(ahc), devinfo->channel,
3478                        devinfo->target, devinfo->lun);
3479         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
3480                 int tag_type;
3481                 int mask;
3482
3483                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
3484
3485                 if (tag_type == MSG_SIMPLE_TASK) {
3486                         kprintf("(%s:%c:%d:%d): refuses tagged commands.  "
3487                                "Performing non-tagged I/O\n", ahc_name(ahc),
3488                                devinfo->channel, devinfo->target, devinfo->lun);
3489                         ahc_set_tags(ahc, devinfo, AHC_QUEUE_NONE);
3490                         mask = ~0x23;
3491                 } else {
3492                         kprintf("(%s:%c:%d:%d): refuses %s tagged commands.  "
3493                                "Performing simple queue tagged I/O only\n",
3494                                ahc_name(ahc), devinfo->channel, devinfo->target,
3495                                devinfo->lun, tag_type == MSG_ORDERED_TASK
3496                                ? "ordered" : "head of queue");
3497                         ahc_set_tags(ahc, devinfo, AHC_QUEUE_BASIC);
3498                         mask = ~0x03;
3499                 }
3500
3501                 /*
3502                  * Resend the identify for this CCB as the target
3503                  * may believe that the selection is invalid otherwise.
3504                  */
3505                 ahc_outb(ahc, SCB_CONTROL,
3506                          ahc_inb(ahc, SCB_CONTROL) & mask);
3507                 scb->hscb->control &= mask;
3508                 ahc_set_transaction_tag(scb, /*enabled*/FALSE,
3509                                         /*type*/MSG_SIMPLE_TASK);
3510                 ahc_outb(ahc, MSG_OUT, MSG_IDENTIFYFLAG);
3511                 ahc_assert_atn(ahc);
3512
3513                 /*
3514                  * This transaction is now at the head of
3515                  * the untagged queue for this target.
3516                  */
3517                 if ((ahc->flags & AHC_SCB_BTT) == 0) {
3518                         struct scb_tailq *untagged_q;
3519
3520                         untagged_q =
3521                             &(ahc->untagged_queues[devinfo->target_offset]);
3522                         TAILQ_INSERT_HEAD(untagged_q, scb, links.tqe);
3523                         scb->flags |= SCB_UNTAGGEDQ;
3524                 }
3525                 ahc_busy_tcl(ahc, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
3526                              scb->hscb->tag);
3527
3528                 /*
3529                  * Requeue all tagged commands for this target
3530                  * currently in our posession so they can be
3531                  * converted to untagged commands.
3532                  */
3533                 ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
3534                                    SCB_GET_CHANNEL(ahc, scb),
3535                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
3536                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
3537                                    SEARCH_COMPLETE);
3538         } else {
3539                 /*
3540                  * Otherwise, we ignore it.
3541                  */
3542                 kprintf("%s:%c:%d: Message reject for %x -- ignored\n",
3543                        ahc_name(ahc), devinfo->channel, devinfo->target,
3544                        last_msg);
3545         }
3546         return (response);
3547 }
3548
3549 /*
3550  * Process an ingnore wide residue message.
3551  */
3552 static void
3553 ahc_handle_ign_wide_residue(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3554 {
3555         u_int scb_index;
3556         struct scb *scb;
3557
3558         scb_index = ahc_inb(ahc, SCB_TAG);
3559         scb = ahc_lookup_scb(ahc, scb_index);
3560         /*
3561          * XXX Actually check data direction in the sequencer?
3562          * Perhaps add datadir to some spare bits in the hscb?
3563          */
3564         if ((ahc_inb(ahc, SEQ_FLAGS) & DPHASE) == 0
3565          || ahc_get_transfer_dir(scb) != CAM_DIR_IN) {
3566                 /*
3567                  * Ignore the message if we haven't
3568                  * seen an appropriate data phase yet.
3569                  */
3570         } else {
3571                 /*
3572                  * If the residual occurred on the last
3573                  * transfer and the transfer request was
3574                  * expected to end on an odd count, do
3575                  * nothing.  Otherwise, subtract a byte
3576                  * and update the residual count accordingly.
3577                  */
3578                 uint32_t sgptr;
3579
3580                 sgptr = ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
3581                 if ((sgptr & SG_LIST_NULL) != 0
3582                  && (ahc_inb(ahc, SCB_LUN) & SCB_XFERLEN_ODD) != 0) {
3583                         /*
3584                          * If the residual occurred on the last
3585                          * transfer and the transfer request was
3586                          * expected to end on an odd count, do
3587                          * nothing.
3588                          */
3589                 } else {
3590                         struct ahc_dma_seg *sg;
3591                         uint32_t data_cnt;
3592                         uint32_t data_addr;
3593                         uint32_t sglen;
3594
3595                         /* Pull in all of the sgptr */
3596                         sgptr = ahc_inl(ahc, SCB_RESIDUAL_SGPTR);
3597                         data_cnt = ahc_inl(ahc, SCB_RESIDUAL_DATACNT);
3598
3599                         if ((sgptr & SG_LIST_NULL) != 0) {
3600                                 /*
3601                                  * The residual data count is not updated
3602                                  * for the command run to completion case.
3603                                  * Explicitly zero the count.
3604                                  */
3605                                 data_cnt &= ~AHC_SG_LEN_MASK;
3606                         }
3607
3608                         data_addr = ahc_inl(ahc, SHADDR);
3609
3610                         data_cnt += 1;
3611                         data_addr -= 1;
3612                         sgptr &= SG_PTR_MASK;
3613
3614                         sg = ahc_sg_bus_to_virt(scb, sgptr);
3615
3616                         /*
3617                          * The residual sg ptr points to the next S/G
3618                          * to load so we must go back one.
3619                          */
3620                         sg--;
3621                         sglen = ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
3622                         if (sg != scb->sg_list
3623                          && sglen < (data_cnt & AHC_SG_LEN_MASK)) {
3624
3625                                 sg--;
3626                                 sglen = ahc_le32toh(sg->len);
3627                                 /*
3628                                  * Preserve High Address and SG_LIST bits
3629                                  * while setting the count to 1.
3630                                  */
3631                                 data_cnt = 1 | (sglen & (~AHC_SG_LEN_MASK));
3632                                 data_addr = ahc_le32toh(sg->addr)
3633                                           + (sglen & AHC_SG_LEN_MASK) - 1;
3634
3635                                 /*
3636                                  * Increment sg so it points to the
3637                                  * "next" sg.
3638                                  */
3639                                 sg++;
3640                                 sgptr = ahc_sg_virt_to_bus(scb, sg);
3641                         }
3642                         ahc_outl(ahc, SCB_RESIDUAL_SGPTR, sgptr);
3643                         ahc_outl(ahc, SCB_RESIDUAL_DATACNT, data_cnt);
3644                         /*
3645                          * Toggle the "oddness" of the transfer length
3646                          * to handle this mid-transfer ignore wide
3647                          * residue.  This ensures that the oddness is
3648                          * correct for subsequent data transfers.
3649                          */
3650                         ahc_outb(ahc, SCB_LUN,
3651                                  ahc_inb(ahc, SCB_LUN) ^ SCB_XFERLEN_ODD);
3652                 }
3653         }
3654 }
3655
3656
3657 /*
3658  * Reinitialize the data pointers for the active transfer
3659  * based on its current residual.
3660  */
3661 static void
3662 ahc_reinitialize_dataptrs(struct ahc_softc *ahc)
3663 {
3664         struct   scb *scb;
3665         struct   ahc_dma_seg *sg;
3666         u_int    scb_index;
3667         uint32_t sgptr;
3668         uint32_t resid;
3669         uint32_t dataptr;
3670
3671         scb_index = ahc_inb(ahc, SCB_TAG);
3672         scb = ahc_lookup_scb(ahc, scb_index);
3673         sgptr = (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 3) << 24)
3674               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 2) << 16)
3675               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 1) << 8)
3676               | ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
3677
3678         sgptr &= SG_PTR_MASK;
3679         sg = ahc_sg_bus_to_virt(scb, sgptr);
3680
3681         /* The residual sg_ptr always points to the next sg */
3682         sg--;
3683
3684         resid = (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 2) << 16)
3685               | (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 1) << 8)
3686               | ahc_inb(ahc, SCB_RESIDUAL_DATACNT);
3687
3688         dataptr = ahc_le32toh(sg->addr)
3689                 + (ahc_le32toh(sg->len) & AHC_SG_LEN_MASK)
3690                 - resid;
3691         if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
3692                 u_int dscommand1;
3693
3694                 dscommand1 = ahc_inb(ahc, DSCOMMAND1);
3695                 ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
3696                 ahc_outb(ahc, HADDR,
3697                          (ahc_le32toh(sg->len) >> 24) & SG_HIGH_ADDR_BITS);
3698                 ahc_outb(ahc, DSCOMMAND1, dscommand1);
3699         }
3700         ahc_outb(ahc, HADDR + 3, dataptr >> 24);
3701         ahc_outb(ahc, HADDR + 2, dataptr >> 16);
3702         ahc_outb(ahc, HADDR + 1, dataptr >> 8);
3703         ahc_outb(ahc, HADDR, dataptr);
3704         ahc_outb(ahc, HCNT + 2, resid >> 16);
3705         ahc_outb(ahc, HCNT + 1, resid >> 8);
3706         ahc_outb(ahc, HCNT, resid);
3707         if ((ahc->features & AHC_ULTRA2) == 0) {
3708                 ahc_outb(ahc, STCNT + 2, resid >> 16);
3709                 ahc_outb(ahc, STCNT + 1, resid >> 8);
3710                 ahc_outb(ahc, STCNT, resid);
3711         }
3712 }
3713
3714 /*
3715  * Handle the effects of issuing a bus device reset message.
3716  */
3717 static void
3718 ahc_handle_devreset(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3719                     cam_status status, char *message, int verbose_level)
3720 {
3721 #ifdef AHC_TARGET_MODE
3722         struct ahc_tmode_tstate* tstate;
3723         u_int lun;
3724 #endif
3725         int found;
3726
3727         found = ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3728                                CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
3729                                status);
3730
3731 #ifdef AHC_TARGET_MODE
3732         /*
3733          * Send an immediate notify ccb to all target mord peripheral
3734          * drivers affected by this action.
3735          */
3736         tstate = ahc->enabled_targets[devinfo->our_scsiid];
3737         if (tstate != NULL) {
3738                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
3739                         struct ahc_tmode_lstate* lstate;
3740
3741                         lstate = tstate->enabled_luns[lun];
3742                         if (lstate == NULL)
3743                                 continue;
3744
3745                         ahc_queue_lstate_event(ahc, lstate, devinfo->our_scsiid,
3746                                                MSG_BUS_DEV_RESET, /*arg*/0);
3747                         ahc_send_lstate_events(ahc, lstate);
3748                 }
3749         }
3750 #endif
3751
3752         /*
3753          * Go back to async/narrow transfers and renegotiate.
3754          */
3755         ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3756                       AHC_TRANS_CUR, /*paused*/TRUE);
3757         ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL,
3758                          /*period*/0, /*offset*/0, /*ppr_options*/0,
3759                          AHC_TRANS_CUR, /*paused*/TRUE);
3760
3761         ahc_send_async(ahc, devinfo->channel, devinfo->target,
3762                        CAM_LUN_WILDCARD, AC_SENT_BDR, NULL);
3763
3764         if (message != NULL
3765          && (verbose_level <= bootverbose))
3766                 kprintf("%s: %s on %c:%d. %d SCBs aborted\n", ahc_name(ahc),
3767                        message, devinfo->channel, devinfo->target, found);
3768 }
3769
3770 #ifdef AHC_TARGET_MODE
3771 static void
3772 ahc_setup_target_msgin(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3773                        struct scb *scb)
3774 {
3775
3776         /*
3777          * To facilitate adding multiple messages together,
3778          * each routine should increment the index and len
3779          * variables instead of setting them explicitly.
3780          */
3781         ahc->msgout_index = 0;
3782         ahc->msgout_len = 0;
3783
3784         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
3785                 ahc_build_transfer_msg(ahc, devinfo);
3786         else
3787                 panic("ahc_intr: AWAITING target message with no message");
3788
3789         ahc->msgout_index = 0;
3790         ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3791 }
3792 #endif
3793 /**************************** Initialization **********************************/
3794 /*
3795  * Allocate a controller structure for a new device
3796  * and perform initial initializion.
3797  */
3798 struct ahc_softc *
3799 ahc_alloc(void *platform_arg, char *name)
3800 {
3801         struct  ahc_softc *ahc;
3802         int     i;
3803
3804 #if !defined(__DragonFly__) && !defined(__FreeBSD__)
3805         ahc = kmalloc(sizeof(*ahc), M_DEVBUF, M_WAITOK);
3806 #else
3807         ahc = device_get_softc((device_t)platform_arg);
3808 #endif
3809         memset(ahc, 0, sizeof(*ahc));
3810         ahc->seep_config = kmalloc(sizeof(*ahc->seep_config),
3811                                   M_DEVBUF, M_WAITOK);
3812         LIST_INIT(&ahc->pending_scbs);
3813         /* We don't know our unit number until the OSM sets it */
3814         ahc->name = name;
3815         ahc->unit = -1;
3816         ahc->description = NULL;
3817         ahc->channel = 'A';
3818         ahc->channel_b = 'B';
3819         ahc->chip = AHC_NONE;
3820         ahc->features = AHC_FENONE;
3821         ahc->bugs = AHC_BUGNONE;
3822         ahc->flags = AHC_FNONE;
3823         /*
3824          * Default to all error reporting enabled with the
3825          * sequencer operating at its fastest speed.
3826          * The bus attach code may modify this.
3827          */
3828         ahc->seqctl = FASTMODE;
3829
3830         for (i = 0; i < AHC_NUM_TARGETS; i++)
3831                 TAILQ_INIT(&ahc->untagged_queues[i]);
3832         if (ahc_platform_alloc(ahc, platform_arg) != 0) {
3833                 ahc_free(ahc);
3834                 ahc = NULL;
3835         }
3836         return (ahc);
3837 }
3838
3839 int
3840 ahc_softc_init(struct ahc_softc *ahc)
3841 {
3842
3843         /* The IRQMS bit is only valid on VL and EISA chips */
3844         if ((ahc->chip & AHC_PCI) == 0)
3845                 ahc->unpause = ahc_inb(ahc, HCNTRL) & IRQMS;
3846         else
3847                 ahc->unpause = 0;
3848         ahc->pause = ahc->unpause | PAUSE;
3849         /* XXX The shared scb data stuff should be deprecated */
3850         if (ahc->scb_data == NULL) {
3851                 ahc->scb_data = kmalloc(sizeof(*ahc->scb_data),
3852                                        M_DEVBUF, M_WAITOK | M_ZERO);
3853         }
3854
3855         return (0);
3856 }
3857
3858 void
3859 ahc_softc_insert(struct ahc_softc *ahc)
3860 {
3861         struct ahc_softc *list_ahc;
3862
3863 #if AHC_PCI_CONFIG > 0
3864         /*
3865          * Second Function PCI devices need to inherit some
3866          * settings from function 0.
3867          */
3868         if ((ahc->chip & AHC_BUS_MASK) == AHC_PCI
3869          && (ahc->features & AHC_MULTI_FUNC) != 0) {
3870                 TAILQ_FOREACH(list_ahc, &ahc_tailq, links) {
3871                         ahc_dev_softc_t list_pci;
3872                         ahc_dev_softc_t pci;
3873
3874                         list_pci = list_ahc->dev_softc;
3875                         pci = ahc->dev_softc;
3876                         if (ahc_get_pci_slot(list_pci) == ahc_get_pci_slot(pci)
3877                          && ahc_get_pci_bus(list_pci) == ahc_get_pci_bus(pci)) {
3878                                 struct ahc_softc *master;
3879                                 struct ahc_softc *slave;
3880
3881                                 if (ahc_get_pci_function(list_pci) == 0) {
3882                                         master = list_ahc;
3883                                         slave = ahc;
3884                                 } else {
3885                                         master = ahc;
3886                                         slave = list_ahc;
3887                                 }
3888                                 slave->flags &= ~AHC_BIOS_ENABLED;
3889                                 slave->flags |=
3890                                     master->flags & AHC_BIOS_ENABLED;
3891                                 slave->flags &= ~AHC_PRIMARY_CHANNEL;
3892                                 slave->flags |=
3893                                     master->flags & AHC_PRIMARY_CHANNEL;
3894                                 break;
3895                         }
3896                 }
3897         }
3898 #endif
3899
3900         /*
3901          * Insertion sort into our list of softcs.
3902          */
3903         list_ahc = TAILQ_FIRST(&ahc_tailq);
3904         while (list_ahc != NULL
3905             && ahc_softc_comp(ahc, list_ahc) <= 0)
3906                 list_ahc = TAILQ_NEXT(list_ahc, links);
3907         if (list_ahc != NULL)
3908                 TAILQ_INSERT_BEFORE(list_ahc, ahc, links);
3909         else
3910                 TAILQ_INSERT_TAIL(&ahc_tailq, ahc, links);
3911         ahc->init_level++;
3912 }
3913
3914 /*
3915  * Verify that the passed in softc pointer is for a
3916  * controller that is still configured.
3917  */
3918 struct ahc_softc *
3919 ahc_find_softc(struct ahc_softc *ahc)
3920 {
3921         struct ahc_softc *list_ahc;
3922
3923         TAILQ_FOREACH(list_ahc, &ahc_tailq, links) {
3924                 if (list_ahc == ahc)
3925                         return (ahc);
3926         }
3927         return (NULL);
3928 }
3929
3930 void
3931 ahc_set_unit(struct ahc_softc *ahc, int unit)
3932 {
3933         ahc->unit = unit;
3934 }
3935
3936 void
3937 ahc_set_name(struct ahc_softc *ahc, char *name)
3938 {
3939         if (ahc->name != NULL)
3940                 kfree(ahc->name, M_DEVBUF);
3941         ahc->name = name;
3942 }
3943
3944 void
3945 ahc_free(struct ahc_softc *ahc)
3946 {
3947         int i;
3948
3949         switch (ahc->init_level) {
3950         default:
3951         case 5:
3952                 ahc_shutdown(ahc);
3953                 TAILQ_REMOVE(&ahc_tailq, ahc, links);
3954                 /* FALLTHROUGH */
3955         case 4:
3956                 ahc_dmamap_unload(ahc, ahc->shared_data_dmat,
3957                                   ahc->shared_data_dmamap);
3958                 /* FALLTHROUGH */
3959         case 3:
3960                 ahc_dmamem_free(ahc, ahc->shared_data_dmat, ahc->qoutfifo,
3961                                 ahc->shared_data_dmamap);
3962                 ahc_dmamap_destroy(ahc, ahc->shared_data_dmat,
3963                                    ahc->shared_data_dmamap);
3964                 /* FALLTHROUGH */
3965         case 2:
3966                 ahc_dma_tag_destroy(ahc, ahc->shared_data_dmat);
3967         case 1:
3968 #ifndef __linux__
3969                 ahc_dma_tag_destroy(ahc, ahc->buffer_dmat);
3970 #endif
3971                 break;
3972         case 0:
3973                 break;
3974         }
3975
3976 #ifndef __linux__
3977         ahc_dma_tag_destroy(ahc, ahc->parent_dmat);
3978 #endif
3979         ahc_platform_free(ahc);
3980         ahc_fini_scbdata(ahc);
3981         for (i = 0; i < AHC_NUM_TARGETS; i++) {
3982                 struct ahc_tmode_tstate *tstate;
3983
3984                 tstate = ahc->enabled_targets[i];
3985                 if (tstate != NULL) {
3986 #ifdef AHC_TARGET_MODE
3987                         int j;
3988
3989                         for (j = 0; j < AHC_NUM_LUNS; j++) {
3990                                 struct ahc_tmode_lstate *lstate;
3991
3992                                 lstate = tstate->enabled_luns[j];
3993                                 if (lstate != NULL) {
3994                                         xpt_free_path(lstate->path);
3995                                         kfree(lstate, M_DEVBUF);
3996                                 }
3997                         }
3998 #endif
3999                         kfree(tstate, M_DEVBUF);
4000                 }
4001         }
4002 #ifdef AHC_TARGET_MODE
4003         if (ahc->black_hole != NULL) {
4004                 xpt_free_path(ahc->black_hole->path);
4005                 kfree(ahc->black_hole, M_DEVBUF);
4006         }
4007 #endif
4008         if (ahc->name != NULL)
4009                 kfree(ahc->name, M_DEVBUF);
4010         if (ahc->seep_config != NULL)
4011                 kfree(ahc->seep_config, M_DEVBUF);
4012 #if !defined(__DragonFly__) && !defined(__FreeBSD__)
4013         kfree(ahc, M_DEVBUF);
4014 #endif
4015         return;
4016 }
4017
4018 void
4019 ahc_shutdown(void *arg)
4020 {
4021         struct  ahc_softc *ahc;
4022         int     i;
4023
4024         ahc = (struct ahc_softc *)arg;
4025
4026         /* This will reset most registers to 0, but not all */
4027         ahc_reset(ahc, /*reinit*/FALSE);
4028         ahc_outb(ahc, SCSISEQ, 0);
4029         ahc_outb(ahc, SXFRCTL0, 0);
4030         ahc_outb(ahc, DSPCISTATUS, 0);
4031
4032         for (i = TARG_SCSIRATE; i < SCSICONF; i++)
4033                 ahc_outb(ahc, i, 0);
4034 }
4035
4036 /*
4037  * Reset the controller and record some information about it
4038  * that is only available just after a reset.  If "reinit" is
4039  * non-zero, this reset occured after initial configuration
4040  * and the caller requests that the chip be fully reinitialized
4041  * to a runable state.  Chip interrupts are *not* enabled after
4042  * a reinitialization.  The caller must enable interrupts via
4043  * ahc_intr_enable().
4044  */
4045 int
4046 ahc_reset(struct ahc_softc *ahc, int reinit)
4047 {
4048         u_int   sblkctl;
4049         u_int   sxfrctl1_a, sxfrctl1_b;
4050         int     error;
4051         int     wait;
4052
4053         /*
4054          * Preserve the value of the SXFRCTL1 register for all channels.
4055          * It contains settings that affect termination and we don't want
4056          * to disturb the integrity of the bus.
4057          */
4058         ahc_pause(ahc);
4059         if ((ahc_inb(ahc, HCNTRL) & CHIPRST) != 0) {
4060                 /*
4061                  * The chip has not been initialized since
4062                  * PCI/EISA/VLB bus reset.  Don't trust
4063                  * "left over BIOS data".
4064                  */
4065                 ahc->flags |= AHC_NO_BIOS_INIT;
4066         }
4067         sxfrctl1_b = 0;
4068         if ((ahc->chip & AHC_CHIPID_MASK) == AHC_AIC7770) {
4069                 u_int sblkctl;
4070
4071                 /*
4072                  * Save channel B's settings in case this chip
4073                  * is setup for TWIN channel operation.
4074                  */
4075                 sblkctl = ahc_inb(ahc, SBLKCTL);
4076                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4077                 sxfrctl1_b = ahc_inb(ahc, SXFRCTL1);
4078                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4079         }
4080         sxfrctl1_a = ahc_inb(ahc, SXFRCTL1);
4081
4082         ahc_outb(ahc, HCNTRL, CHIPRST | ahc->pause);
4083
4084         /*
4085          * Ensure that the reset has finished.  We delay 1000us
4086          * prior to reading the register to make sure the chip
4087          * has sufficiently completed its reset to handle register
4088          * accesses.
4089          */
4090         wait = 1000;
4091         do {
4092                 ahc_delay(1000);
4093         } while (--wait && !(ahc_inb(ahc, HCNTRL) & CHIPRSTACK));
4094
4095         if (wait == 0) {
4096                 kprintf("%s: WARNING - Failed chip reset!  "
4097                        "Trying to initialize anyway.\n", ahc_name(ahc));
4098         }
4099         ahc_outb(ahc, HCNTRL, ahc->pause);
4100
4101         /* Determine channel configuration */
4102         sblkctl = ahc_inb(ahc, SBLKCTL) & (SELBUSB|SELWIDE);
4103         /* No Twin Channel PCI cards */
4104         if ((ahc->chip & AHC_PCI) != 0)
4105                 sblkctl &= ~SELBUSB;
4106         switch (sblkctl) {
4107         case 0:
4108                 /* Single Narrow Channel */
4109                 break;
4110         case 2:
4111                 /* Wide Channel */
4112                 ahc->features |= AHC_WIDE;
4113                 break;
4114         case 8:
4115                 /* Twin Channel */
4116                 ahc->features |= AHC_TWIN;
4117                 break;
4118         default:
4119                 kprintf(" Unsupported adapter type.  Ignoring\n");
4120                 return(-1);
4121         }
4122
4123         /*
4124          * Reload sxfrctl1.
4125          *
4126          * We must always initialize STPWEN to 1 before we
4127          * restore the saved values.  STPWEN is initialized
4128          * to a tri-state condition which can only be cleared
4129          * by turning it on.
4130          */
4131         if ((ahc->features & AHC_TWIN) != 0) {
4132                 u_int sblkctl;
4133
4134                 sblkctl = ahc_inb(ahc, SBLKCTL);
4135                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4136                 ahc_outb(ahc, SXFRCTL1, sxfrctl1_b);
4137                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4138         }
4139         ahc_outb(ahc, SXFRCTL1, sxfrctl1_a);
4140
4141         error = 0;
4142         if (reinit != 0)
4143                 /*
4144                  * If a recovery action has forced a chip reset,
4145                  * re-initialize the chip to our liking.
4146                  */
4147                 error = ahc->bus_chip_init(ahc);
4148 #ifdef AHC_DUMP_SEQ
4149         else
4150                 ahc_dumpseq(ahc);
4151 #endif
4152
4153         return (error);
4154 }
4155
4156 /*
4157  * Determine the number of SCBs available on the controller
4158  */
4159 int
4160 ahc_probe_scbs(struct ahc_softc *ahc) {
4161         int i;
4162
4163         for (i = 0; i < AHC_SCB_MAX; i++) {
4164
4165                 ahc_outb(ahc, SCBPTR, i);
4166                 ahc_outb(ahc, SCB_BASE, i);
4167                 if (ahc_inb(ahc, SCB_BASE) != i)
4168                         break;
4169                 ahc_outb(ahc, SCBPTR, 0);
4170                 if (ahc_inb(ahc, SCB_BASE) != 0)
4171                         break;
4172         }
4173         return (i);
4174 }
4175
4176 static void
4177 ahc_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4178 {
4179         bus_addr_t *baddr;
4180
4181         baddr = (bus_addr_t *)arg;
4182         *baddr = segs->ds_addr;
4183 }
4184
4185 static void
4186 ahc_build_free_scb_list(struct ahc_softc *ahc)
4187 {
4188         int scbsize;
4189         int i;
4190
4191         scbsize = 32;
4192         if ((ahc->flags & AHC_LSCBS_ENABLED) != 0)
4193                 scbsize = 64;
4194
4195         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
4196                 int j;
4197
4198                 ahc_outb(ahc, SCBPTR, i);
4199
4200                 /*
4201                  * Touch all SCB bytes to avoid parity errors
4202                  * should one of our debugging routines read
4203                  * an otherwise uninitiatlized byte.
4204                  */
4205                 for (j = 0; j < scbsize; j++)
4206                         ahc_outb(ahc, SCB_BASE+j, 0xFF);
4207
4208                 /* Clear the control byte. */
4209                 ahc_outb(ahc, SCB_CONTROL, 0);
4210
4211                 /* Set the next pointer */
4212                 if ((ahc->flags & AHC_PAGESCBS) != 0)
4213                         ahc_outb(ahc, SCB_NEXT, i+1);
4214                 else
4215                         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4216
4217                 /* Make the tag number, SCSIID, and lun invalid */
4218                 ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
4219                 ahc_outb(ahc, SCB_SCSIID, 0xFF);
4220                 ahc_outb(ahc, SCB_LUN, 0xFF);
4221         }
4222
4223         if ((ahc->flags & AHC_PAGESCBS) != 0) {
4224                 /* SCB 0 heads the free list. */
4225                 ahc_outb(ahc, FREE_SCBH, 0);
4226         } else {
4227                 /* No free list. */
4228                 ahc_outb(ahc, FREE_SCBH, SCB_LIST_NULL);
4229         }
4230
4231         /* Make sure that the last SCB terminates the free list */
4232         ahc_outb(ahc, SCBPTR, i-1);
4233         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4234 }
4235
4236 static int
4237 ahc_init_scbdata(struct ahc_softc *ahc)
4238 {
4239         struct scb_data *scb_data;
4240
4241         scb_data = ahc->scb_data;
4242         SLIST_INIT(&scb_data->free_scbs);
4243         SLIST_INIT(&scb_data->sg_maps);
4244
4245         /* Allocate SCB resources */
4246         scb_data->scbarray = kmalloc(sizeof(struct scb) * AHC_SCB_MAX_ALLOC,
4247                                     M_DEVBUF, M_INTWAIT | M_ZERO);
4248
4249         /* Determine the number of hardware SCBs and initialize them */
4250
4251         scb_data->maxhscbs = ahc_probe_scbs(ahc);
4252         if (ahc->scb_data->maxhscbs == 0) {
4253                 kprintf("%s: No SCB space found\n", ahc_name(ahc));
4254                 return (ENXIO);
4255         }
4256
4257         /*
4258          * Create our DMA tags.  These tags define the kinds of device
4259          * accessible memory allocations and memory mappings we will
4260          * need to perform during normal operation.
4261          *
4262          * Unless we need to further restrict the allocation, we rely
4263          * on the restrictions of the parent dmat, hence the common
4264          * use of MAXADDR and MAXSIZE.
4265          */
4266
4267         /* DMA tag for our hardware scb structures */
4268         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4269                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4270                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4271                                /*highaddr*/BUS_SPACE_MAXADDR,
4272                                /*filter*/NULL, /*filterarg*/NULL,
4273                                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4274                                /*nsegments*/1,
4275                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4276                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
4277                 goto error_exit;
4278         }
4279
4280         scb_data->init_level++;
4281
4282         /* Allocation for our hscbs */
4283         if (ahc_dmamem_alloc(ahc, scb_data->hscb_dmat,
4284                              (void **)&scb_data->hscbs,
4285                              BUS_DMA_NOWAIT, &scb_data->hscb_dmamap) != 0) {
4286                 goto error_exit;
4287         }
4288
4289         scb_data->init_level++;
4290
4291         /* And permanently map them */
4292         ahc_dmamap_load(ahc, scb_data->hscb_dmat, scb_data->hscb_dmamap,
4293                         scb_data->hscbs,
4294                         AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4295                         ahc_dmamap_cb, &scb_data->hscb_busaddr, /*flags*/0);
4296
4297         scb_data->init_level++;
4298
4299         /* DMA tag for our sense buffers */
4300         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4301                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4302                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4303                                /*highaddr*/BUS_SPACE_MAXADDR,
4304                                /*filter*/NULL, /*filterarg*/NULL,
4305                                AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4306                                /*nsegments*/1,
4307                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4308                                /*flags*/0, &scb_data->sense_dmat) != 0) {
4309                 goto error_exit;
4310         }
4311
4312         scb_data->init_level++;
4313
4314         /* Allocate them */
4315         if (ahc_dmamem_alloc(ahc, scb_data->sense_dmat,
4316                              (void **)&scb_data->sense,
4317                              BUS_DMA_NOWAIT, &scb_data->sense_dmamap) != 0) {
4318                 goto error_exit;
4319         }
4320
4321         scb_data->init_level++;
4322
4323         /* And permanently map them */
4324         ahc_dmamap_load(ahc, scb_data->sense_dmat, scb_data->sense_dmamap,
4325                         scb_data->sense,
4326                         AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4327                         ahc_dmamap_cb, &scb_data->sense_busaddr, /*flags*/0);
4328
4329         scb_data->init_level++;
4330
4331         /* DMA tag for our S/G structures.  We allocate in page sized chunks */
4332         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/8,
4333                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4334                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4335                                /*highaddr*/BUS_SPACE_MAXADDR,
4336                                /*filter*/NULL, /*filterarg*/NULL,
4337                                PAGE_SIZE, /*nsegments*/1,
4338                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4339                                /*flags*/0, &scb_data->sg_dmat) != 0) {
4340                 goto error_exit;
4341         }
4342
4343         scb_data->init_level++;
4344
4345         /* Perform initial CCB allocation */
4346         memset(scb_data->hscbs, 0,
4347                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb));
4348         ahc_alloc_scbs(ahc);
4349
4350         if (scb_data->numscbs == 0) {
4351                 kprintf("%s: ahc_init_scbdata - "
4352                        "Unable to allocate initial scbs\n",
4353                        ahc_name(ahc));
4354                 goto error_exit;
4355         }
4356
4357         /*
4358          * Reserve the next queued SCB.
4359          */
4360         ahc->next_queued_scb = ahc_get_scb(ahc);
4361
4362         /*
4363          * Note that we were successful
4364          */
4365         return (0);
4366
4367 error_exit:
4368
4369         return (ENOMEM);
4370 }
4371
4372 static void
4373 ahc_fini_scbdata(struct ahc_softc *ahc)
4374 {
4375         struct scb_data *scb_data;
4376
4377         scb_data = ahc->scb_data;
4378         if (scb_data == NULL)
4379                 return;
4380
4381         switch (scb_data->init_level) {
4382         default:
4383         case 7:
4384         {
4385                 struct sg_map_node *sg_map;
4386
4387                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps))!= NULL) {
4388                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4389                         ahc_dmamap_unload(ahc, scb_data->sg_dmat,
4390                                           sg_map->sg_dmamap);
4391                         ahc_dmamem_free(ahc, scb_data->sg_dmat,
4392                                         sg_map->sg_vaddr,
4393                                         sg_map->sg_dmamap);
4394                         kfree(sg_map, M_DEVBUF);
4395                 }
4396                 ahc_dma_tag_destroy(ahc, scb_data->sg_dmat);
4397         }
4398         case 6:
4399                 ahc_dmamap_unload(ahc, scb_data->sense_dmat,
4400                                   scb_data->sense_dmamap);
4401         case 5:
4402                 ahc_dmamem_free(ahc, scb_data->sense_dmat, scb_data->sense,
4403                                 scb_data->sense_dmamap);
4404                 ahc_dmamap_destroy(ahc, scb_data->sense_dmat,
4405                                    scb_data->sense_dmamap);
4406         case 4:
4407                 ahc_dma_tag_destroy(ahc, scb_data->sense_dmat);
4408         case 3:
4409                 ahc_dmamap_unload(ahc, scb_data->hscb_dmat,
4410                                   scb_data->hscb_dmamap);
4411         case 2:
4412                 ahc_dmamem_free(ahc, scb_data->hscb_dmat, scb_data->hscbs,
4413                                 scb_data->hscb_dmamap);
4414                 ahc_dmamap_destroy(ahc, scb_data->hscb_dmat,
4415                                    scb_data->hscb_dmamap);
4416         case 1:
4417                 ahc_dma_tag_destroy(ahc, scb_data->hscb_dmat);
4418                 break;
4419         case 0:
4420                 break;
4421         }
4422         if (scb_data->scbarray != NULL)
4423                 kfree(scb_data->scbarray, M_DEVBUF);
4424 }
4425
4426 void
4427 ahc_alloc_scbs(struct ahc_softc *ahc)
4428 {
4429         struct scb_data *scb_data;
4430         struct scb *next_scb;
4431         struct sg_map_node *sg_map;
4432         bus_addr_t physaddr;
4433         struct ahc_dma_seg *segs;
4434         int newcount;
4435         int i;
4436
4437         scb_data = ahc->scb_data;
4438         if (scb_data->numscbs >= AHC_SCB_MAX_ALLOC)
4439                 /* Can't allocate any more */
4440                 return;
4441
4442         next_scb = &scb_data->scbarray[scb_data->numscbs];
4443
4444         sg_map = kmalloc(sizeof(*sg_map), M_DEVBUF, M_INTWAIT);
4445
4446         /* Allocate S/G space for the next batch of SCBS */
4447         if (ahc_dmamem_alloc(ahc, scb_data->sg_dmat,
4448                              (void **)&sg_map->sg_vaddr,
4449                              BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
4450                 kfree(sg_map, M_DEVBUF);
4451                 return;
4452         }
4453
4454         SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
4455
4456         ahc_dmamap_load(ahc, scb_data->sg_dmat, sg_map->sg_dmamap,
4457                         sg_map->sg_vaddr, PAGE_SIZE, ahc_dmamap_cb,
4458                         &sg_map->sg_physaddr, /*flags*/0);
4459
4460         segs = sg_map->sg_vaddr;
4461         physaddr = sg_map->sg_physaddr;
4462
4463         newcount = (PAGE_SIZE / (AHC_NSEG * sizeof(struct ahc_dma_seg)));
4464         newcount = MIN(newcount, (AHC_SCB_MAX_ALLOC - scb_data->numscbs));
4465         for (i = 0; i < newcount; i++) {
4466                 struct scb_platform_data *pdata;
4467 #ifndef __linux__
4468                 int error;
4469 #endif
4470                 pdata = kmalloc(sizeof(*pdata), M_DEVBUF, M_INTWAIT);
4471                 next_scb->platform_data = pdata;
4472                 next_scb->sg_map = sg_map;
4473                 next_scb->sg_list = segs;
4474                 /*
4475                  * The sequencer always starts with the second entry.
4476                  * The first entry is embedded in the scb.
4477                  */
4478                 next_scb->sg_list_phys = physaddr + sizeof(struct ahc_dma_seg);
4479                 next_scb->ahc_softc = ahc;
4480                 next_scb->flags = SCB_FREE;
4481 #ifndef __linux__
4482                 error = ahc_dmamap_create(ahc, ahc->buffer_dmat, /*flags*/0,
4483                                           &next_scb->dmamap);
4484                 if (error != 0)
4485                         break;
4486 #endif
4487                 next_scb->hscb = &scb_data->hscbs[scb_data->numscbs];
4488                 next_scb->hscb->tag = ahc->scb_data->numscbs;
4489                 SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs,
4490                                   next_scb, links.sle);
4491                 segs += AHC_NSEG;
4492                 physaddr += (AHC_NSEG * sizeof(struct ahc_dma_seg));
4493                 next_scb++;
4494                 ahc->scb_data->numscbs++;
4495         }
4496 }
4497
4498 void
4499 ahc_controller_info(struct ahc_softc *ahc, char *buf)
4500 {
4501         int len;
4502
4503         len = ksprintf(buf, "%s: ",
4504                        ahc_chip_names[ahc->chip & AHC_CHIPID_MASK]);
4505         buf += len;
4506         if ((ahc->features & AHC_TWIN) != 0) {
4507                 len = ksprintf(buf, "Twin Channel, A SCSI Id=%d, "
4508                                "B SCSI Id=%d, primary %c, ",
4509                                ahc->our_id, ahc->our_id_b,
4510                                (ahc->flags & AHC_PRIMARY_CHANNEL) + 'A');
4511         } else {
4512                 const char *speed;
4513                 const char *type;
4514
4515                 speed = "";
4516                 if ((ahc->features & AHC_ULTRA) != 0) {
4517                         speed = "Ultra ";
4518                 } else if ((ahc->features & AHC_DT) != 0) {
4519                         speed = "Ultra160 ";
4520                 } else if ((ahc->features & AHC_ULTRA2) != 0) {
4521                         speed = "Ultra2 ";
4522                 }
4523                 if ((ahc->features & AHC_WIDE) != 0) {
4524                         type = "Wide";
4525                 } else {
4526                         type = "Single";
4527                 }
4528                 len = ksprintf(buf, "%s%s Channel %c, SCSI Id=%d, ",
4529                                speed, type, ahc->channel, ahc->our_id);
4530         }
4531         buf += len;
4532
4533         if ((ahc->flags & AHC_PAGESCBS) != 0)
4534                 ksprintf(buf, "%d/%d SCBs",
4535                          ahc->scb_data->maxhscbs, AHC_MAX_QUEUE);
4536         else
4537                 ksprintf(buf, "%d SCBs", ahc->scb_data->maxhscbs);
4538 }
4539
4540 int
4541 ahc_chip_init(struct ahc_softc *ahc)
4542 {
4543         int      term;
4544         int      error;
4545         u_int    i;
4546         u_int    scsi_conf;
4547         u_int    scsiseq_template;
4548         uint32_t physaddr;
4549
4550         ahc_outb(ahc, SEQ_FLAGS, 0);
4551         ahc_outb(ahc, SEQ_FLAGS2, 0);
4552
4553         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1, for both channels*/
4554         if (ahc->features & AHC_TWIN) {
4555
4556                 /*
4557                  * Setup Channel B first.
4558                  */
4559                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) | SELBUSB);
4560                 term = (ahc->flags & AHC_TERM_ENB_B) != 0 ? STPWEN : 0;
4561                 ahc_outb(ahc, SCSIID, ahc->our_id_b);
4562                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
4563                 ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
4564                                         |term|ahc->seltime_b|ENSTIMER|ACTNEGEN);
4565                 if ((ahc->features & AHC_ULTRA2) != 0)
4566                         ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
4567                 ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
4568                 ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
4569
4570                 /* Select Channel A */
4571                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) & ~SELBUSB);
4572         }
4573         term = (ahc->flags & AHC_TERM_ENB_A) != 0 ? STPWEN : 0;
4574         if ((ahc->features & AHC_ULTRA2) != 0)
4575                 ahc_outb(ahc, SCSIID_ULTRA2, ahc->our_id);
4576         else
4577                 ahc_outb(ahc, SCSIID, ahc->our_id);
4578         scsi_conf = ahc_inb(ahc, SCSICONF);
4579         ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
4580                                 |term|ahc->seltime
4581                                 |ENSTIMER|ACTNEGEN);
4582         if ((ahc->features & AHC_ULTRA2) != 0)
4583                 ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
4584         ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
4585         ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
4586
4587         /* There are no untagged SCBs active yet. */
4588         for (i = 0; i < 16; i++) {
4589                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, 0));
4590                 if ((ahc->flags & AHC_SCB_BTT) != 0) {
4591                         int lun;
4592
4593                         /*
4594                          * The SCB based BTT allows an entry per
4595                          * target and lun pair.
4596                          */
4597                         for (lun = 1; lun < AHC_NUM_LUNS; lun++)
4598                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, lun));
4599                 }
4600         }
4601
4602         /* All of our queues are empty */
4603         for (i = 0; i < 256; i++)
4604                 ahc->qoutfifo[i] = SCB_LIST_NULL;
4605         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_PREREAD);
4606
4607         for (i = 0; i < 256; i++)
4608                 ahc->qinfifo[i] = SCB_LIST_NULL;
4609
4610         if ((ahc->features & AHC_MULTI_TID) != 0) {
4611                 ahc_outb(ahc, TARGID, 0);
4612                 ahc_outb(ahc, TARGID + 1, 0);
4613         }
4614
4615         /*
4616          * Tell the sequencer where it can find our arrays in memory.
4617          */
4618         physaddr = ahc->scb_data->hscb_busaddr;
4619         ahc_outb(ahc, HSCB_ADDR, physaddr & 0xFF);
4620         ahc_outb(ahc, HSCB_ADDR + 1, (physaddr >> 8) & 0xFF);
4621         ahc_outb(ahc, HSCB_ADDR + 2, (physaddr >> 16) & 0xFF);
4622         ahc_outb(ahc, HSCB_ADDR + 3, (physaddr >> 24) & 0xFF);
4623
4624         physaddr = ahc->shared_data_busaddr;
4625         ahc_outb(ahc, SHARED_DATA_ADDR, physaddr & 0xFF);
4626         ahc_outb(ahc, SHARED_DATA_ADDR + 1, (physaddr >> 8) & 0xFF);
4627         ahc_outb(ahc, SHARED_DATA_ADDR + 2, (physaddr >> 16) & 0xFF);
4628         ahc_outb(ahc, SHARED_DATA_ADDR + 3, (physaddr >> 24) & 0xFF);
4629
4630         /*
4631          * Initialize the group code to command length table.
4632          * This overrides the values in TARG_SCSIRATE, so only
4633          * setup the table after we have processed that information.
4634          */
4635         ahc_outb(ahc, CMDSIZE_TABLE, 5);
4636         ahc_outb(ahc, CMDSIZE_TABLE + 1, 9);
4637         ahc_outb(ahc, CMDSIZE_TABLE + 2, 9);
4638         ahc_outb(ahc, CMDSIZE_TABLE + 3, 0);
4639         ahc_outb(ahc, CMDSIZE_TABLE + 4, 15);
4640         ahc_outb(ahc, CMDSIZE_TABLE + 5, 11);
4641         ahc_outb(ahc, CMDSIZE_TABLE + 6, 0);
4642         ahc_outb(ahc, CMDSIZE_TABLE + 7, 0);
4643
4644         if ((ahc->features & AHC_HS_MAILBOX) != 0)
4645                 ahc_outb(ahc, HS_MAILBOX, 0);
4646
4647         /* Tell the sequencer of our initial queue positions */
4648         if ((ahc->features & AHC_TARGETMODE) != 0) {
4649                 ahc->tqinfifonext = 1;
4650                 ahc_outb(ahc, KERNEL_TQINPOS, ahc->tqinfifonext - 1);
4651                 ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
4652         }
4653         ahc->qinfifonext = 0;
4654         ahc->qoutfifonext = 0;
4655         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
4656                 ahc_outb(ahc, QOFF_CTLSTA, SCB_QSIZE_256);
4657                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
4658                 ahc_outb(ahc, SNSCB_QOFF, ahc->qinfifonext);
4659                 ahc_outb(ahc, SDSCB_QOFF, 0);
4660         } else {
4661                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
4662                 ahc_outb(ahc, QINPOS, ahc->qinfifonext);
4663                 ahc_outb(ahc, QOUTPOS, ahc->qoutfifonext);
4664         }
4665
4666         /* We don't have any waiting selections */
4667         ahc_outb(ahc, WAITING_SCBH, SCB_LIST_NULL);
4668
4669         /* Our disconnection list is empty too */
4670         ahc_outb(ahc, DISCONNECTED_SCBH, SCB_LIST_NULL);
4671
4672         /* Message out buffer starts empty */
4673         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
4674
4675         /*
4676          * Setup the allowed SCSI Sequences based on operational mode.
4677          * If we are a target, we'll enalbe select in operations once
4678          * we've had a lun enabled.
4679          */
4680         scsiseq_template = ENSELO|ENAUTOATNO|ENAUTOATNP;
4681         if ((ahc->flags & AHC_INITIATORROLE) != 0)
4682                 scsiseq_template |= ENRSELI;
4683         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq_template);
4684
4685         /* Initialize our list of free SCBs. */
4686         ahc_build_free_scb_list(ahc);
4687
4688         /*
4689          * Tell the sequencer which SCB will be the next one it receives.
4690          */
4691         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
4692
4693         /*
4694          * Load the Sequencer program and Enable the adapter
4695          * in "fast" mode.
4696          */
4697         if (bootverbose)
4698                 kprintf("%s: Downloading Sequencer Program...",
4699                        ahc_name(ahc));
4700
4701         error = ahc_loadseq(ahc);
4702         if (error != 0)
4703                 return (error);
4704
4705         if ((ahc->features & AHC_ULTRA2) != 0) {
4706                 int wait;
4707
4708                 /*
4709                  * Wait for up to 500ms for our transceivers
4710                  * to settle.  If the adapter does not have
4711                  * a cable attached, the transceivers may
4712                  * never settle, so don't complain if we
4713                  * fail here.
4714                  */
4715                 for (wait = 5000;
4716                      (ahc_inb(ahc, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
4717                      wait--)
4718                         ahc_delay(100);
4719         }
4720         ahc_restart(ahc);
4721         return (0);
4722 }
4723
4724 /*
4725  * Start the board, ready for normal operation
4726  */
4727 int
4728 ahc_init(struct ahc_softc *ahc)
4729 {
4730         int      max_targ;
4731         u_int    i;
4732         u_int    scsi_conf;
4733         u_int    ultraenb;
4734         u_int    discenable;
4735         u_int    tagenable;
4736         size_t   driver_data_size;
4737
4738 #ifdef AHC_DEBUG
4739         if ((ahc_debug & AHC_DEBUG_SEQUENCER) != 0)
4740                 ahc->flags |= AHC_SEQUENCER_DEBUG;
4741 #endif
4742
4743 #ifdef AHC_PRINT_SRAM
4744         kprintf("Scratch Ram:");
4745         for (i = 0x20; i < 0x5f; i++) {
4746                 if (((i % 8) == 0) && (i != 0)) {
4747                         kprintf ("\n              ");
4748                 }
4749                 kprintf (" 0x%x", ahc_inb(ahc, i));
4750         }
4751         if ((ahc->features & AHC_MORE_SRAM) != 0) {
4752                 for (i = 0x70; i < 0x7f; i++) {
4753                         if (((i % 8) == 0) && (i != 0)) {
4754                                 kprintf ("\n              ");
4755                         }
4756                         kprintf (" 0x%x", ahc_inb(ahc, i));
4757                 }
4758         }
4759         kprintf ("\n");
4760         /*
4761          * Reading uninitialized scratch ram may
4762          * generate parity errors.
4763          */
4764         ahc_outb(ahc, CLRINT, CLRPARERR);
4765         ahc_outb(ahc, CLRINT, CLRBRKADRINT);
4766 #endif
4767         max_targ = 15;
4768
4769         /*
4770          * Assume we have a board at this stage and it has been reset.
4771          */
4772         if ((ahc->flags & AHC_USEDEFAULTS) != 0)
4773                 ahc->our_id = ahc->our_id_b = 7;
4774
4775         /*
4776          * Default to allowing initiator operations.
4777          */
4778         ahc->flags |= AHC_INITIATORROLE;
4779
4780         /*
4781          * Only allow target mode features if this unit has them enabled.
4782          */
4783         if ((AHC_TMODE_ENABLE & (0x1 << ahc->unit)) == 0)
4784                 ahc->features &= ~AHC_TARGETMODE;
4785
4786 #ifndef __linux__
4787         /* DMA tag for mapping buffers into device visible space. */
4788         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4789                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4790                                /*lowaddr*/ahc->flags & AHC_39BIT_ADDRESSING
4791                                         ? (bus_addr_t)0x7FFFFFFFFFULL
4792                                         : BUS_SPACE_MAXADDR_32BIT,
4793                                /*highaddr*/BUS_SPACE_MAXADDR,
4794                                /*filter*/NULL, /*filterarg*/NULL,
4795                                /*maxsize*/(AHC_NSEG - 1) * PAGE_SIZE,
4796                                /*nsegments*/AHC_NSEG,
4797                                /*maxsegsz*/AHC_MAXTRANSFER_SIZE,
4798                                /*flags*/BUS_DMA_ALLOCNOW,
4799                                &ahc->buffer_dmat) != 0) {
4800                 return (ENOMEM);
4801         }
4802 #endif
4803
4804         ahc->init_level++;
4805
4806         /*
4807          * DMA tag for our command fifos and other data in system memory
4808          * the card's sequencer must be able to access.  For initiator
4809          * roles, we need to allocate space for the qinfifo and qoutfifo.
4810          * The qinfifo and qoutfifo are composed of 256 1 byte elements.
4811          * When providing for the target mode role, we must additionally
4812          * provide space for the incoming target command fifo and an extra
4813          * byte to deal with a dma bug in some chip versions.
4814          */
4815         driver_data_size = 2 * 256 * sizeof(uint8_t);
4816         if ((ahc->features & AHC_TARGETMODE) != 0)
4817                 driver_data_size += AHC_TMODE_CMDS * sizeof(struct target_cmd)
4818                                  + /*DMA WideOdd Bug Buffer*/1;
4819         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4820                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4821                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4822                                /*highaddr*/BUS_SPACE_MAXADDR,
4823                                /*filter*/NULL, /*filterarg*/NULL,
4824                                driver_data_size,
4825                                /*nsegments*/1,
4826                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4827                                /*flags*/0, &ahc->shared_data_dmat) != 0) {
4828                 return (ENOMEM);
4829         }
4830
4831         ahc->init_level++;
4832
4833         /* Allocation of driver data */
4834         if (ahc_dmamem_alloc(ahc, ahc->shared_data_dmat,
4835                              (void **)&ahc->qoutfifo,
4836                              BUS_DMA_NOWAIT, &ahc->shared_data_dmamap) != 0) {
4837                 return (ENOMEM);
4838         }
4839
4840         ahc->init_level++;
4841
4842         /* And permanently map it in */
4843         ahc_dmamap_load(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
4844                         ahc->qoutfifo, driver_data_size, ahc_dmamap_cb,
4845                         &ahc->shared_data_busaddr, /*flags*/0);
4846
4847         if ((ahc->features & AHC_TARGETMODE) != 0) {
4848                 ahc->targetcmds = (struct target_cmd *)ahc->qoutfifo;
4849                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[AHC_TMODE_CMDS];
4850                 ahc->dma_bug_buf = ahc->shared_data_busaddr
4851                                  + driver_data_size - 1;
4852                 /* All target command blocks start out invalid. */
4853                 for (i = 0; i < AHC_TMODE_CMDS; i++)
4854                         ahc->targetcmds[i].cmd_valid = 0;
4855                 ahc_sync_tqinfifo(ahc, BUS_DMASYNC_PREREAD);
4856                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[256];
4857         }
4858         ahc->qinfifo = &ahc->qoutfifo[256];
4859
4860         ahc->init_level++;
4861
4862         /* Allocate SCB data now that buffer_dmat is initialized */
4863         if (ahc->scb_data->maxhscbs == 0)
4864                 if (ahc_init_scbdata(ahc) != 0)
4865                         return (ENOMEM);
4866
4867         /*
4868          * Allocate a tstate to house information for our
4869          * initiator presence on the bus as well as the user
4870          * data for any target mode initiator.
4871          */
4872         if (ahc_alloc_tstate(ahc, ahc->our_id, 'A') == NULL) {
4873                 kprintf("%s: unable to allocate ahc_tmode_tstate.  "
4874                        "Failing attach\n", ahc_name(ahc));
4875                 return (ENOMEM);
4876         }
4877
4878         if ((ahc->features & AHC_TWIN) != 0) {
4879                 if (ahc_alloc_tstate(ahc, ahc->our_id_b, 'B') == NULL) {
4880                         kprintf("%s: unable to allocate ahc_tmode_tstate.  "
4881                                "Failing attach\n", ahc_name(ahc));
4882                         return (ENOMEM);
4883                 }
4884         }
4885
4886         if (ahc->scb_data->maxhscbs < AHC_SCB_MAX_ALLOC) {
4887                 ahc->flags |= AHC_PAGESCBS;
4888         } else {
4889                 ahc->flags &= ~AHC_PAGESCBS;
4890         }
4891
4892 #ifdef AHC_DEBUG
4893         if (ahc_debug & AHC_SHOW_MISC) {
4894                 kprintf("%s: hardware scb %u bytes; kernel scb %u bytes; "
4895                        "ahc_dma %u bytes\n",
4896                         ahc_name(ahc),
4897                         (u_int)sizeof(struct hardware_scb),
4898                         (u_int)sizeof(struct scb),
4899                         (u_int)sizeof(struct ahc_dma_seg));
4900         }
4901 #endif /* AHC_DEBUG */
4902
4903         /*
4904          * Look at the information that board initialization or
4905          * the board bios has left us.
4906          */
4907         if (ahc->features & AHC_TWIN) {
4908                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
4909                 if ((scsi_conf & RESET_SCSI) != 0
4910                  && (ahc->flags & AHC_INITIATORROLE) != 0)
4911                         ahc->flags |= AHC_RESET_BUS_B;
4912         }
4913
4914         scsi_conf = ahc_inb(ahc, SCSICONF);
4915         if ((scsi_conf & RESET_SCSI) != 0
4916          && (ahc->flags & AHC_INITIATORROLE) != 0)
4917                 ahc->flags |= AHC_RESET_BUS_A;
4918
4919         ultraenb = 0;
4920         tagenable = ALL_TARGETS_MASK;
4921
4922         /* Grab the disconnection disable table and invert it for our needs */
4923         if ((ahc->flags & AHC_USEDEFAULTS) != 0) {
4924                 kprintf("%s: Host Adapter Bios disabled.  Using default SCSI "
4925                         "device parameters\n", ahc_name(ahc));
4926                 ahc->flags |= AHC_EXTENDED_TRANS_A|AHC_EXTENDED_TRANS_B|
4927                               AHC_TERM_ENB_A|AHC_TERM_ENB_B;
4928                 discenable = ALL_TARGETS_MASK;
4929                 if ((ahc->features & AHC_ULTRA) != 0)
4930                         ultraenb = ALL_TARGETS_MASK;
4931         } else {
4932                 discenable = ~((ahc_inb(ahc, DISC_DSB + 1) << 8)
4933                            | ahc_inb(ahc, DISC_DSB));
4934                 if ((ahc->features & (AHC_ULTRA|AHC_ULTRA2)) != 0)
4935                         ultraenb = (ahc_inb(ahc, ULTRA_ENB + 1) << 8)
4936                                       | ahc_inb(ahc, ULTRA_ENB);
4937         }
4938
4939         if ((ahc->features & (AHC_WIDE|AHC_TWIN)) == 0)
4940                 max_targ = 7;
4941
4942         for (i = 0; i <= max_targ; i++) {
4943                 struct ahc_initiator_tinfo *tinfo;
4944                 struct ahc_tmode_tstate *tstate;
4945                 u_int our_id;
4946                 u_int target_id;
4947                 char channel;
4948
4949                 channel = 'A';
4950                 our_id = ahc->our_id;
4951                 target_id = i;
4952                 if (i > 7 && (ahc->features & AHC_TWIN) != 0) {
4953                         channel = 'B';
4954                         our_id = ahc->our_id_b;
4955                         target_id = i % 8;
4956                 }
4957                 tinfo = ahc_fetch_transinfo(ahc, channel, our_id,
4958                                             target_id, &tstate);
4959                 /* Default to async narrow across the board */
4960                 memset(tinfo, 0, sizeof(*tinfo));
4961                 if (ahc->flags & AHC_USEDEFAULTS) {
4962                         if ((ahc->features & AHC_WIDE) != 0)
4963                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
4964
4965                         /*
4966                          * These will be truncated when we determine the
4967                          * connection type we have with the target.
4968                          */
4969                         tinfo->user.period = ahc_syncrates->period;
4970                         tinfo->user.offset = MAX_OFFSET;
4971                 } else {
4972                         u_int scsirate;
4973                         uint16_t mask;
4974
4975                         /* Take the settings leftover in scratch RAM. */
4976                         scsirate = ahc_inb(ahc, TARG_SCSIRATE + i);
4977                         mask = (0x01 << i);
4978                         if ((ahc->features & AHC_ULTRA2) != 0) {
4979                                 u_int offset;
4980                                 u_int maxsync;
4981
4982                                 if ((scsirate & SOFS) == 0x0F) {
4983                                         /*
4984                                          * Haven't negotiated yet,
4985                                          * so the format is different.
4986                                          */
4987                                         scsirate = (scsirate & SXFR) >> 4
4988                                                  | (ultraenb & mask)
4989                                                   ? 0x08 : 0x0
4990                                                  | (scsirate & WIDEXFER);
4991                                         offset = MAX_OFFSET_ULTRA2;
4992                                 } else
4993                                         offset = ahc_inb(ahc, TARG_OFFSET + i);
4994                                 if ((scsirate & ~WIDEXFER) == 0 && offset != 0)
4995                                         /* Set to the lowest sync rate, 5MHz */
4996                                         scsirate |= 0x1c;
4997                                 maxsync = AHC_SYNCRATE_ULTRA2;
4998                                 if ((ahc->features & AHC_DT) != 0)
4999                                         maxsync = AHC_SYNCRATE_DT;
5000                                 tinfo->user.period =
5001                                     ahc_find_period(ahc, scsirate, maxsync);
5002                                 if (offset == 0)
5003                                         tinfo->user.period = 0;
5004                                 else
5005                                         tinfo->user.offset = MAX_OFFSET;
5006                                 if ((scsirate & SXFR_ULTRA2) <= 8/*10MHz*/
5007                                  && (ahc->features & AHC_DT) != 0)
5008                                         tinfo->user.ppr_options =
5009                                             MSG_EXT_PPR_DT_REQ;
5010                         } else if ((scsirate & SOFS) != 0) {
5011                                 if ((scsirate & SXFR) == 0x40
5012                                  && (ultraenb & mask) != 0) {
5013                                         /* Treat 10MHz as a non-ultra speed */
5014                                         scsirate &= ~SXFR;
5015                                         ultraenb &= ~mask;
5016                                 }
5017                                 tinfo->user.period =
5018                                     ahc_find_period(ahc, scsirate,
5019                                                     (ultraenb & mask)
5020                                                    ? AHC_SYNCRATE_ULTRA
5021                                                    : AHC_SYNCRATE_FAST);
5022                                 if (tinfo->user.period != 0)
5023                                         tinfo->user.offset = MAX_OFFSET;
5024                         }
5025                         if (tinfo->user.period == 0)
5026                                 tinfo->user.offset = 0;
5027                         if ((scsirate & WIDEXFER) != 0
5028                          && (ahc->features & AHC_WIDE) != 0)
5029                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5030                         tinfo->user.protocol_version = 4;
5031                         if ((ahc->features & AHC_DT) != 0)
5032                                 tinfo->user.transport_version = 3;
5033                         else
5034                                 tinfo->user.transport_version = 2;
5035                         tinfo->goal.protocol_version = 2;
5036                         tinfo->goal.transport_version = 2;
5037                         tinfo->curr.protocol_version = 2;
5038                         tinfo->curr.transport_version = 2;
5039                 }
5040                 tstate->ultraenb = 0;
5041         }
5042         ahc->user_discenable = discenable;
5043         ahc->user_tagenable = tagenable;
5044
5045         return (ahc->bus_chip_init(ahc));
5046 }
5047
5048 void
5049 ahc_intr_enable(struct ahc_softc *ahc, int enable)
5050 {
5051         u_int hcntrl;
5052
5053         hcntrl = ahc_inb(ahc, HCNTRL);
5054         hcntrl &= ~INTEN;
5055         ahc->pause &= ~INTEN;
5056         ahc->unpause &= ~INTEN;
5057         if (enable) {
5058                 hcntrl |= INTEN;
5059                 ahc->pause |= INTEN;
5060                 ahc->unpause |= INTEN;
5061         }
5062         ahc_outb(ahc, HCNTRL, hcntrl);
5063 }
5064
5065 /*
5066  * Ensure that the card is paused in a location
5067  * outside of all critical sections and that all
5068  * pending work is completed prior to returning.
5069  * This routine should only be called from outside
5070  * an interrupt context.
5071  */
5072 void
5073 ahc_pause_and_flushwork(struct ahc_softc *ahc)
5074 {
5075         int intstat;
5076         int maxloops;
5077         int paused;
5078
5079         maxloops = 1000;
5080         ahc->flags |= AHC_ALL_INTERRUPTS;
5081         paused = FALSE;
5082         do {
5083                 if (paused)
5084                         ahc_unpause(ahc);
5085                 ahc_intr(ahc);
5086                 ahc_pause(ahc);
5087                 paused = TRUE;
5088                 ahc_outb(ahc, SCSISEQ, ahc_inb(ahc, SCSISEQ) & ~ENSELO);
5089                 ahc_clear_critical_section(ahc);
5090                 intstat = ahc_inb(ahc, INTSTAT);
5091         } while (--maxloops
5092               && (intstat != 0xFF || (ahc->features & AHC_REMOVABLE) == 0)
5093               && ((intstat & INT_PEND) != 0
5094                || (ahc_inb(ahc, SSTAT0) & (SELDO|SELINGO)) != 0));
5095         if (maxloops == 0) {
5096                 kprintf("Infinite interrupt loop, INTSTAT = %x",
5097                        ahc_inb(ahc, INTSTAT));
5098         }
5099         ahc_platform_flushwork(ahc);
5100         ahc->flags &= ~AHC_ALL_INTERRUPTS;
5101 }
5102
5103 int
5104 ahc_suspend(struct ahc_softc *ahc)
5105 {
5106
5107         ahc_pause_and_flushwork(ahc);
5108
5109         if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
5110                 ahc_unpause(ahc);
5111                 return (EBUSY);
5112         }
5113
5114 #ifdef AHC_TARGET_MODE
5115         /*
5116          * XXX What about ATIOs that have not yet been serviced?
5117          * Perhaps we should just refuse to be suspended if we
5118          * are acting in a target role.
5119          */
5120         if (ahc->pending_device != NULL) {
5121                 ahc_unpause(ahc);
5122                 return (EBUSY);
5123         }
5124 #endif
5125         ahc_shutdown(ahc);
5126         return (0);
5127 }
5128
5129 int
5130 ahc_resume(struct ahc_softc *ahc)
5131 {
5132
5133         ahc_reset(ahc, /*reinit*/TRUE);
5134         ahc_intr_enable(ahc, TRUE);
5135         ahc_restart(ahc);
5136         return (0);
5137 }
5138
5139 /************************** Busy Target Table *********************************/
5140 /*
5141  * Return the untagged transaction id for a given target/channel lun.
5142  * Optionally, clear the entry.
5143  */
5144 u_int
5145 ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl)
5146 {
5147         u_int scbid;
5148         u_int target_offset;
5149
5150         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5151                 u_int saved_scbptr;
5152
5153                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5154                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5155                 scbid = ahc_inb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl));
5156                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5157         } else {
5158                 target_offset = TCL_TARGET_OFFSET(tcl);
5159                 scbid = ahc_inb(ahc, BUSY_TARGETS + target_offset);
5160         }
5161
5162         return (scbid);
5163 }
5164
5165 void
5166 ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl)
5167 {
5168         u_int target_offset;
5169
5170         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5171                 u_int saved_scbptr;
5172
5173                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5174                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5175                 ahc_outb(ahc, SCB_64_BTT+TCL_TARGET_OFFSET(tcl), SCB_LIST_NULL);
5176                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5177         } else {
5178                 target_offset = TCL_TARGET_OFFSET(tcl);
5179                 ahc_outb(ahc, BUSY_TARGETS + target_offset, SCB_LIST_NULL);
5180         }
5181 }
5182
5183 void
5184 ahc_busy_tcl(struct ahc_softc *ahc, u_int tcl, u_int scbid)
5185 {
5186         u_int target_offset;
5187
5188         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5189                 u_int saved_scbptr;
5190
5191                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5192                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5193                 ahc_outb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl), scbid);
5194                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5195         } else {
5196                 target_offset = TCL_TARGET_OFFSET(tcl);
5197                 ahc_outb(ahc, BUSY_TARGETS + target_offset, scbid);
5198         }
5199 }
5200
5201 /************************** SCB and SCB queue management **********************/
5202 int
5203 ahc_match_scb(struct ahc_softc *ahc, struct scb *scb, int target,
5204               char channel, int lun, u_int tag, role_t role)
5205 {
5206         int targ = SCB_GET_TARGET(ahc, scb);
5207         char chan = SCB_GET_CHANNEL(ahc, scb);
5208         int slun = SCB_GET_LUN(scb);
5209         int match;
5210
5211         match = ((chan == channel) || (channel == ALL_CHANNELS));
5212         if (match != 0)
5213                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
5214         if (match != 0)
5215                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
5216         if (match != 0) {
5217 #ifdef AHC_TARGET_MODE
5218                 int group;
5219
5220                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
5221                 if (role == ROLE_INITIATOR) {
5222                         match = (group != XPT_FC_GROUP_TMODE)
5223                               && ((tag == scb->hscb->tag)
5224                                || (tag == SCB_LIST_NULL));
5225                 } else if (role == ROLE_TARGET) {
5226                         match = (group == XPT_FC_GROUP_TMODE)
5227                               && ((tag == scb->io_ctx->csio.tag_id)
5228                                || (tag == SCB_LIST_NULL));
5229                 }
5230 #else /* !AHC_TARGET_MODE */
5231                 match = ((tag == scb->hscb->tag) || (tag == SCB_LIST_NULL));
5232 #endif /* AHC_TARGET_MODE */
5233         }
5234
5235         return match;
5236 }
5237
5238 void
5239 ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb)
5240 {
5241         int     target;
5242         char    channel;
5243         int     lun;
5244
5245         target = SCB_GET_TARGET(ahc, scb);
5246         lun = SCB_GET_LUN(scb);
5247         channel = SCB_GET_CHANNEL(ahc, scb);
5248
5249         ahc_search_qinfifo(ahc, target, channel, lun,
5250                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
5251                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5252
5253         ahc_platform_freeze_devq(ahc, scb);
5254 }
5255
5256 void
5257 ahc_qinfifo_requeue_tail(struct ahc_softc *ahc, struct scb *scb)
5258 {
5259         struct scb *prev_scb;
5260
5261         prev_scb = NULL;
5262         if (ahc_qinfifo_count(ahc) != 0) {
5263                 u_int prev_tag;
5264                 uint8_t prev_pos;
5265
5266                 prev_pos = ahc->qinfifonext - 1;
5267                 prev_tag = ahc->qinfifo[prev_pos];
5268                 prev_scb = ahc_lookup_scb(ahc, prev_tag);
5269         }
5270         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5271         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5272                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5273         } else {
5274                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5275         }
5276 }
5277
5278 static void
5279 ahc_qinfifo_requeue(struct ahc_softc *ahc, struct scb *prev_scb,
5280                     struct scb *scb)
5281 {
5282         if (prev_scb == NULL) {
5283                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5284         } else {
5285                 prev_scb->hscb->next = scb->hscb->tag;
5286                 ahc_sync_scb(ahc, prev_scb,
5287                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5288         }
5289         ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
5290         scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5291         ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5292 }
5293
5294 static int
5295 ahc_qinfifo_count(struct ahc_softc *ahc)
5296 {
5297         uint8_t qinpos;
5298         uint8_t diff;
5299
5300         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5301                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
5302                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
5303         } else
5304                 qinpos = ahc_inb(ahc, QINPOS);
5305         diff = ahc->qinfifonext - qinpos;
5306         return (diff);
5307 }
5308
5309 int
5310 ahc_search_qinfifo(struct ahc_softc *ahc, int target, char channel,
5311                    int lun, u_int tag, role_t role, uint32_t status,
5312                    ahc_search_action action)
5313 {
5314         struct  scb *scb;
5315         struct  scb *prev_scb;
5316         uint8_t qinstart;
5317         uint8_t qinpos;
5318         uint8_t qintail;
5319         uint8_t next;
5320         uint8_t prev;
5321         uint8_t curscbptr;
5322         int     found;
5323         int     have_qregs;
5324
5325         qintail = ahc->qinfifonext;
5326         have_qregs = (ahc->features & AHC_QUEUE_REGS) != 0;
5327         if (have_qregs) {
5328                 qinstart = ahc_inb(ahc, SNSCB_QOFF);
5329                 ahc_outb(ahc, SNSCB_QOFF, qinstart);
5330         } else
5331                 qinstart = ahc_inb(ahc, QINPOS);
5332         qinpos = qinstart;
5333         found = 0;
5334         prev_scb = NULL;
5335
5336         if (action == SEARCH_COMPLETE) {
5337                 /*
5338                  * Don't attempt to run any queued untagged transactions
5339                  * until we are done with the abort process.
5340                  */
5341                 ahc_freeze_untagged_queues(ahc);
5342         }
5343
5344         /*
5345          * Start with an empty queue.  Entries that are not chosen
5346          * for removal will be re-added to the queue as we go.
5347          */
5348         ahc->qinfifonext = qinpos;
5349         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5350
5351         while (qinpos != qintail) {
5352                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinpos]);
5353                 if (scb == NULL) {
5354                         kprintf("qinpos = %d, SCB index = %d\n",
5355                                 qinpos, ahc->qinfifo[qinpos]);
5356                         panic("Loop 1\n");
5357                 }
5358
5359                 if (ahc_match_scb(ahc, scb, target, channel, lun, tag, role)) {
5360                         /*
5361                          * We found an scb that needs to be acted on.
5362                          */
5363                         found++;
5364                         switch (action) {
5365                         case SEARCH_COMPLETE:
5366                         {
5367                                 cam_status ostat;
5368                                 cam_status cstat;
5369
5370                                 ostat = ahc_get_transaction_status(scb);
5371                                 if (ostat == CAM_REQ_INPROG)
5372                                         ahc_set_transaction_status(scb, status);
5373                                 cstat = ahc_get_transaction_status(scb);
5374                                 if (cstat != CAM_REQ_CMP)
5375                                         ahc_freeze_scb(scb);
5376                                 if ((scb->flags & SCB_ACTIVE) == 0)
5377                                         kprintf("Inactive SCB in qinfifo\n");
5378                                 ahc_done(ahc, scb);
5379
5380                                 /* FALLTHROUGH */
5381                         }
5382                         case SEARCH_REMOVE:
5383                                 break;
5384                         case SEARCH_COUNT:
5385                                 ahc_qinfifo_requeue(ahc, prev_scb, scb);
5386                                 prev_scb = scb;
5387                                 break;
5388                         }
5389                 } else {
5390                         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5391                         prev_scb = scb;
5392                 }
5393                 qinpos++;
5394         }
5395
5396         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5397                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5398         } else {
5399                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5400         }
5401
5402         if (action != SEARCH_COUNT
5403          && (found != 0)
5404          && (qinstart != ahc->qinfifonext)) {
5405                 /*
5406                  * The sequencer may be in the process of dmaing
5407                  * down the SCB at the beginning of the queue.
5408                  * This could be problematic if either the first,
5409                  * or the second SCB is removed from the queue
5410                  * (the first SCB includes a pointer to the "next"
5411                  * SCB to dma). If we have removed any entries, swap
5412                  * the first element in the queue with the next HSCB
5413                  * so the sequencer will notice that NEXT_QUEUED_SCB
5414                  * has changed during its dma attempt and will retry
5415                  * the DMA.
5416                  */
5417                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinstart]);
5418
5419                 if (scb == NULL) {
5420                         kprintf("found = %d, qinstart = %d, qinfifionext = %d\n",
5421                                 found, qinstart, ahc->qinfifonext);
5422                         panic("First/Second Qinfifo fixup\n");
5423                 }
5424                 /*
5425                  * ahc_swap_with_next_hscb forces our next pointer to
5426                  * point to the reserved SCB for future commands.  Save
5427                  * and restore our original next pointer to maintain
5428                  * queue integrity.
5429                  */
5430                 next = scb->hscb->next;
5431                 ahc->scb_data->scbindex[scb->hscb->tag] = NULL;
5432                 ahc_swap_with_next_hscb(ahc, scb);
5433                 scb->hscb->next = next;
5434                 ahc->qinfifo[qinstart] = scb->hscb->tag;
5435
5436                 /* Tell the card about the new head of the qinfifo. */
5437                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5438
5439                 /* Fixup the tail "next" pointer. */
5440                 qintail = ahc->qinfifonext - 1;
5441                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qintail]);
5442                 scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5443         }
5444
5445         /*
5446          * Search waiting for selection list.
5447          */
5448         curscbptr = ahc_inb(ahc, SCBPTR);
5449         next = ahc_inb(ahc, WAITING_SCBH);  /* Start at head of list. */
5450         prev = SCB_LIST_NULL;
5451
5452         while (next != SCB_LIST_NULL) {
5453                 uint8_t scb_index;
5454
5455                 ahc_outb(ahc, SCBPTR, next);
5456                 scb_index = ahc_inb(ahc, SCB_TAG);
5457                 if (scb_index >= ahc->scb_data->numscbs) {
5458                         kprintf("Waiting List inconsistency. "
5459                                "SCB index == %d, yet numscbs == %d.",
5460                                scb_index, ahc->scb_data->numscbs);
5461                         ahc_dump_card_state(ahc);
5462                         panic("for safety");
5463                 }
5464                 scb = ahc_lookup_scb(ahc, scb_index);
5465                 if (scb == NULL) {
5466                         kprintf("scb_index = %d, next = %d\n",
5467                                 scb_index, next);
5468                         panic("Waiting List traversal\n");
5469                 }
5470                 if (ahc_match_scb(ahc, scb, target, channel,
5471                                   lun, SCB_LIST_NULL, role)) {
5472                         /*
5473                          * We found an scb that needs to be acted on.
5474                          */
5475                         found++;
5476                         switch (action) {
5477                         case SEARCH_COMPLETE:
5478                         {
5479                                 cam_status ostat;
5480                                 cam_status cstat;
5481
5482                                 ostat = ahc_get_transaction_status(scb);
5483                                 if (ostat == CAM_REQ_INPROG)
5484                                         ahc_set_transaction_status(scb,
5485                                                                    status);
5486                                 cstat = ahc_get_transaction_status(scb);
5487                                 if (cstat != CAM_REQ_CMP)
5488                                         ahc_freeze_scb(scb);
5489                                 if ((scb->flags & SCB_ACTIVE) == 0)
5490                                         kprintf("Inactive SCB in Waiting List\n");
5491                                 ahc_done(ahc, scb);
5492                                 /* FALLTHROUGH */
5493                         }
5494                         case SEARCH_REMOVE:
5495                                 next = ahc_rem_wscb(ahc, next, prev);
5496                                 break;
5497                         case SEARCH_COUNT:
5498                                 prev = next;
5499                                 next = ahc_inb(ahc, SCB_NEXT);
5500                                 break;
5501                         }
5502                 } else {
5503
5504                         prev = next;
5505                         next = ahc_inb(ahc, SCB_NEXT);
5506                 }
5507         }
5508         ahc_outb(ahc, SCBPTR, curscbptr);
5509
5510         found += ahc_search_untagged_queues(ahc, /*ahc_io_ctx_t*/NULL, target,
5511                                             channel, lun, status, action);
5512
5513         if (action == SEARCH_COMPLETE)
5514                 ahc_release_untagged_queues(ahc);
5515         return (found);
5516 }
5517
5518 int
5519 ahc_search_untagged_queues(struct ahc_softc *ahc, ahc_io_ctx_t ctx,
5520                            int target, char channel, int lun, uint32_t status,
5521                            ahc_search_action action)
5522 {
5523         struct  scb *scb;
5524         int     maxtarget;
5525         int     found;
5526         int     i;
5527
5528         if (action == SEARCH_COMPLETE) {
5529                 /*
5530                  * Don't attempt to run any queued untagged transactions
5531                  * until we are done with the abort process.
5532                  */
5533                 ahc_freeze_untagged_queues(ahc);
5534         }
5535
5536         found = 0;
5537         i = 0;
5538         if ((ahc->flags & AHC_SCB_BTT) == 0) {
5539
5540                 maxtarget = 16;
5541                 if (target != CAM_TARGET_WILDCARD) {
5542
5543                         i = target;
5544                         if (channel == 'B')
5545                                 i += 8;
5546                         maxtarget = i + 1;
5547                 }
5548         } else {
5549                 maxtarget = 0;
5550         }
5551
5552         for (; i < maxtarget; i++) {
5553                 struct scb_tailq *untagged_q;
5554                 struct scb *next_scb;
5555
5556                 untagged_q = &(ahc->untagged_queues[i]);
5557                 next_scb = TAILQ_FIRST(untagged_q);
5558                 while (next_scb != NULL) {
5559
5560                         scb = next_scb;
5561                         next_scb = TAILQ_NEXT(scb, links.tqe);
5562
5563                         /*
5564                          * The head of the list may be the currently
5565                          * active untagged command for a device.
5566                          * We're only searching for commands that
5567                          * have not been started.  A transaction
5568                          * marked active but still in the qinfifo
5569                          * is removed by the qinfifo scanning code
5570                          * above.
5571                          */
5572                         if ((scb->flags & SCB_ACTIVE) != 0)
5573                                 continue;
5574
5575                         if (ahc_match_scb(ahc, scb, target, channel, lun,
5576                                           SCB_LIST_NULL, ROLE_INITIATOR) == 0
5577                          || (ctx != NULL && ctx != scb->io_ctx))
5578                                 continue;
5579
5580                         /*
5581                          * We found an scb that needs to be acted on.
5582                          */
5583                         found++;
5584                         switch (action) {
5585                         case SEARCH_COMPLETE:
5586                         {
5587                                 cam_status ostat;
5588                                 cam_status cstat;
5589
5590                                 ostat = ahc_get_transaction_status(scb);
5591                                 if (ostat == CAM_REQ_INPROG)
5592                                         ahc_set_transaction_status(scb, status);
5593                                 cstat = ahc_get_transaction_status(scb);
5594                                 if (cstat != CAM_REQ_CMP)
5595                                         ahc_freeze_scb(scb);
5596                                 if ((scb->flags & SCB_ACTIVE) == 0)
5597                                         kprintf("Inactive SCB in untaggedQ\n");
5598                                 ahc_done(ahc, scb);
5599                                 break;
5600                         }
5601                         case SEARCH_REMOVE:
5602                                 scb->flags &= ~SCB_UNTAGGEDQ;
5603                                 TAILQ_REMOVE(untagged_q, scb, links.tqe);
5604                                 break;
5605                         case SEARCH_COUNT:
5606                                 break;
5607                         }
5608                 }
5609         }
5610
5611         if (action == SEARCH_COMPLETE)
5612                 ahc_release_untagged_queues(ahc);
5613         return (found);
5614 }
5615
5616 int
5617 ahc_search_disc_list(struct ahc_softc *ahc, int target, char channel,
5618                      int lun, u_int tag, int stop_on_first, int remove,
5619                      int save_state)
5620 {
5621         struct  scb *scbp;
5622         u_int   next;
5623         u_int   prev;
5624         u_int   count;
5625         u_int   active_scb;
5626
5627         count = 0;
5628         next = ahc_inb(ahc, DISCONNECTED_SCBH);
5629         prev = SCB_LIST_NULL;
5630
5631         if (save_state) {
5632                 /* restore this when we're done */
5633                 active_scb = ahc_inb(ahc, SCBPTR);
5634         } else
5635                 /* Silence compiler */
5636                 active_scb = SCB_LIST_NULL;
5637
5638         while (next != SCB_LIST_NULL) {
5639                 u_int scb_index;
5640
5641                 ahc_outb(ahc, SCBPTR, next);
5642                 scb_index = ahc_inb(ahc, SCB_TAG);
5643                 if (scb_index >= ahc->scb_data->numscbs) {
5644                         kprintf("Disconnected List inconsistency. "
5645                                "SCB index == %d, yet numscbs == %d.",
5646                                scb_index, ahc->scb_data->numscbs);
5647                         ahc_dump_card_state(ahc);
5648                         panic("for safety");
5649                 }
5650
5651                 if (next == prev) {
5652                         panic("Disconnected List Loop. "
5653                               "cur SCBPTR == %x, prev SCBPTR == %x.",
5654                               next, prev);
5655                 }
5656                 scbp = ahc_lookup_scb(ahc, scb_index);
5657                 if (ahc_match_scb(ahc, scbp, target, channel, lun,
5658                                   tag, ROLE_INITIATOR)) {
5659                         count++;
5660                         if (remove) {
5661                                 next =
5662                                     ahc_rem_scb_from_disc_list(ahc, prev, next);
5663                         } else {
5664                                 prev = next;
5665                                 next = ahc_inb(ahc, SCB_NEXT);
5666                         }
5667                         if (stop_on_first)
5668                                 break;
5669                 } else {
5670                         prev = next;
5671                         next = ahc_inb(ahc, SCB_NEXT);
5672                 }
5673         }
5674         if (save_state)
5675                 ahc_outb(ahc, SCBPTR, active_scb);
5676         return (count);
5677 }
5678
5679 /*
5680  * Remove an SCB from the on chip list of disconnected transactions.
5681  * This is empty/unused if we are not performing SCB paging.
5682  */
5683 static u_int
5684 ahc_rem_scb_from_disc_list(struct ahc_softc *ahc, u_int prev, u_int scbptr)
5685 {
5686         u_int next;
5687
5688         ahc_outb(ahc, SCBPTR, scbptr);
5689         next = ahc_inb(ahc, SCB_NEXT);
5690
5691         ahc_outb(ahc, SCB_CONTROL, 0);
5692
5693         ahc_add_curscb_to_free_list(ahc);
5694
5695         if (prev != SCB_LIST_NULL) {
5696                 ahc_outb(ahc, SCBPTR, prev);
5697                 ahc_outb(ahc, SCB_NEXT, next);
5698         } else
5699                 ahc_outb(ahc, DISCONNECTED_SCBH, next);
5700
5701         return (next);
5702 }
5703
5704 /*
5705  * Add the SCB as selected by SCBPTR onto the on chip list of
5706  * free hardware SCBs.  This list is empty/unused if we are not
5707  * performing SCB paging.
5708  */
5709 static void
5710 ahc_add_curscb_to_free_list(struct ahc_softc *ahc)
5711 {
5712         /*
5713          * Invalidate the tag so that our abort
5714          * routines don't think it's active.
5715          */
5716         ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
5717
5718         if ((ahc->flags & AHC_PAGESCBS) != 0) {
5719                 ahc_outb(ahc, SCB_NEXT, ahc_inb(ahc, FREE_SCBH));
5720                 ahc_outb(ahc, FREE_SCBH, ahc_inb(ahc, SCBPTR));
5721         }
5722 }
5723
5724 /*
5725  * Manipulate the waiting for selection list and return the
5726  * scb that follows the one that we remove.
5727  */
5728 static u_int
5729 ahc_rem_wscb(struct ahc_softc *ahc, u_int scbpos, u_int prev)
5730 {
5731         u_int curscb, next;
5732
5733         /*
5734          * Select the SCB we want to abort and
5735          * pull the next pointer out of it.
5736          */
5737         curscb = ahc_inb(ahc, SCBPTR);
5738         ahc_outb(ahc, SCBPTR, scbpos);
5739         next = ahc_inb(ahc, SCB_NEXT);
5740
5741         /* Clear the necessary fields */
5742         ahc_outb(ahc, SCB_CONTROL, 0);
5743
5744         ahc_add_curscb_to_free_list(ahc);
5745
5746         /* update the waiting list */
5747         if (prev == SCB_LIST_NULL) {
5748                 /* First in the list */
5749                 ahc_outb(ahc, WAITING_SCBH, next);
5750
5751                 /*
5752                  * Ensure we aren't attempting to perform
5753                  * selection for this entry.
5754                  */
5755                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
5756         } else {
5757                 /*
5758                  * Select the scb that pointed to us
5759                  * and update its next pointer.
5760                  */
5761                 ahc_outb(ahc, SCBPTR, prev);
5762                 ahc_outb(ahc, SCB_NEXT, next);
5763         }
5764
5765         /*
5766          * Point us back at the original scb position.
5767          */
5768         ahc_outb(ahc, SCBPTR, curscb);
5769         return next;
5770 }
5771
5772 /******************************** Error Handling ******************************/
5773 /*
5774  * Abort all SCBs that match the given description (target/channel/lun/tag),
5775  * setting their status to the passed in status if the status has not already
5776  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
5777  * is paused before it is called.
5778  */
5779 int
5780 ahc_abort_scbs(struct ahc_softc *ahc, int target, char channel,
5781                int lun, u_int tag, role_t role, uint32_t status)
5782 {
5783         struct  scb *scbp;
5784         struct  scb *scbp_next;
5785         u_int   active_scb;
5786         int     i, j;
5787         int     maxtarget;
5788         int     minlun;
5789         int     maxlun;
5790
5791         int     found;
5792
5793         /*
5794          * Don't attempt to run any queued untagged transactions
5795          * until we are done with the abort process.
5796          */
5797         ahc_freeze_untagged_queues(ahc);
5798
5799         /* restore this when we're done */
5800         active_scb = ahc_inb(ahc, SCBPTR);
5801
5802         found = ahc_search_qinfifo(ahc, target, channel, lun, SCB_LIST_NULL,
5803                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5804
5805         /*
5806          * Clean out the busy target table for any untagged commands.
5807          */
5808         i = 0;
5809         maxtarget = 16;
5810         if (target != CAM_TARGET_WILDCARD) {
5811                 i = target;
5812                 if (channel == 'B')
5813                         i += 8;
5814                 maxtarget = i + 1;
5815         }
5816
5817         if (lun == CAM_LUN_WILDCARD) {
5818
5819                 /*
5820                  * Unless we are using an SCB based
5821                  * busy targets table, there is only
5822                  * one table entry for all luns of
5823                  * a target.
5824                  */
5825                 minlun = 0;
5826                 maxlun = 1;
5827                 if ((ahc->flags & AHC_SCB_BTT) != 0)
5828                         maxlun = AHC_NUM_LUNS;
5829         } else {
5830                 minlun = lun;
5831                 maxlun = lun + 1;
5832         }
5833
5834         if (role != ROLE_TARGET) {
5835                 for (;i < maxtarget; i++) {
5836                         for (j = minlun;j < maxlun; j++) {
5837                                 u_int scbid;
5838                                 u_int tcl;
5839
5840                                 tcl = BUILD_TCL(i << 4, j);
5841                                 scbid = ahc_index_busy_tcl(ahc, tcl);
5842                                 scbp = ahc_lookup_scb(ahc, scbid);
5843                                 if (scbp == NULL
5844                                  || ahc_match_scb(ahc, scbp, target, channel,
5845                                                   lun, tag, role) == 0)
5846                                         continue;
5847                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, j));
5848                         }
5849                 }
5850
5851                 /*
5852                  * Go through the disconnected list and remove any entries we
5853                  * have queued for completion, 0'ing their control byte too.
5854                  * We save the active SCB and restore it ourselves, so there
5855                  * is no reason for this search to restore it too.
5856                  */
5857                 ahc_search_disc_list(ahc, target, channel, lun, tag,
5858                                      /*stop_on_first*/FALSE, /*remove*/TRUE,
5859                                      /*save_state*/FALSE);
5860         }
5861
5862         /*
5863          * Go through the hardware SCB array looking for commands that
5864          * were active but not on any list.  In some cases, these remnants
5865          * might not still have mappings in the scbindex array (e.g. unexpected
5866          * bus free with the same scb queued for an abort).  Don't hold this
5867          * against them.
5868          */
5869         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
5870                 u_int scbid;
5871
5872                 ahc_outb(ahc, SCBPTR, i);
5873                 scbid = ahc_inb(ahc, SCB_TAG);
5874                 scbp = ahc_lookup_scb(ahc, scbid);
5875                 if ((scbp == NULL && scbid != SCB_LIST_NULL)
5876                  || (scbp != NULL
5877                   && ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)))
5878                         ahc_add_curscb_to_free_list(ahc);
5879         }
5880
5881         /*
5882          * Go through the pending CCB list and look for
5883          * commands for this target that are still active.
5884          * These are other tagged commands that were
5885          * disconnected when the reset occurred.
5886          */
5887         scbp_next = LIST_FIRST(&ahc->pending_scbs);
5888         while (scbp_next != NULL) {
5889                 scbp = scbp_next;
5890                 scbp_next = LIST_NEXT(scbp, pending_links);
5891                 if (ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)) {
5892                         cam_status ostat;
5893
5894                         ostat = ahc_get_transaction_status(scbp);
5895                         if (ostat == CAM_REQ_INPROG)
5896                                 ahc_set_transaction_status(scbp, status);
5897                         if (ahc_get_transaction_status(scbp) != CAM_REQ_CMP)
5898                                 ahc_freeze_scb(scbp);
5899                         if ((scbp->flags & SCB_ACTIVE) == 0)
5900                                 kprintf("Inactive SCB on pending list\n");
5901                         ahc_done(ahc, scbp);
5902                         found++;
5903                 }
5904         }
5905         ahc_outb(ahc, SCBPTR, active_scb);
5906         ahc_platform_abort_scbs(ahc, target, channel, lun, tag, role, status);
5907         ahc_release_untagged_queues(ahc);
5908         return found;
5909 }
5910
5911 static void
5912 ahc_reset_current_bus(struct ahc_softc *ahc)
5913 {
5914         uint8_t scsiseq;
5915
5916         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENSCSIRST);
5917         scsiseq = ahc_inb(ahc, SCSISEQ);
5918         ahc_outb(ahc, SCSISEQ, scsiseq | SCSIRSTO);
5919         ahc_flush_device_writes(ahc);
5920         ahc_delay(AHC_BUSRESET_DELAY);
5921         /* Turn off the bus reset */
5922         ahc_outb(ahc, SCSISEQ, scsiseq & ~SCSIRSTO);
5923
5924         ahc_clear_intstat(ahc);
5925
5926         /* Re-enable reset interrupts */
5927         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) | ENSCSIRST);
5928 }
5929
5930 int
5931 ahc_reset_channel(struct ahc_softc *ahc, char channel, int initiate_reset)
5932 {
5933         struct  ahc_devinfo devinfo;
5934         u_int   initiator, target, max_scsiid;
5935         u_int   sblkctl;
5936         u_int   scsiseq;
5937         u_int   simode1;
5938         int     found;
5939         int     restart_needed;
5940         char    cur_channel;
5941
5942         ahc->pending_device = NULL;
5943
5944         ahc_compile_devinfo(&devinfo,
5945                             CAM_TARGET_WILDCARD,
5946                             CAM_TARGET_WILDCARD,
5947                             CAM_LUN_WILDCARD,
5948                             channel, ROLE_UNKNOWN);
5949         ahc_pause(ahc);
5950
5951         /* Make sure the sequencer is in a safe location. */
5952         ahc_clear_critical_section(ahc);
5953
5954         /*
5955          * Run our command complete fifos to ensure that we perform
5956          * completion processing on any commands that 'completed'
5957          * before the reset occurred.
5958          */
5959         ahc_run_qoutfifo(ahc);
5960 #ifdef AHC_TARGET_MODE
5961         /*
5962          * XXX - In Twin mode, the tqinfifo may have commands
5963          *       for an unaffected channel in it.  However, if
5964          *       we have run out of ATIO resources to drain that
5965          *       queue, we may not get them all out here.  Further,
5966          *       the blocked transactions for the reset channel
5967          *       should just be killed off, irrespecitve of whether
5968          *       we are blocked on ATIO resources.  Write a routine
5969          *       to compact the tqinfifo appropriately.
5970          */
5971         if ((ahc->flags & AHC_TARGETROLE) != 0) {
5972                 ahc_run_tqinfifo(ahc, /*paused*/TRUE);
5973         }
5974 #endif
5975
5976         /*
5977          * Reset the bus if we are initiating this reset
5978          */
5979         sblkctl = ahc_inb(ahc, SBLKCTL);
5980         cur_channel = 'A';
5981         if ((ahc->features & AHC_TWIN) != 0
5982          && ((sblkctl & SELBUSB) != 0))
5983             cur_channel = 'B';
5984         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
5985         if (cur_channel != channel) {
5986                 /* Case 1: Command for another bus is active
5987                  * Stealthily reset the other bus without
5988                  * upsetting the current bus.
5989                  */
5990                 ahc_outb(ahc, SBLKCTL, sblkctl ^ SELBUSB);
5991                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
5992 #ifdef AHC_TARGET_MODE
5993                 /*
5994                  * Bus resets clear ENSELI, so we cannot
5995                  * defer re-enabling bus reset interrupts
5996                  * if we are in target mode.
5997                  */
5998                 if ((ahc->flags & AHC_TARGETROLE) != 0)
5999                         simode1 |= ENSCSIRST;
6000 #endif
6001                 ahc_outb(ahc, SIMODE1, simode1);
6002                 if (initiate_reset)
6003                         ahc_reset_current_bus(ahc);
6004                 ahc_clear_intstat(ahc);
6005                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6006                 ahc_outb(ahc, SBLKCTL, sblkctl);
6007                 restart_needed = FALSE;
6008         } else {
6009                 /* Case 2: A command from this bus is active or we're idle */
6010                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6011 #ifdef AHC_TARGET_MODE
6012                 /*
6013                  * Bus resets clear ENSELI, so we cannot
6014                  * defer re-enabling bus reset interrupts
6015                  * if we are in target mode.
6016                  */
6017                 if ((ahc->flags & AHC_TARGETROLE) != 0)
6018                         simode1 |= ENSCSIRST;
6019 #endif
6020                 ahc_outb(ahc, SIMODE1, simode1);
6021                 if (initiate_reset)
6022                         ahc_reset_current_bus(ahc);
6023                 ahc_clear_intstat(ahc);
6024                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6025                 restart_needed = TRUE;
6026         }
6027
6028         /*
6029          * Clean up all the state information for the
6030          * pending transactions on this bus.
6031          */
6032         found = ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, channel,
6033                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
6034                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6035
6036         max_scsiid = (ahc->features & AHC_WIDE) ? 15 : 7;
6037
6038 #ifdef AHC_TARGET_MODE
6039         /*
6040          * Send an immediate notify ccb to all target more peripheral
6041          * drivers affected by this action.
6042          */
6043         for (target = 0; target <= max_scsiid; target++) {
6044                 struct ahc_tmode_tstate* tstate;
6045                 u_int lun;
6046
6047                 tstate = ahc->enabled_targets[target];
6048                 if (tstate == NULL)
6049                         continue;
6050                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
6051                         struct ahc_tmode_lstate* lstate;
6052
6053                         lstate = tstate->enabled_luns[lun];
6054                         if (lstate == NULL)
6055                                 continue;
6056
6057                         ahc_queue_lstate_event(ahc, lstate, CAM_TARGET_WILDCARD,
6058                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
6059                         ahc_send_lstate_events(ahc, lstate);
6060                 }
6061         }
6062 #endif
6063         /* Notify the XPT that a bus reset occurred */
6064         ahc_send_async(ahc, devinfo.channel, CAM_TARGET_WILDCARD,
6065                        CAM_LUN_WILDCARD, AC_BUS_RESET, NULL);
6066
6067         /*
6068          * Revert to async/narrow transfers until we renegotiate.
6069          */
6070         for (target = 0; target <= max_scsiid; target++) {
6071
6072                 if (ahc->enabled_targets[target] == NULL)
6073                         continue;
6074                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
6075                         struct ahc_devinfo devinfo;
6076
6077                         ahc_compile_devinfo(&devinfo, target, initiator,
6078                                             CAM_LUN_WILDCARD,
6079                                             channel, ROLE_UNKNOWN);
6080                         ahc_set_width(ahc, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6081                                       AHC_TRANS_CUR, /*paused*/TRUE);
6082                         ahc_set_syncrate(ahc, &devinfo, /*syncrate*/NULL,
6083                                          /*period*/0, /*offset*/0,
6084                                          /*ppr_options*/0, AHC_TRANS_CUR,
6085                                          /*paused*/TRUE);
6086                 }
6087         }
6088
6089         if (restart_needed)
6090                 ahc_restart(ahc);
6091         else
6092                 ahc_unpause(ahc);
6093         return found;
6094 }
6095
6096
6097 /***************************** Residual Processing ****************************/
6098 /*
6099  * Calculate the residual for a just completed SCB.
6100  */
6101 void
6102 ahc_calc_residual(struct ahc_softc *ahc, struct scb *scb)
6103 {
6104         struct hardware_scb *hscb;
6105         struct status_pkt *spkt;
6106         uint32_t sgptr;
6107         uint32_t resid_sgptr;
6108         uint32_t resid;
6109
6110         /*
6111          * 5 cases.
6112          * 1) No residual.
6113          *    SG_RESID_VALID clear in sgptr.
6114          * 2) Transferless command
6115          * 3) Never performed any transfers.
6116          *    sgptr has SG_FULL_RESID set.
6117          * 4) No residual but target did not
6118          *    save data pointers after the
6119          *    last transfer, so sgptr was
6120          *    never updated.
6121          * 5) We have a partial residual.
6122          *    Use residual_sgptr to determine
6123          *    where we are.
6124          */
6125
6126         hscb = scb->hscb;
6127         sgptr = ahc_le32toh(hscb->sgptr);
6128         if ((sgptr & SG_RESID_VALID) == 0)
6129                 /* Case 1 */
6130                 return;
6131         sgptr &= ~SG_RESID_VALID;
6132
6133         if ((sgptr & SG_LIST_NULL) != 0)
6134                 /* Case 2 */
6135                 return;
6136
6137         spkt = &hscb->shared_data.status;
6138         resid_sgptr = ahc_le32toh(spkt->residual_sg_ptr);
6139         if ((sgptr & SG_FULL_RESID) != 0) {
6140                 /* Case 3 */
6141                 resid = ahc_get_transfer_length(scb);
6142         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
6143                 /* Case 4 */
6144                 return;
6145         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
6146                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
6147         } else {
6148                 struct ahc_dma_seg *sg;
6149
6150                 /*
6151                  * Remainder of the SG where the transfer
6152                  * stopped.
6153                  */
6154                 resid = ahc_le32toh(spkt->residual_datacnt) & AHC_SG_LEN_MASK;
6155                 sg = ahc_sg_bus_to_virt(scb, resid_sgptr & SG_PTR_MASK);
6156
6157                 /* The residual sg_ptr always points to the next sg */
6158                 sg--;
6159
6160                 /*
6161                  * Add up the contents of all residual
6162                  * SG segments that are after the SG where
6163                  * the transfer stopped.
6164                  */
6165                 while ((ahc_le32toh(sg->len) & AHC_DMA_LAST_SEG) == 0) {
6166                         sg++;
6167                         resid += ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
6168                 }
6169         }
6170         if ((scb->flags & SCB_SENSE) == 0)
6171                 ahc_set_residual(scb, resid);
6172         else
6173                 ahc_set_sense_residual(scb, resid);
6174
6175 #ifdef AHC_DEBUG
6176         if ((ahc_debug & AHC_SHOW_MISC) != 0) {
6177                 ahc_print_path(ahc, scb);
6178                 kprintf("Handled %sResidual of %d bytes\n",
6179                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
6180         }
6181 #endif
6182 }
6183
6184 /******************************* Target Mode **********************************/
6185 #ifdef AHC_TARGET_MODE
6186 /*
6187  * Add a target mode event to this lun's queue
6188  */
6189 static void
6190 ahc_queue_lstate_event(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate,
6191                        u_int initiator_id, u_int event_type, u_int event_arg)
6192 {
6193         struct ahc_tmode_event *event;
6194         int pending;
6195
6196         xpt_freeze_devq(lstate->path, /*count*/1);
6197         if (lstate->event_w_idx >= lstate->event_r_idx)
6198                 pending = lstate->event_w_idx - lstate->event_r_idx;
6199         else
6200                 pending = AHC_TMODE_EVENT_BUFFER_SIZE + 1
6201                         - (lstate->event_r_idx - lstate->event_w_idx);
6202
6203         if (event_type == EVENT_TYPE_BUS_RESET
6204          || event_type == MSG_BUS_DEV_RESET) {
6205                 /*
6206                  * Any earlier events are irrelevant, so reset our buffer.
6207                  * This has the effect of allowing us to deal with reset
6208                  * floods (an external device holding down the reset line)
6209                  * without losing the event that is really interesting.
6210                  */
6211                 lstate->event_r_idx = 0;
6212                 lstate->event_w_idx = 0;
6213                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
6214         }
6215
6216         if (pending == AHC_TMODE_EVENT_BUFFER_SIZE) {
6217                 xpt_print_path(lstate->path);
6218                 kprintf("immediate event %x:%x lost\n",
6219                        lstate->event_buffer[lstate->event_r_idx].event_type,
6220                        lstate->event_buffer[lstate->event_r_idx].event_arg);
6221                 lstate->event_r_idx++;
6222                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6223                         lstate->event_r_idx = 0;
6224                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
6225         }
6226
6227         event = &lstate->event_buffer[lstate->event_w_idx];
6228         event->initiator_id = initiator_id;
6229         event->event_type = event_type;
6230         event->event_arg = event_arg;
6231         lstate->event_w_idx++;
6232         if (lstate->event_w_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6233                 lstate->event_w_idx = 0;
6234 }
6235
6236 /*
6237  * Send any target mode events queued up waiting
6238  * for immediate notify resources.
6239  */
6240 void
6241 ahc_send_lstate_events(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate)
6242 {
6243         struct ccb_hdr *ccbh;
6244         struct ccb_immed_notify *inot;
6245
6246         while (lstate->event_r_idx != lstate->event_w_idx
6247             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
6248                 struct ahc_tmode_event *event;
6249
6250                 event = &lstate->event_buffer[lstate->event_r_idx];
6251                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
6252                 inot = (struct ccb_immed_notify *)ccbh;
6253                 switch (event->event_type) {
6254                 case EVENT_TYPE_BUS_RESET:
6255                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
6256                         break;
6257                 default:
6258                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
6259                         inot->message_args[0] = event->event_type;
6260                         inot->message_args[1] = event->event_arg;
6261                         break;
6262                 }
6263                 inot->initiator_id = event->initiator_id;
6264                 inot->sense_len = 0;
6265                 xpt_done((union ccb *)inot);
6266                 lstate->event_r_idx++;
6267                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6268                         lstate->event_r_idx = 0;
6269         }
6270 }
6271 #endif
6272
6273 /******************** Sequencer Program Patching/Download *********************/
6274
6275 #ifdef AHC_DUMP_SEQ
6276 void
6277 ahc_dumpseq(struct ahc_softc* ahc)
6278 {
6279         int i;
6280
6281         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6282         ahc_outb(ahc, SEQADDR0, 0);
6283         ahc_outb(ahc, SEQADDR1, 0);
6284         for (i = 0; i < ahc->instruction_ram_size; i++) {
6285                 uint8_t ins_bytes[4];
6286
6287                 ahc_insb(ahc, SEQRAM, ins_bytes, 4);
6288                 kprintf("0x%08x\n", ins_bytes[0] << 24
6289                                  | ins_bytes[1] << 16
6290                                  | ins_bytes[2] << 8
6291                                  | ins_bytes[3]);
6292         }
6293 }
6294 #endif
6295
6296 static int
6297 ahc_loadseq(struct ahc_softc *ahc)
6298 {
6299         struct  cs cs_table[num_critical_sections];
6300         u_int   begin_set[num_critical_sections];
6301         u_int   end_set[num_critical_sections];
6302         struct  patch *cur_patch;
6303         u_int   cs_count;
6304         u_int   cur_cs;
6305         u_int   i;
6306         u_int   skip_addr;
6307         u_int   sg_prefetch_cnt;
6308         int     downloaded;
6309         uint8_t download_consts[7];
6310
6311         /*
6312          * Start out with 0 critical sections
6313          * that apply to this firmware load.
6314          */
6315         cs_count = 0;
6316         cur_cs = 0;
6317         memset(begin_set, 0, sizeof(begin_set));
6318         memset(end_set, 0, sizeof(end_set));
6319
6320         /* Setup downloadable constant table */
6321         download_consts[QOUTFIFO_OFFSET] = 0;
6322         if (ahc->targetcmds != NULL)
6323                 download_consts[QOUTFIFO_OFFSET] += 32;
6324         download_consts[QINFIFO_OFFSET] = download_consts[QOUTFIFO_OFFSET] + 1;
6325         download_consts[CACHESIZE_MASK] = ahc->pci_cachesize - 1;
6326         download_consts[INVERTED_CACHESIZE_MASK] = ~(ahc->pci_cachesize - 1);
6327         sg_prefetch_cnt = ahc->pci_cachesize;
6328         if (sg_prefetch_cnt < (2 * sizeof(struct ahc_dma_seg)))
6329                 sg_prefetch_cnt = 2 * sizeof(struct ahc_dma_seg);
6330         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
6331         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_cnt - 1);
6332         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_cnt - 1);
6333
6334         cur_patch = patches;
6335         downloaded = 0;
6336         skip_addr = 0;
6337         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6338         ahc_outb(ahc, SEQADDR0, 0);
6339         ahc_outb(ahc, SEQADDR1, 0);
6340
6341         for (i = 0; i < sizeof(seqprog)/4; i++) {
6342                 if (ahc_check_patch(ahc, &cur_patch, i, &skip_addr) == 0) {
6343                         /*
6344                          * Don't download this instruction as it
6345                          * is in a patch that was removed.
6346                          */
6347                         continue;
6348                 }
6349
6350                 if (downloaded == ahc->instruction_ram_size) {
6351                         /*
6352                          * We're about to exceed the instruction
6353                          * storage capacity for this chip.  Fail
6354                          * the load.
6355                          */
6356                         kprintf("\n%s: Program too large for instruction memory "
6357                                "size of %d!\n", ahc_name(ahc),
6358                                ahc->instruction_ram_size);
6359                         return (ENOMEM);
6360                 }
6361
6362                 /*
6363                  * Move through the CS table until we find a CS
6364                  * that might apply to this instruction.
6365                  */
6366                 for (; cur_cs < num_critical_sections; cur_cs++) {
6367                         if (critical_sections[cur_cs].end <= i) {
6368                                 if (begin_set[cs_count] == TRUE
6369                                  && end_set[cs_count] == FALSE) {
6370                                         cs_table[cs_count].end = downloaded;
6371                                         end_set[cs_count] = TRUE;
6372                                         cs_count++;
6373                                 }
6374                                 continue;
6375                         }
6376                         if (critical_sections[cur_cs].begin <= i
6377                          && begin_set[cs_count] == FALSE) {
6378                                 cs_table[cs_count].begin = downloaded;
6379                                 begin_set[cs_count] = TRUE;
6380                         }
6381                         break;
6382                 }
6383                 ahc_download_instr(ahc, i, download_consts);
6384                 downloaded++;
6385         }
6386
6387         ahc->num_critical_sections = cs_count;
6388         if (cs_count != 0) {
6389                 cs_count *= sizeof(struct cs);
6390                 ahc->critical_sections = kmalloc(cs_count, M_DEVBUF, M_INTWAIT);
6391                 memcpy(ahc->critical_sections, cs_table, cs_count);
6392         }
6393         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE);
6394
6395         if (bootverbose) {
6396                 kprintf(" %d instructions downloaded\n", downloaded);
6397                 kprintf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
6398                        ahc_name(ahc), ahc->features, ahc->bugs, ahc->flags);
6399         }
6400         return (0);
6401 }
6402
6403 static int
6404 ahc_check_patch(struct ahc_softc *ahc, struct patch **start_patch,
6405                 u_int start_instr, u_int *skip_addr)
6406 {
6407         struct  patch *cur_patch;
6408         struct  patch *last_patch;
6409         u_int   num_patches;
6410
6411         num_patches = sizeof(patches)/sizeof(struct patch);
6412         last_patch = &patches[num_patches];
6413         cur_patch = *start_patch;
6414
6415         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
6416
6417                 if (cur_patch->patch_func(ahc) == 0) {
6418
6419                         /* Start rejecting code */
6420                         *skip_addr = start_instr + cur_patch->skip_instr;
6421                         cur_patch += cur_patch->skip_patch;
6422                 } else {
6423                         /* Accepted this patch.  Advance to the next
6424                          * one and wait for our intruction pointer to
6425                          * hit this point.
6426                          */
6427                         cur_patch++;
6428                 }
6429         }
6430
6431         *start_patch = cur_patch;
6432         if (start_instr < *skip_addr)
6433                 /* Still skipping */
6434                 return (0);
6435
6436         return (1);
6437 }
6438
6439 static void
6440 ahc_download_instr(struct ahc_softc *ahc, u_int instrptr, uint8_t *dconsts)
6441 {
6442         union   ins_formats instr;
6443         struct  ins_format1 *fmt1_ins;
6444         struct  ins_format3 *fmt3_ins;
6445         u_int   opcode;
6446
6447         /*
6448          * The firmware is always compiled into a little endian format.
6449          */
6450         instr.integer = ahc_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
6451
6452         fmt1_ins = &instr.format1;
6453         fmt3_ins = NULL;
6454
6455         /* Pull the opcode */
6456         opcode = instr.format1.opcode;
6457         switch (opcode) {
6458         case AIC_OP_JMP:
6459         case AIC_OP_JC:
6460         case AIC_OP_JNC:
6461         case AIC_OP_CALL:
6462         case AIC_OP_JNE:
6463         case AIC_OP_JNZ:
6464         case AIC_OP_JE:
6465         case AIC_OP_JZ:
6466         {
6467                 struct patch *cur_patch;
6468                 int address_offset;
6469                 u_int address;
6470                 u_int skip_addr;
6471                 u_int i;
6472
6473                 fmt3_ins = &instr.format3;
6474                 address_offset = 0;
6475                 address = fmt3_ins->address;
6476                 cur_patch = patches;
6477                 skip_addr = 0;
6478
6479                 for (i = 0; i < address;) {
6480
6481                         ahc_check_patch(ahc, &cur_patch, i, &skip_addr);
6482
6483                         if (skip_addr > i) {
6484                                 int end_addr;
6485
6486                                 end_addr = MIN(address, skip_addr);
6487                                 address_offset += end_addr - i;
6488                                 i = skip_addr;
6489                         } else {
6490                                 i++;
6491                         }
6492                 }
6493                 address -= address_offset;
6494                 fmt3_ins->address = address;
6495                 /* FALLTHROUGH */
6496         }
6497         case AIC_OP_OR:
6498         case AIC_OP_AND:
6499         case AIC_OP_XOR:
6500         case AIC_OP_ADD:
6501         case AIC_OP_ADC:
6502         case AIC_OP_BMOV:
6503                 if (fmt1_ins->parity != 0) {
6504                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
6505                 }
6506                 fmt1_ins->parity = 0;
6507                 if ((ahc->features & AHC_CMD_CHAN) == 0
6508                  && opcode == AIC_OP_BMOV) {
6509                         /*
6510                          * Block move was added at the same time
6511                          * as the command channel.  Verify that
6512                          * this is only a move of a single element
6513                          * and convert the BMOV to a MOV
6514                          * (AND with an immediate of FF).
6515                          */
6516                         if (fmt1_ins->immediate != 1)
6517                                 panic("%s: BMOV not supported\n",
6518                                       ahc_name(ahc));
6519                         fmt1_ins->opcode = AIC_OP_AND;
6520                         fmt1_ins->immediate = 0xff;
6521                 }
6522                 /* FALLTHROUGH */
6523         case AIC_OP_ROL:
6524                 if ((ahc->features & AHC_ULTRA2) != 0) {
6525                         int i, count;
6526
6527                         /* Calculate odd parity for the instruction */
6528                         for (i = 0, count = 0; i < 31; i++) {
6529                                 uint32_t mask;
6530
6531                                 mask = 0x01 << i;
6532                                 if ((instr.integer & mask) != 0)
6533                                         count++;
6534                         }
6535                         if ((count & 0x01) == 0)
6536                                 instr.format1.parity = 1;
6537                 } else {
6538                         /* Compress the instruction for older sequencers */
6539                         if (fmt3_ins != NULL) {
6540                                 instr.integer =
6541                                         fmt3_ins->immediate
6542                                       | (fmt3_ins->source << 8)
6543                                       | (fmt3_ins->address << 16)
6544                                       | (fmt3_ins->opcode << 25);
6545                         } else {
6546                                 instr.integer =
6547                                         fmt1_ins->immediate
6548                                       | (fmt1_ins->source << 8)
6549                                       | (fmt1_ins->destination << 16)
6550                                       | (fmt1_ins->ret << 24)
6551                                       | (fmt1_ins->opcode << 25);
6552                         }
6553                 }
6554                 /* The sequencer is a little endian cpu */
6555                 instr.integer = ahc_htole32(instr.integer);
6556                 ahc_outsb(ahc, SEQRAM, instr.bytes, 4);
6557                 break;
6558         default:
6559                 panic("Unknown opcode encountered in seq program");
6560                 break;
6561         }
6562 }
6563
6564 int
6565 ahc_print_register(ahc_reg_parse_entry_t *table, u_int num_entries,
6566                    const char *name, u_int address, u_int value,
6567                    u_int *cur_column, u_int wrap_point)
6568 {
6569         int     printed;
6570         u_int   printed_mask;
6571
6572         if (cur_column != NULL && *cur_column >= wrap_point) {
6573                 kprintf("\n");
6574                 *cur_column = 0;
6575         }
6576         printed = kprintf("%s[0x%x]", name, value);
6577         if (table == NULL) {
6578                 printed += kprintf(" ");
6579                 *cur_column += printed;
6580                 return (printed);
6581         }
6582         printed_mask = 0;
6583         while (printed_mask != 0xFF) {
6584                 int entry;
6585
6586                 for (entry = 0; entry < num_entries; entry++) {
6587                         if (((value & table[entry].mask)
6588                           != table[entry].value)
6589                          || ((printed_mask & table[entry].mask)
6590                           == table[entry].mask))
6591                                 continue;
6592
6593                         printed += kprintf("%s%s",
6594                                           printed_mask == 0 ? ":(" : "|",
6595                                           table[entry].name);
6596                         printed_mask |= table[entry].mask;
6597
6598                         break;
6599                 }
6600                 if (entry >= num_entries)
6601                         break;
6602         }
6603         if (printed_mask != 0)
6604                 printed += kprintf(") ");
6605         else
6606                 printed += kprintf(" ");
6607         if (cur_column != NULL)
6608                 *cur_column += printed;
6609         return (printed);
6610 }
6611
6612 void
6613 ahc_dump_card_state(struct ahc_softc *ahc)
6614 {
6615         struct  scb *scb;
6616         struct  scb_tailq *untagged_q;
6617         u_int   cur_col;
6618         int     paused;
6619         int     target;
6620         int     maxtarget;
6621         int     i;
6622         uint8_t last_phase;
6623         uint8_t qinpos;
6624         uint8_t qintail;
6625         uint8_t qoutpos;
6626         uint8_t scb_index;
6627         uint8_t saved_scbptr;
6628
6629         if (ahc_is_paused(ahc)) {
6630                 paused = 1;
6631         } else {
6632                 paused = 0;
6633                 ahc_pause(ahc);
6634         }
6635
6636         saved_scbptr = ahc_inb(ahc, SCBPTR);
6637         last_phase = ahc_inb(ahc, LASTPHASE);
6638         kprintf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
6639                "%s: Dumping Card State %s, at SEQADDR 0x%x\n",
6640                ahc_name(ahc), ahc_lookup_phase_entry(last_phase)->phasemsg,
6641                ahc_inb(ahc, SEQADDR0) | (ahc_inb(ahc, SEQADDR1) << 8));
6642         if (paused)
6643                 kprintf("Card was paused\n");
6644         kprintf("ACCUM = 0x%x, SINDEX = 0x%x, DINDEX = 0x%x, ARG_2 = 0x%x\n",
6645                ahc_inb(ahc, ACCUM), ahc_inb(ahc, SINDEX), ahc_inb(ahc, DINDEX),
6646                ahc_inb(ahc, ARG_2));
6647         kprintf("HCNT = 0x%x SCBPTR = 0x%x\n", ahc_inb(ahc, HCNT),
6648                ahc_inb(ahc, SCBPTR));
6649         cur_col = 0;
6650         if ((ahc->features & AHC_DT) != 0)
6651                 ahc_scsiphase_print(ahc_inb(ahc, SCSIPHASE), &cur_col, 50);
6652         ahc_scsisigi_print(ahc_inb(ahc, SCSISIGI), &cur_col, 50);
6653         ahc_error_print(ahc_inb(ahc, ERROR), &cur_col, 50);
6654         ahc_scsibusl_print(ahc_inb(ahc, SCSIBUSL), &cur_col, 50);
6655         ahc_lastphase_print(ahc_inb(ahc, LASTPHASE), &cur_col, 50);
6656         ahc_scsiseq_print(ahc_inb(ahc, SCSISEQ), &cur_col, 50);
6657         ahc_sblkctl_print(ahc_inb(ahc, SBLKCTL), &cur_col, 50);
6658         ahc_scsirate_print(ahc_inb(ahc, SCSIRATE), &cur_col, 50);
6659         ahc_seqctl_print(ahc_inb(ahc, SEQCTL), &cur_col, 50);
6660         ahc_seq_flags_print(ahc_inb(ahc, SEQ_FLAGS), &cur_col, 50);
6661         ahc_sstat0_print(ahc_inb(ahc, SSTAT0), &cur_col, 50);
6662         ahc_sstat1_print(ahc_inb(ahc, SSTAT1), &cur_col, 50);
6663         ahc_sstat2_print(ahc_inb(ahc, SSTAT2), &cur_col, 50);
6664         ahc_sstat3_print(ahc_inb(ahc, SSTAT3), &cur_col, 50);
6665         ahc_simode0_print(ahc_inb(ahc, SIMODE0), &cur_col, 50);
6666         ahc_simode1_print(ahc_inb(ahc, SIMODE1), &cur_col, 50);
6667         ahc_sxfrctl0_print(ahc_inb(ahc, SXFRCTL0), &cur_col, 50);
6668         ahc_dfcntrl_print(ahc_inb(ahc, DFCNTRL), &cur_col, 50);
6669         ahc_dfstatus_print(ahc_inb(ahc, DFSTATUS), &cur_col, 50);
6670         if (cur_col != 0)
6671                 kprintf("\n");
6672         kprintf("STACK:");
6673         for (i = 0; i < STACK_SIZE; i++)
6674                kprintf(" 0x%x", ahc_inb(ahc, STACK)|(ahc_inb(ahc, STACK) << 8));
6675         kprintf("\nSCB count = %d\n", ahc->scb_data->numscbs);
6676         kprintf("Kernel NEXTQSCB = %d\n", ahc->next_queued_scb->hscb->tag);
6677         kprintf("Card NEXTQSCB = %d\n", ahc_inb(ahc, NEXT_QUEUED_SCB));
6678         /* QINFIFO */
6679         kprintf("QINFIFO entries: ");
6680         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
6681                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
6682                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
6683         } else
6684                 qinpos = ahc_inb(ahc, QINPOS);
6685         qintail = ahc->qinfifonext;
6686         while (qinpos != qintail) {
6687                 kprintf("%d ", ahc->qinfifo[qinpos]);
6688                 qinpos++;
6689         }
6690         kprintf("\n");
6691
6692         kprintf("Waiting Queue entries: ");
6693         scb_index = ahc_inb(ahc, WAITING_SCBH);
6694         i = 0;
6695         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6696                 ahc_outb(ahc, SCBPTR, scb_index);
6697                 kprintf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
6698                 scb_index = ahc_inb(ahc, SCB_NEXT);
6699         }
6700         kprintf("\n");
6701
6702         kprintf("Disconnected Queue entries: ");
6703         scb_index = ahc_inb(ahc, DISCONNECTED_SCBH);
6704         i = 0;
6705         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6706                 ahc_outb(ahc, SCBPTR, scb_index);
6707                 kprintf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
6708                 scb_index = ahc_inb(ahc, SCB_NEXT);
6709         }
6710         kprintf("\n");
6711
6712         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
6713         kprintf("QOUTFIFO entries: ");
6714         qoutpos = ahc->qoutfifonext;
6715         i = 0;
6716         while (ahc->qoutfifo[qoutpos] != SCB_LIST_NULL && i++ < 256) {
6717                 kprintf("%d ", ahc->qoutfifo[qoutpos]);
6718                 qoutpos++;
6719         }
6720         kprintf("\n");
6721
6722         kprintf("Sequencer Free SCB List: ");
6723         scb_index = ahc_inb(ahc, FREE_SCBH);
6724         i = 0;
6725         while (scb_index != SCB_LIST_NULL && i++ < 256) {
6726                 ahc_outb(ahc, SCBPTR, scb_index);
6727                 kprintf("%d ", scb_index);
6728                 scb_index = ahc_inb(ahc, SCB_NEXT);
6729         }
6730         kprintf("\n");
6731
6732         kprintf("Sequencer SCB Info: ");
6733         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
6734                 ahc_outb(ahc, SCBPTR, i);
6735                 cur_col = kprintf("\n%3d ", i);
6736
6737                 ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL), &cur_col, 60);
6738                 ahc_scb_scsiid_print(ahc_inb(ahc, SCB_SCSIID), &cur_col, 60);
6739                 ahc_scb_lun_print(ahc_inb(ahc, SCB_LUN), &cur_col, 60);
6740                 ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
6741         }
6742         kprintf("\n");
6743
6744         kprintf("Pending list: ");
6745         i = 0;
6746         LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
6747                 if (i++ > 256)
6748                         break;
6749                 cur_col = kprintf("\n%3d ", scb->hscb->tag);
6750                 ahc_scb_control_print(scb->hscb->control, &cur_col, 60);
6751                 ahc_scb_scsiid_print(scb->hscb->scsiid, &cur_col, 60);
6752                 ahc_scb_lun_print(scb->hscb->lun, &cur_col, 60);
6753                 if ((ahc->flags & AHC_PAGESCBS) == 0) {
6754                         ahc_outb(ahc, SCBPTR, scb->hscb->tag);
6755                         kprintf("(");
6756                         ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL),
6757                                               &cur_col, 60);
6758                         ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
6759                         kprintf(")");
6760                 }
6761         }
6762         kprintf("\n");
6763
6764         kprintf("Kernel Free SCB list: ");
6765         i = 0;
6766         SLIST_FOREACH(scb, &ahc->scb_data->free_scbs, links.sle) {
6767                 if (i++ > 256)
6768                         break;
6769                 kprintf("%d ", scb->hscb->tag);
6770         }
6771         kprintf("\n");
6772
6773         maxtarget = (ahc->features & (AHC_WIDE|AHC_TWIN)) ? 15 : 7;
6774         for (target = 0; target <= maxtarget; target++) {
6775                 untagged_q = &ahc->untagged_queues[target];
6776                 if (TAILQ_FIRST(untagged_q) == NULL)
6777                         continue;
6778                 kprintf("Untagged Q(%d): ", target);
6779                 i = 0;
6780                 TAILQ_FOREACH(scb, untagged_q, links.tqe) {
6781                         if (i++ > 256)
6782                                 break;
6783                         kprintf("%d ", scb->hscb->tag);
6784                 }
6785                 kprintf("\n");
6786         }
6787
6788         ahc_platform_dump_card_state(ahc);
6789         kprintf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
6790         ahc_outb(ahc, SCBPTR, saved_scbptr);
6791         if (paused == 0)
6792                 ahc_unpause(ahc);
6793 }
6794
6795 /************************* Target Mode ****************************************/
6796 #ifdef AHC_TARGET_MODE
6797 cam_status
6798 ahc_find_tmode_devs(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb,
6799                     struct ahc_tmode_tstate **tstate,
6800                     struct ahc_tmode_lstate **lstate,
6801                     int notfound_failure)
6802 {
6803
6804         if ((ahc->features & AHC_TARGETMODE) == 0)
6805                 return (CAM_REQ_INVALID);
6806
6807         /*
6808          * Handle the 'black hole' device that sucks up
6809          * requests to unattached luns on enabled targets.
6810          */
6811         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
6812          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
6813                 *tstate = NULL;
6814                 *lstate = ahc->black_hole;
6815         } else {
6816                 u_int max_id;
6817
6818                 max_id = (ahc->features & AHC_WIDE) ? 15 : 7;
6819                 if (ccb->ccb_h.target_id > max_id)
6820                         return (CAM_TID_INVALID);
6821
6822                 if (ccb->ccb_h.target_lun >= AHC_NUM_LUNS)
6823                         return (CAM_LUN_INVALID);
6824
6825                 *tstate = ahc->enabled_targets[ccb->ccb_h.target_id];
6826                 *lstate = NULL;
6827                 if (*tstate != NULL)
6828                         *lstate =
6829                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
6830         }
6831
6832         if (notfound_failure != 0 && *lstate == NULL)
6833                 return (CAM_PATH_INVALID);
6834
6835         return (CAM_REQ_CMP);
6836 }
6837
6838 void
6839 ahc_handle_en_lun(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb)
6840 {
6841         struct     ahc_tmode_tstate *tstate;
6842         struct     ahc_tmode_lstate *lstate;
6843         struct     ccb_en_lun *cel;
6844         cam_status status;
6845         u_int      target;
6846         u_int      lun;
6847         u_int      target_mask;
6848         u_int      our_id;
6849         int        error;
6850         char       channel;
6851
6852         status = ahc_find_tmode_devs(ahc, sim, ccb, &tstate, &lstate,
6853                                      /*notfound_failure*/FALSE);
6854
6855         if (status != CAM_REQ_CMP) {
6856                 ccb->ccb_h.status = status;
6857                 return;
6858         }
6859
6860         if (cam_sim_bus(sim) == 0)
6861                 our_id = ahc->our_id;
6862         else
6863                 our_id = ahc->our_id_b;
6864
6865         if (ccb->ccb_h.target_id != our_id) {
6866                 /*
6867                  * our_id represents our initiator ID, or
6868                  * the ID of the first target to have an
6869                  * enabled lun in target mode.  There are
6870                  * two cases that may preclude enabling a
6871                  * target id other than our_id.
6872                  *
6873                  *   o our_id is for an active initiator role.
6874                  *     Since the hardware does not support
6875                  *     reselections to the initiator role at
6876                  *     anything other than our_id, and our_id
6877                  *     is used by the hardware to indicate the
6878                  *     ID to use for both select-out and
6879                  *     reselect-out operations, the only target
6880                  *     ID we can support in this mode is our_id.
6881                  *
6882                  *   o The MULTARGID feature is not available and
6883                  *     a previous target mode ID has been enabled.
6884                  */
6885                 if ((ahc->features & AHC_MULTIROLE) != 0) {
6886
6887                         if ((ahc->features & AHC_MULTI_TID) != 0
6888                          && (ahc->flags & AHC_INITIATORROLE) != 0) {
6889                                 /*
6890                                  * Only allow additional targets if
6891                                  * the initiator role is disabled.
6892                                  * The hardware cannot handle a re-select-in
6893                                  * on the initiator id during a re-select-out
6894                                  * on a different target id.
6895                                  */
6896                                 status = CAM_TID_INVALID;
6897                         } else if ((ahc->flags & AHC_INITIATORROLE) != 0
6898                                 || ahc->enabled_luns > 0) {
6899                                 /*
6900                                  * Only allow our target id to change
6901                                  * if the initiator role is not configured
6902                                  * and there are no enabled luns which
6903                                  * are attached to the currently registered
6904                                  * scsi id.
6905                                  */
6906                                 status = CAM_TID_INVALID;
6907                         }
6908                 } else if ((ahc->features & AHC_MULTI_TID) == 0
6909                         && ahc->enabled_luns > 0) {
6910
6911                         status = CAM_TID_INVALID;
6912                 }
6913         }
6914
6915         if (status != CAM_REQ_CMP) {
6916                 ccb->ccb_h.status = status;
6917                 return;
6918         }
6919
6920         /*
6921          * We now have an id that is valid.
6922          * If we aren't in target mode, switch modes.
6923          */
6924         if ((ahc->flags & AHC_TARGETROLE) == 0
6925          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
6926                 ahc_flag saved_flags;
6927
6928                 kprintf("Configuring Target Mode\n");
6929                 ahc_lock();
6930                 if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
6931                         ccb->ccb_h.status = CAM_BUSY;
6932                         ahc_unlock();
6933                         return;
6934                 }
6935                 saved_flags = ahc->flags;
6936                 ahc->flags |= AHC_TARGETROLE;
6937                 if ((ahc->features & AHC_MULTIROLE) == 0)
6938                         ahc->flags &= ~AHC_INITIATORROLE;
6939                 ahc_pause(ahc);
6940                 error = ahc_loadseq(ahc);
6941                 if (error != 0) {
6942                         /*
6943                          * Restore original configuration and notify
6944                          * the caller that we cannot support target mode.
6945                          * Since the adapter started out in this
6946                          * configuration, the firmware load will succeed,
6947                          * so there is no point in checking ahc_loadseq's
6948                          * return value.
6949                          */
6950                         ahc->flags = saved_flags;
6951                         (void)ahc_loadseq(ahc);
6952                         ahc_restart(ahc);
6953                         ahc_unlock();
6954                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
6955                         return;
6956                 }
6957                 ahc_restart(ahc);
6958                 ahc_unlock();
6959         }
6960         cel = &ccb->cel;
6961         target = ccb->ccb_h.target_id;
6962         lun = ccb->ccb_h.target_lun;
6963         channel = SIM_CHANNEL(ahc, sim);
6964         target_mask = 0x01 << target;
6965         if (channel == 'B')
6966                 target_mask <<= 8;
6967
6968         if (cel->enable != 0) {
6969                 u_int scsiseq;
6970
6971                 /* Are we already enabled?? */
6972                 if (lstate != NULL) {
6973                         xpt_print_path(ccb->ccb_h.path);
6974                         kprintf("Lun already enabled\n");
6975                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
6976                         return;
6977                 }
6978
6979                 if (cel->grp6_len != 0
6980                  || cel->grp7_len != 0) {
6981                         /*
6982                          * Don't (yet?) support vendor
6983                          * specific commands.
6984                          */
6985                         ccb->ccb_h.status = CAM_REQ_INVALID;
6986                         kprintf("Non-zero Group Codes\n");
6987                         return;
6988                 }
6989
6990                 /*
6991                  * Seems to be okay.
6992                  * Setup our data structures.
6993                  */
6994                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
6995                         tstate = ahc_alloc_tstate(ahc, target, channel);
6996                         if (tstate == NULL) {
6997                                 xpt_print_path(ccb->ccb_h.path);
6998                                 kprintf("Couldn't allocate tstate\n");
6999                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7000                                 return;
7001                         }
7002                 }
7003                 lstate = kmalloc(sizeof(*lstate), M_DEVBUF, M_INTWAIT | M_ZERO);
7004                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
7005                                          xpt_path_path_id(ccb->ccb_h.path),
7006                                          xpt_path_target_id(ccb->ccb_h.path),
7007                                          xpt_path_lun_id(ccb->ccb_h.path));
7008                 if (status != CAM_REQ_CMP) {
7009                         kfree(lstate, M_DEVBUF);
7010                         xpt_print_path(ccb->ccb_h.path);
7011                         kprintf("Couldn't allocate path\n");
7012                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7013                         return;
7014                 }
7015                 SLIST_INIT(&lstate->accept_tios);
7016                 SLIST_INIT(&lstate->immed_notifies);
7017                 ahc_lock();
7018                 ahc_pause(ahc);
7019                 if (target != CAM_TARGET_WILDCARD) {
7020                         tstate->enabled_luns[lun] = lstate;
7021                         ahc->enabled_luns++;
7022
7023                         if ((ahc->features & AHC_MULTI_TID) != 0) {
7024                                 u_int targid_mask;
7025
7026                                 targid_mask = ahc_inb(ahc, TARGID)
7027                                             | (ahc_inb(ahc, TARGID + 1) << 8);
7028
7029                                 targid_mask |= target_mask;
7030                                 ahc_outb(ahc, TARGID, targid_mask);
7031                                 ahc_outb(ahc, TARGID+1, (targid_mask >> 8));
7032
7033                                 ahc_update_scsiid(ahc, targid_mask);
7034                         } else {
7035                                 u_int our_id;
7036                                 char  channel;
7037
7038                                 channel = SIM_CHANNEL(ahc, sim);
7039                                 our_id = SIM_SCSI_ID(ahc, sim);
7040
7041                                 /*
7042                                  * This can only happen if selections
7043                                  * are not enabled
7044                                  */
7045                                 if (target != our_id) {
7046                                         u_int sblkctl;
7047                                         char  cur_channel;
7048                                         int   swap;
7049
7050                                         sblkctl = ahc_inb(ahc, SBLKCTL);
7051                                         cur_channel = (sblkctl & SELBUSB)
7052                                                     ? 'B' : 'A';
7053                                         if ((ahc->features & AHC_TWIN) == 0)
7054                                                 cur_channel = 'A';
7055                                         swap = cur_channel != channel;
7056                                         if (channel == 'A')
7057                                                 ahc->our_id = target;
7058                                         else
7059                                                 ahc->our_id_b = target;
7060
7061                                         if (swap)
7062                                                 ahc_outb(ahc, SBLKCTL,
7063                                                          sblkctl ^ SELBUSB);
7064
7065                                         ahc_outb(ahc, SCSIID, target);
7066
7067                                         if (swap)
7068                                                 ahc_outb(ahc, SBLKCTL, sblkctl);
7069                                 }
7070                         }
7071                 } else
7072                         ahc->black_hole = lstate;
7073                 /* Allow select-in operations */
7074                 if (ahc->black_hole != NULL && ahc->enabled_luns > 0) {
7075                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7076                         scsiseq |= ENSELI;
7077                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7078                         scsiseq = ahc_inb(ahc, SCSISEQ);
7079                         scsiseq |= ENSELI;
7080                         ahc_outb(ahc, SCSISEQ, scsiseq);
7081                 }
7082                 ahc_unpause(ahc);
7083                 ahc_unlock();
7084                 ccb->ccb_h.status = CAM_REQ_CMP;
7085                 xpt_print_path(ccb->ccb_h.path);
7086                 kprintf("Lun now enabled for target mode\n");
7087         } else {
7088                 struct scb *scb;
7089                 int i, empty;
7090
7091                 if (lstate == NULL) {
7092                         ccb->ccb_h.status = CAM_LUN_INVALID;
7093                         return;
7094                 }
7095
7096                 ahc_lock();
7097
7098                 ccb->ccb_h.status = CAM_REQ_CMP;
7099                 LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7100                         struct ccb_hdr *ccbh;
7101
7102                         ccbh = &scb->io_ctx->ccb_h;
7103                         if (ccbh->func_code == XPT_CONT_TARGET_IO
7104                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
7105                                 kprintf("CTIO pending\n");
7106                                 ccb->ccb_h.status = CAM_REQ_INVALID;
7107                                 ahc_unlock();
7108                                 return;
7109                         }
7110                 }
7111
7112                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
7113                         kprintf("ATIOs pending\n");
7114                         ccb->ccb_h.status = CAM_REQ_INVALID;
7115                 }
7116
7117                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
7118                         kprintf("INOTs pending\n");
7119                         ccb->ccb_h.status = CAM_REQ_INVALID;
7120                 }
7121
7122                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
7123                         ahc_unlock();
7124                         return;
7125                 }
7126
7127                 xpt_print_path(ccb->ccb_h.path);
7128                 kprintf("Target mode disabled\n");
7129                 xpt_free_path(lstate->path);
7130                 kfree(lstate, M_DEVBUF);
7131
7132                 ahc_pause(ahc);
7133                 /* Can we clean up the target too? */
7134                 if (target != CAM_TARGET_WILDCARD) {
7135                         tstate->enabled_luns[lun] = NULL;
7136                         ahc->enabled_luns--;
7137                         for (empty = 1, i = 0; i < 8; i++)
7138                                 if (tstate->enabled_luns[i] != NULL) {
7139                                         empty = 0;
7140                                         break;
7141                                 }
7142
7143                         if (empty) {
7144                                 ahc_free_tstate(ahc, target, channel,
7145                                                 /*force*/FALSE);
7146                                 if (ahc->features & AHC_MULTI_TID) {
7147                                         u_int targid_mask;
7148
7149                                         targid_mask = ahc_inb(ahc, TARGID)
7150                                                     | (ahc_inb(ahc, TARGID + 1)
7151                                                        << 8);
7152
7153                                         targid_mask &= ~target_mask;
7154                                         ahc_outb(ahc, TARGID, targid_mask);
7155                                         ahc_outb(ahc, TARGID+1,
7156                                                  (targid_mask >> 8));
7157                                         ahc_update_scsiid(ahc, targid_mask);
7158                                 }
7159                         }
7160                 } else {
7161
7162                         ahc->black_hole = NULL;
7163
7164                         /*
7165                          * We can't allow selections without
7166                          * our black hole device.
7167                          */
7168                         empty = TRUE;
7169                 }
7170                 if (ahc->enabled_luns == 0) {
7171                         /* Disallow select-in */
7172                         u_int scsiseq;
7173
7174                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7175                         scsiseq &= ~ENSELI;
7176                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7177                         scsiseq = ahc_inb(ahc, SCSISEQ);
7178                         scsiseq &= ~ENSELI;
7179                         ahc_outb(ahc, SCSISEQ, scsiseq);
7180
7181                         if ((ahc->features & AHC_MULTIROLE) == 0) {
7182                                 kprintf("Configuring Initiator Mode\n");
7183                                 ahc->flags &= ~AHC_TARGETROLE;
7184                                 ahc->flags |= AHC_INITIATORROLE;
7185                                 /*
7186                                  * Returning to a configuration that
7187                                  * fit previously will always succeed.
7188                                  */
7189                                 (void)ahc_loadseq(ahc);
7190                                 ahc_restart(ahc);
7191                                 /*
7192                                  * Unpaused.  The extra unpause
7193                                  * that follows is harmless.
7194                                  */
7195                         }
7196                 }
7197                 ahc_unpause(ahc);
7198                 ahc_unlock();
7199         }
7200 }
7201
7202 static void
7203 ahc_update_scsiid(struct ahc_softc *ahc, u_int targid_mask)
7204 {
7205         u_int scsiid_mask;
7206         u_int scsiid;
7207
7208         if ((ahc->features & AHC_MULTI_TID) == 0)
7209                 panic("ahc_update_scsiid called on non-multitid unit\n");
7210
7211         /*
7212          * Since we will rely on the TARGID mask
7213          * for selection enables, ensure that OID
7214          * in SCSIID is not set to some other ID
7215          * that we don't want to allow selections on.
7216          */
7217         if ((ahc->features & AHC_ULTRA2) != 0)
7218                 scsiid = ahc_inb(ahc, SCSIID_ULTRA2);
7219         else
7220                 scsiid = ahc_inb(ahc, SCSIID);
7221         scsiid_mask = 0x1 << (scsiid & OID);
7222         if ((targid_mask & scsiid_mask) == 0) {
7223                 u_int our_id;
7224
7225                 /* ffs counts from 1 */
7226                 our_id = ffs(targid_mask);
7227                 if (our_id == 0)
7228                         our_id = ahc->our_id;
7229                 else
7230                         our_id--;
7231                 scsiid &= TID;
7232                 scsiid |= our_id;
7233         }
7234         if ((ahc->features & AHC_ULTRA2) != 0)
7235                 ahc_outb(ahc, SCSIID_ULTRA2, scsiid);
7236         else
7237                 ahc_outb(ahc, SCSIID, scsiid);
7238 }
7239
7240 void
7241 ahc_run_tqinfifo(struct ahc_softc *ahc, int paused)
7242 {
7243         struct target_cmd *cmd;
7244
7245         /*
7246          * If the card supports auto-access pause,
7247          * we can access the card directly regardless
7248          * of whether it is paused or not.
7249          */
7250         if ((ahc->features & AHC_AUTOPAUSE) != 0)
7251                 paused = TRUE;
7252
7253         ahc_sync_tqinfifo(ahc, BUS_DMASYNC_POSTREAD);
7254         while ((cmd = &ahc->targetcmds[ahc->tqinfifonext])->cmd_valid != 0) {
7255
7256                 /*
7257                  * Only advance through the queue if we
7258                  * have the resources to process the command.
7259                  */
7260                 if (ahc_handle_target_cmd(ahc, cmd) != 0)
7261                         break;
7262
7263                 cmd->cmd_valid = 0;
7264                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
7265                                 ahc->shared_data_dmamap,
7266                                 ahc_targetcmd_offset(ahc, ahc->tqinfifonext),
7267                                 sizeof(struct target_cmd),
7268                                 BUS_DMASYNC_PREREAD);
7269                 ahc->tqinfifonext++;
7270
7271                 /*
7272                  * Lazily update our position in the target mode incoming
7273                  * command queue as seen by the sequencer.
7274                  */
7275                 if ((ahc->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
7276                         if ((ahc->features & AHC_HS_MAILBOX) != 0) {
7277                                 u_int hs_mailbox;
7278
7279                                 hs_mailbox = ahc_inb(ahc, HS_MAILBOX);
7280                                 hs_mailbox &= ~HOST_TQINPOS;
7281                                 hs_mailbox |= ahc->tqinfifonext & HOST_TQINPOS;
7282                                 ahc_outb(ahc, HS_MAILBOX, hs_mailbox);
7283                         } else {
7284                                 if (!paused)
7285                                         ahc_pause(ahc);
7286                                 ahc_outb(ahc, KERNEL_TQINPOS,
7287                                          ahc->tqinfifonext & HOST_TQINPOS);
7288                                 if (!paused)
7289                                         ahc_unpause(ahc);
7290                         }
7291                 }
7292         }
7293 }
7294
7295 static int
7296 ahc_handle_target_cmd(struct ahc_softc *ahc, struct target_cmd *cmd)
7297 {
7298         struct    ahc_tmode_tstate *tstate;
7299         struct    ahc_tmode_lstate *lstate;
7300         struct    ccb_accept_tio *atio;
7301         uint8_t *byte;
7302         int       initiator;
7303         int       target;
7304         int       lun;
7305
7306         initiator = SCSIID_TARGET(ahc, cmd->scsiid);
7307         target = SCSIID_OUR_ID(cmd->scsiid);
7308         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
7309
7310         byte = cmd->bytes;
7311         tstate = ahc->enabled_targets[target];
7312         lstate = NULL;
7313         if (tstate != NULL)
7314                 lstate = tstate->enabled_luns[lun];
7315
7316         /*
7317          * Commands for disabled luns go to the black hole driver.
7318          */
7319         if (lstate == NULL)
7320                 lstate = ahc->black_hole;
7321
7322         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
7323         if (atio == NULL) {
7324                 ahc->flags |= AHC_TQINFIFO_BLOCKED;
7325                 /*
7326                  * Wait for more ATIOs from the peripheral driver for this lun.
7327                  */
7328                 if (bootverbose)
7329                         kprintf("%s: ATIOs exhausted\n", ahc_name(ahc));
7330                 return (1);
7331         } else
7332                 ahc->flags &= ~AHC_TQINFIFO_BLOCKED;
7333 #if 0
7334         kprintf("Incoming command from %d for %d:%d%s\n",
7335                initiator, target, lun,
7336                lstate == ahc->black_hole ? "(Black Holed)" : "");
7337 #endif
7338         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
7339
7340         if (lstate == ahc->black_hole) {
7341                 /* Fill in the wildcards */
7342                 atio->ccb_h.target_id = target;
7343                 atio->ccb_h.target_lun = lun;
7344         }
7345
7346         /*
7347          * Package it up and send it off to
7348          * whomever has this lun enabled.
7349          */
7350         atio->sense_len = 0;
7351         atio->init_id = initiator;
7352         if (byte[0] != 0xFF) {
7353                 /* Tag was included */
7354                 atio->tag_action = *byte++;
7355                 atio->tag_id = *byte++;
7356                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
7357         } else {
7358                 atio->ccb_h.flags = 0;
7359         }
7360         byte++;
7361
7362         /* Okay.  Now determine the cdb size based on the command code */
7363         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
7364         case 0:
7365                 atio->cdb_len = 6;
7366                 break;
7367         case 1:
7368         case 2:
7369                 atio->cdb_len = 10;
7370                 break;
7371         case 4:
7372                 atio->cdb_len = 16;
7373                 break;
7374         case 5:
7375                 atio->cdb_len = 12;
7376                 break;
7377         case 3:
7378         default:
7379                 /* Only copy the opcode. */
7380                 atio->cdb_len = 1;
7381                 kprintf("Reserved or VU command code type encountered\n");
7382                 break;
7383         }
7384
7385         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
7386
7387         atio->ccb_h.status |= CAM_CDB_RECVD;
7388
7389         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
7390                 /*
7391                  * We weren't allowed to disconnect.
7392                  * We're hanging on the bus until a
7393                  * continue target I/O comes in response
7394                  * to this accept tio.
7395                  */
7396 #if 0
7397                 kprintf("Received Immediate Command %d:%d:%d - %p\n",
7398                        initiator, target, lun, ahc->pending_device);
7399 #endif
7400                 ahc->pending_device = lstate;
7401                 ahc_freeze_ccb((union ccb *)atio);
7402                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
7403         }
7404         xpt_done((union ccb*)atio);
7405         return (0);
7406 }
7407
7408 #endif