sys/dev/raid/aac/aac.c

   1 /*-
   2  * Copyright (c) 2000 Michael Smith
   3  * Copyright (c) 2001 Scott Long
   4  * Copyright (c) 2000 BSDi
   5  * Copyright (c) 2001 Adaptec, Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  *      $FreeBSD: src/sys/dev/aac/aac.c,v 1.9.2.14 2003/04/08 13:22:08 scottl Exp $
  30  *      $DragonFly: src/sys/dev/raid/aac/aac.c,v 1.34 2008/01/20 03:40:35 pavalos Exp $
  31  */
  32
  33 /*
  34  * Driver for the Adaptec 'FSA' family of PCI/SCSI RAID adapters.
  35  */
  36 #define AAC_DRIVER_VERSION              0x02000000
  37 #define AAC_DRIVERNAME                  "aac"
  38
  39 #include "opt_aac.h"
  40
  41 /* #include <stddef.h> */
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/malloc.h>
  45 #include <sys/kernel.h>
  46 #include <sys/kthread.h>
  47 #include <sys/sysctl.h>
  48 #include <sys/poll.h>
  49
  50 #include <sys/bus.h>
  51 #include <sys/conf.h>
  52 #include <sys/devicestat.h>
  53 #include <sys/disk.h>
  54 #include <sys/signalvar.h>
  55 #include <sys/time.h>
  56 #include <sys/eventhandler.h>
  57 #include <sys/rman.h>
  58
  59 #include <bus/pci/pcireg.h>
  60 #include <bus/pci/pcivar.h>
  61
  62 #include "aacreg.h"
  63 #include "aac_ioctl.h"
  64 #include "aacvar.h"
  65 #include "aac_tables.h"
  66
  67 static void     aac_startup(void *arg);
  68 static void     aac_add_container(struct aac_softc *sc,
  69                                   struct aac_mntinforesp *mir, int f);
  70 static void     aac_get_bus_info(struct aac_softc *sc);
  71 static int      aac_shutdown(device_t dev);
  72
  73 /* Command Processing */
  74 static void     aac_timeout(void *ssc);
  75 static int      aac_map_command(struct aac_command *cm);
  76 static void     aac_complete(void *context, int pending);
  77 static int      aac_bio_command(struct aac_softc *sc, struct aac_command **cmp);
  78 static void     aac_bio_complete(struct aac_command *cm);
  79 static int      aac_wait_command(struct aac_command *cm);
  80 static void     aac_command_thread(struct aac_softc *sc);
  81
  82 /* Command Buffer Management */
  83 static void     aac_map_command_sg(void *arg, bus_dma_segment_t *segs,
  84                                    int nseg, int error);
  85 static void     aac_map_command_helper(void *arg, bus_dma_segment_t *segs,
  86                                        int nseg, int error);
  87 static int      aac_alloc_commands(struct aac_softc *sc);
  88 static void     aac_free_commands(struct aac_softc *sc);
  89 static void     aac_unmap_command(struct aac_command *cm);
  90
  91 /* Hardware Interface */
  92 static void     aac_common_map(void *arg, bus_dma_segment_t *segs, int nseg,
  93                                int error);
  94 static int      aac_check_firmware(struct aac_softc *sc);
  95 static int      aac_init(struct aac_softc *sc);
  96 static int      aac_sync_command(struct aac_softc *sc, u_int32_t command,
  97                                  u_int32_t arg0, u_int32_t arg1, u_int32_t arg2,
  98                                  u_int32_t arg3, u_int32_t *sp);
  99 static int      aac_enqueue_fib(struct aac_softc *sc, int queue,
 100                                 struct aac_command *cm);
 101 static int      aac_dequeue_fib(struct aac_softc *sc, int queue,
 102                                 u_int32_t *fib_size, struct aac_fib **fib_addr);
 103 static int      aac_enqueue_response(struct aac_softc *sc, int queue,
 104                                      struct aac_fib *fib);
 105
 106 /* Falcon/PPC interface */
 107 static int      aac_fa_get_fwstatus(struct aac_softc *sc);
 108 static void     aac_fa_qnotify(struct aac_softc *sc, int qbit);
 109 static int      aac_fa_get_istatus(struct aac_softc *sc);
 110 static void     aac_fa_clear_istatus(struct aac_softc *sc, int mask);
 111 static void     aac_fa_set_mailbox(struct aac_softc *sc, u_int32_t command,
 112                                    u_int32_t arg0, u_int32_t arg1,
 113                                    u_int32_t arg2, u_int32_t arg3);
 114 static int      aac_fa_get_mailbox(struct aac_softc *sc, int mb);
 115 static void     aac_fa_set_interrupts(struct aac_softc *sc, int enable);
 116
 117 struct aac_interface aac_fa_interface = {
 118         aac_fa_get_fwstatus,
 119         aac_fa_qnotify,
 120         aac_fa_get_istatus,
 121         aac_fa_clear_istatus,
 122         aac_fa_set_mailbox,
 123         aac_fa_get_mailbox,
 124         aac_fa_set_interrupts,
 125         NULL, NULL, NULL
 126 };
 127
 128 /* StrongARM interface */
 129 static int      aac_sa_get_fwstatus(struct aac_softc *sc);
 130 static void     aac_sa_qnotify(struct aac_softc *sc, int qbit);
 131 static int      aac_sa_get_istatus(struct aac_softc *sc);
 132 static void     aac_sa_clear_istatus(struct aac_softc *sc, int mask);
 133 static void     aac_sa_set_mailbox(struct aac_softc *sc, u_int32_t command,
 134                                    u_int32_t arg0, u_int32_t arg1,
 135                                    u_int32_t arg2, u_int32_t arg3);
 136 static int      aac_sa_get_mailbox(struct aac_softc *sc, int mb);
 137 static void     aac_sa_set_interrupts(struct aac_softc *sc, int enable);
 138
 139 struct aac_interface aac_sa_interface = {
 140         aac_sa_get_fwstatus,
 141         aac_sa_qnotify,
 142         aac_sa_get_istatus,
 143         aac_sa_clear_istatus,
 144         aac_sa_set_mailbox,
 145         aac_sa_get_mailbox,
 146         aac_sa_set_interrupts,
 147         NULL, NULL, NULL
 148 };
 149
 150 /* i960Rx interface */
 151 static int      aac_rx_get_fwstatus(struct aac_softc *sc);
 152 static void     aac_rx_qnotify(struct aac_softc *sc, int qbit);
 153 static int      aac_rx_get_istatus(struct aac_softc *sc);
 154 static void     aac_rx_clear_istatus(struct aac_softc *sc, int mask);
 155 static void     aac_rx_set_mailbox(struct aac_softc *sc, u_int32_t command,
 156                                    u_int32_t arg0, u_int32_t arg1,
 157                                    u_int32_t arg2, u_int32_t arg3);
 158 static int      aac_rx_get_mailbox(struct aac_softc *sc, int mb);
 159 static void     aac_rx_set_interrupts(struct aac_softc *sc, int enable);
 160 static int aac_rx_send_command(struct aac_softc *sc, struct aac_command *cm);
 161 static int aac_rx_get_outb_queue(struct aac_softc *sc);
 162 static void aac_rx_set_outb_queue(struct aac_softc *sc, int index);
 163
 164 struct aac_interface aac_rx_interface = {
 165         aac_rx_get_fwstatus,
 166         aac_rx_qnotify,
 167         aac_rx_get_istatus,
 168         aac_rx_clear_istatus,
 169         aac_rx_set_mailbox,
 170         aac_rx_get_mailbox,
 171         aac_rx_set_interrupts,
 172         aac_rx_send_command,
 173         aac_rx_get_outb_queue,
 174         aac_rx_set_outb_queue
 175 };
 176
 177 /* Rocket/MIPS interface */
 178 static int      aac_rkt_get_fwstatus(struct aac_softc *sc);
 179 static void     aac_rkt_qnotify(struct aac_softc *sc, int qbit);
 180 static int      aac_rkt_get_istatus(struct aac_softc *sc);
 181 static void     aac_rkt_clear_istatus(struct aac_softc *sc, int mask);
 182 static void     aac_rkt_set_mailbox(struct aac_softc *sc, u_int32_t command,
 183                                     u_int32_t arg0, u_int32_t arg1,
 184                                     u_int32_t arg2, u_int32_t arg3);
 185 static int      aac_rkt_get_mailbox(struct aac_softc *sc, int mb);
 186 static void     aac_rkt_set_interrupts(struct aac_softc *sc, int enable);
 187 static int aac_rkt_send_command(struct aac_softc *sc, struct aac_command *cm);
 188 static int aac_rkt_get_outb_queue(struct aac_softc *sc);
 189 static void aac_rkt_set_outb_queue(struct aac_softc *sc, int index);
 190
 191 struct aac_interface aac_rkt_interface = {
 192         aac_rkt_get_fwstatus,
 193         aac_rkt_qnotify,
 194         aac_rkt_get_istatus,
 195         aac_rkt_clear_istatus,
 196         aac_rkt_set_mailbox,
 197         aac_rkt_get_mailbox,
 198         aac_rkt_set_interrupts,
 199         aac_rkt_send_command,
 200         aac_rkt_get_outb_queue,
 201         aac_rkt_set_outb_queue
 202 };
 203
 204 /* Debugging and Diagnostics */
 205 static void     aac_describe_controller(struct aac_softc *sc);
 206 static char     *aac_describe_code(struct aac_code_lookup *table,
 207                                    u_int32_t code);
 208
 209 /* Management Interface */
 210 static d_open_t         aac_open;
 211 static d_close_t        aac_close;
 212 static d_ioctl_t        aac_ioctl;
 213 static d_poll_t         aac_poll;
 214 static int              aac_ioctl_sendfib(struct aac_softc *sc, caddr_t ufib) __unused;
 215 static void             aac_handle_aif(struct aac_softc *sc,
 216                                            struct aac_fib *fib);
 217 static int              aac_rev_check(struct aac_softc *sc, caddr_t udata);
 218 static int              aac_getnext_aif(struct aac_softc *sc, caddr_t arg);
 219 static int              aac_return_aif(struct aac_softc *sc, caddr_t uptr);
 220 static int              aac_query_disk(struct aac_softc *sc, caddr_t uptr);
 221 static int              aac_get_pci_info(struct aac_softc *sc, caddr_t uptr);
 222 static void             aac_ioctl_event(struct aac_softc *sc,
 223                                         struct aac_event *event, void *arg);
 224
 225 #define AAC_CDEV_MAJOR  150
 226
 227 static struct dev_ops aac_ops = {
 228         { "aac", AAC_CDEV_MAJOR, 0 },
 229         .d_open =       aac_open,
 230         .d_close =      aac_close,
 231         .d_ioctl =      aac_ioctl,
 232         .d_poll =       aac_poll,
 233 };
 234
 235 DECLARE_DUMMY_MODULE(aac);
 236
 237 MALLOC_DEFINE(M_AACBUF, "aacbuf", "Buffers for the AAC driver");
 238
 239 /* sysctl node */
 240 SYSCTL_NODE(_hw, OID_AUTO, aac, CTLFLAG_RD, 0, "AAC driver parameters");
 241
 242 /*
 243  * Device Interface
 244  */
 245
 246 /*
 247  * Initialise the controller and softc
 248  */
 249 int
 250 aac_attach(struct aac_softc *sc)
 251 {
 252         int error, unit;
 253
 254         debug_called(1);
 255         callout_init(&sc->aac_watchdog);
 256
 257         /*
 258          * Initialise per-controller queues.
 259          */
 260         aac_initq_free(sc);
 261         aac_initq_ready(sc);
 262         aac_initq_busy(sc);
 263         aac_initq_complete(sc);
 264         aac_initq_bio(sc);
 265
 266         /*
 267          * Initialise command-completion task.
 268          */
 269         TASK_INIT(&sc->aac_task_complete, 0, aac_complete, sc);
 270
 271         /* mark controller as suspended until we get ourselves organised */
 272         sc->aac_state |= AAC_STATE_SUSPEND;
 273
 274         /*
 275          * Check that the firmware on the card is supported.
 276          */
 277         if ((error = aac_check_firmware(sc)) != 0)
 278                 return(error);
 279
 280         /*
 281          * Initialize locks
 282          */
 283         AAC_LOCK_INIT(&sc->aac_aifq_lock, "AAC AIF lock");
 284         AAC_LOCK_INIT(&sc->aac_io_lock, "AAC I/O lock");
 285         AAC_LOCK_INIT(&sc->aac_container_lock, "AAC container lock");
 286         TAILQ_INIT(&sc->aac_container_tqh);
 287         TAILQ_INIT(&sc->aac_ev_cmfree);
 288
 289
 290         /* Initialize the local AIF queue pointers */
 291         sc->aac_aifq_head = sc->aac_aifq_tail = AAC_AIFQ_LENGTH;
 292
 293         /*
 294          * Initialise the adapter.
 295          */
 296         if ((error = aac_init(sc)) != 0)
 297                 return(error);
 298
 299         /*
 300          * Allocate and connect our interrupt.
 301          */
 302         sc->aac_irq_rid = 0;
 303         if ((sc->aac_irq = bus_alloc_resource_any(sc->aac_dev, SYS_RES_IRQ,
 304                                                   &sc->aac_irq_rid,
 305                                                   RF_SHAREABLE |
 306                                                   RF_ACTIVE)) == NULL) {
 307                 device_printf(sc->aac_dev, "can't allocate interrupt\n");
 308                 return (EINVAL);
 309         }
 310         if (sc->flags & AAC_FLAGS_NEW_COMM) {
 311                 if (bus_setup_intr(sc->aac_dev, sc->aac_irq,
 312                                    INTR_MPSAFE, aac_new_intr,
 313                                    sc, &sc->aac_intr, NULL)) {
 314                         device_printf(sc->aac_dev, "can't set up interrupt\n");
 315                         return (EINVAL);
 316                 }
 317         } else {
 318                 if (bus_setup_intr(sc->aac_dev, sc->aac_irq,
 319                                    INTR_FAST, aac_fast_intr,
 320                                    sc, &sc->aac_intr, NULL)) {
 321                         device_printf(sc->aac_dev,
 322                                       "can't set up FAST interrupt\n");
 323                         if (bus_setup_intr(sc->aac_dev, sc->aac_irq,
 324                                            INTR_MPSAFE, aac_fast_intr,
 325                                            sc, &sc->aac_intr, NULL)) {
 326                                 device_printf(sc->aac_dev,
 327                                              "can't set up MPSAFE interrupt\n");
 328                                 return (EINVAL);
 329                         }
 330                 }
 331         }
 332
 333         /*
 334          * Print a little information about the controller.
 335          */
 336         aac_describe_controller(sc);
 337
 338         /*
 339          * Register to probe our containers later.
 340          */
 341         sc->aac_ich.ich_func = aac_startup;
 342         sc->aac_ich.ich_arg = sc;
 343         sc->aac_ich.ich_desc = "aac";
 344         if (config_intrhook_establish(&sc->aac_ich) != 0) {
 345                 device_printf(sc->aac_dev,
 346                               "can't establish configuration hook\n");
 347                 return(ENXIO);
 348         }
 349
 350         /*
 351          * Make the control device.
 352          */
 353         unit = device_get_unit(sc->aac_dev);
 354         dev_ops_add(&aac_ops, -1, unit);
 355         sc->aac_dev_t = make_dev(&aac_ops, unit, UID_ROOT, GID_OPERATOR, 0640,
 356                                  "aac%d", unit);
 357         sc->aac_dev_t->si_drv1 = sc;
 358         reference_dev(sc->aac_dev_t);
 359
 360         /* Create the AIF thread */
 361         if (kthread_create((void(*)(void *))aac_command_thread, sc,
 362                            &sc->aifthread, "aac%daif", unit))
 363                 panic("Could not create AIF thread\n");
 364
 365         /* Register the shutdown method to only be called post-dump */
 366         if ((sc->eh = EVENTHANDLER_REGISTER(shutdown_post_sync, aac_shutdown,
 367             sc->aac_dev, SHUTDOWN_PRI_DRIVER)) == NULL)
 368                 device_printf(sc->aac_dev,
 369                               "shutdown event registration failed\n");
 370
 371         /* Register with CAM for the non-DASD devices */
 372         if ((sc->flags & AAC_FLAGS_ENABLE_CAM) != 0) {
 373                 TAILQ_INIT(&sc->aac_sim_tqh);
 374                 aac_get_bus_info(sc);
 375         }
 376
 377         return(0);
 378 }
 379
 380 void
 381 aac_add_event(struct aac_softc *sc, struct aac_event *event)
 382 {
 383
 384         switch (event->ev_type & AAC_EVENT_MASK) {
 385         case AAC_EVENT_CMFREE:
 386                 TAILQ_INSERT_TAIL(&sc->aac_ev_cmfree, event, ev_links);
 387                 break;
 388         default:
 389                 device_printf(sc->aac_dev, "aac_add event: unknown event %d\n",
 390                     event->ev_type);
 391                 break;
 392         }
 393
 394         return;
 395 }
 396
 397 /*
 398  * Probe for containers, create disks.
 399  */
 400 static void
 401 aac_startup(void *arg)
 402 {
 403         struct aac_softc *sc;
 404         struct aac_fib *fib;
 405         struct aac_mntinfo *mi;
 406         struct aac_mntinforesp *mir = NULL;
 407         int count = 0, i = 0;
 408
 409         debug_called(1);
 410
 411         sc = (struct aac_softc *)arg;
 412
 413         /* disconnect ourselves from the intrhook chain */
 414         config_intrhook_disestablish(&sc->aac_ich);
 415
 416         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 417         aac_alloc_sync_fib(sc, &fib);
 418         mi = (struct aac_mntinfo *)&fib->data[0];
 419
 420         /* loop over possible containers */
 421         do {
 422                 /* request information on this container */
 423                 bzero(mi, sizeof(struct aac_mntinfo));
 424                 mi->Command = VM_NameServe;
 425                 mi->MntType = FT_FILESYS;
 426                 mi->MntCount = i;
 427                 if (aac_sync_fib(sc, ContainerCommand, 0, fib,
 428                                  sizeof(struct aac_mntinfo))) {
 429                         device_printf(sc->aac_dev,
 430                             "error probing container %d", i);
 431
 432                         continue;
 433                 }
 434
 435                 mir = (struct aac_mntinforesp *)&fib->data[0];
 436                 /* XXX Need to check if count changed */
 437                 count = mir->MntRespCount;
 438                 aac_add_container(sc, mir, 0);
 439                 i++;
 440         } while ((i < count) && (i < AAC_MAX_CONTAINERS));
 441
 442         aac_release_sync_fib(sc);
 443         AAC_LOCK_RELEASE(&sc->aac_io_lock);
 444
 445         /* poke the bus to actually attach the child devices */
 446         if (bus_generic_attach(sc->aac_dev))
 447                 device_printf(sc->aac_dev, "bus_generic_attach failed\n");
 448
 449         /* mark the controller up */
 450         sc->aac_state &= ~AAC_STATE_SUSPEND;
 451
 452         /* enable interrupts now */
 453         AAC_UNMASK_INTERRUPTS(sc);
 454 }
 455
 456 /*
 457  * Create a device to respresent a new container
 458  */
 459 static void
 460 aac_add_container(struct aac_softc *sc, struct aac_mntinforesp *mir, int f)
 461 {
 462         struct aac_container *co;
 463         device_t child;
 464
 465         /*
 466          * Check container volume type for validity.  Note that many of
 467          * the possible types may never show up.
 468          */
 469         if ((mir->Status == ST_OK) && (mir->MntTable[0].VolType != CT_NONE)) {
 470                 co = (struct aac_container *)kmalloc(sizeof *co, M_AACBUF,
 471                        M_INTWAIT | M_ZERO);
 472                 debug(1, "id %x  name '%.16s'  size %u  type %d",
 473                       mir->MntTable[0].ObjectId,
 474                       mir->MntTable[0].FileSystemName,
 475                       mir->MntTable[0].Capacity, mir->MntTable[0].VolType);
 476
 477                 if ((child = device_add_child(sc->aac_dev, "aacd", -1)) == NULL)
 478                         device_printf(sc->aac_dev, "device_add_child failed\n");
 479                 else
 480                         device_set_ivars(child, co);
 481                 device_set_desc(child, aac_describe_code(aac_container_types,
 482                                 mir->MntTable[0].VolType));
 483                 co->co_disk = child;
 484                 co->co_found = f;
 485                 bcopy(&mir->MntTable[0], &co->co_mntobj,
 486                       sizeof(struct aac_mntobj));
 487                 AAC_LOCK_ACQUIRE(&sc->aac_container_lock);
 488                 TAILQ_INSERT_TAIL(&sc->aac_container_tqh, co, co_link);
 489                 AAC_LOCK_RELEASE(&sc->aac_container_lock);
 490         }
 491 }
 492
 493 /*
 494  * Free all of the resources associated with (sc)
 495  *
 496  * Should not be called if the controller is active.
 497  */
 498 void
 499 aac_free(struct aac_softc *sc)
 500 {
 501
 502         debug_called(1);
 503
 504         /* remove the control device */
 505         if (sc->aac_dev_t != NULL)
 506                 destroy_dev(sc->aac_dev_t);
 507
 508         /* throw away any FIB buffers, discard the FIB DMA tag */
 509         aac_free_commands(sc);
 510         if (sc->aac_fib_dmat)
 511                 bus_dma_tag_destroy(sc->aac_fib_dmat);
 512
 513         kfree(sc->aac_commands, M_AACBUF);
 514
 515         /* destroy the common area */
 516         if (sc->aac_common) {
 517                 bus_dmamap_unload(sc->aac_common_dmat, sc->aac_common_dmamap);
 518                 bus_dmamem_free(sc->aac_common_dmat, sc->aac_common,
 519                                 sc->aac_common_dmamap);
 520         }
 521         if (sc->aac_common_dmat)
 522                 bus_dma_tag_destroy(sc->aac_common_dmat);
 523
 524         /* disconnect the interrupt handler */
 525         if (sc->aac_intr)
 526                 bus_teardown_intr(sc->aac_dev, sc->aac_irq, sc->aac_intr);
 527         if (sc->aac_irq != NULL)
 528                 bus_release_resource(sc->aac_dev, SYS_RES_IRQ, sc->aac_irq_rid,
 529                                      sc->aac_irq);
 530
 531         /* destroy data-transfer DMA tag */
 532         if (sc->aac_buffer_dmat)
 533                 bus_dma_tag_destroy(sc->aac_buffer_dmat);
 534
 535         /* destroy the parent DMA tag */
 536         if (sc->aac_parent_dmat)
 537                 bus_dma_tag_destroy(sc->aac_parent_dmat);
 538
 539         /* release the register window mapping */
 540         if (sc->aac_regs_resource != NULL) {
 541                 bus_release_resource(sc->aac_dev, SYS_RES_MEMORY,
 542                                      sc->aac_regs_rid, sc->aac_regs_resource);
 543         }
 544         dev_ops_remove(&aac_ops, -1, device_get_unit(sc->aac_dev));
 545 }
 546
 547 /*
 548  * Disconnect from the controller completely, in preparation for unload.
 549  */
 550 int
 551 aac_detach(device_t dev)
 552 {
 553         struct aac_softc *sc;
 554         struct aac_container *co;
 555         struct aac_sim  *sim;
 556         int error;
 557
 558         debug_called(1);
 559
 560         sc = device_get_softc(dev);
 561
 562         callout_stop(&sc->aac_watchdog);
 563
 564         if (sc->aac_state & AAC_STATE_OPEN)
 565                 return(EBUSY);
 566
 567         /* Remove the child containers */
 568         while ((co = TAILQ_FIRST(&sc->aac_container_tqh)) != NULL) {
 569                 error = device_delete_child(dev, co->co_disk);
 570                 if (error)
 571                         return (error);
 572                 TAILQ_REMOVE(&sc->aac_container_tqh, co, co_link);
 573                 kfree(co, M_AACBUF);
 574         }
 575
 576         /* Remove the CAM SIMs */
 577         while ((sim = TAILQ_FIRST(&sc->aac_sim_tqh)) != NULL) {
 578                 TAILQ_REMOVE(&sc->aac_sim_tqh, sim, sim_link);
 579                 error = device_delete_child(dev, sim->sim_dev);
 580                 if (error)
 581                         return (error);
 582                 kfree(sim, M_AACBUF);
 583         }
 584
 585         if (sc->aifflags & AAC_AIFFLAGS_RUNNING) {
 586                 sc->aifflags |= AAC_AIFFLAGS_EXIT;
 587                 wakeup(sc->aifthread);
 588                 tsleep(sc->aac_dev, PCATCH, "aacdch", 30 * hz);
 589         }
 590
 591         if (sc->aifflags & AAC_AIFFLAGS_RUNNING)
 592                 panic("Cannot shutdown AIF thread\n");
 593
 594         if ((error = aac_shutdown(dev)))
 595                 return(error);
 596
 597         EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->eh);
 598
 599         aac_free(sc);
 600
 601         lockuninit(&sc->aac_aifq_lock);
 602         lockuninit(&sc->aac_io_lock);
 603         lockuninit(&sc->aac_container_lock);
 604
 605         return(0);
 606 }
 607
 608 /*
 609  * Bring the controller down to a dormant state and detach all child devices.
 610  *
 611  * This function is called before detach or system shutdown.
 612  *
 613  * Note that we can assume that the bioq on the controller is empty, as we won't
 614  * allow shutdown if any device is open.
 615  */
 616 static int
 617 aac_shutdown(device_t dev)
 618 {
 619         struct aac_softc *sc;
 620         struct aac_fib *fib;
 621         struct aac_close_command *cc;
 622
 623         debug_called(1);
 624
 625         sc = device_get_softc(dev);
 626
 627         sc->aac_state |= AAC_STATE_SUSPEND;
 628
 629         /*
 630          * Send a Container shutdown followed by a HostShutdown FIB to the
 631          * controller to convince it that we don't want to talk to it anymore.
 632          * We've been closed and all I/O completed already
 633          */
 634         device_printf(sc->aac_dev, "shutting down controller...");
 635
 636         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 637         aac_alloc_sync_fib(sc, &fib);
 638         cc = (struct aac_close_command *)&fib->data[0];
 639
 640         bzero(cc, sizeof(struct aac_close_command));
 641         cc->Command = VM_CloseAll;
 642         cc->ContainerId = 0xffffffff;
 643         if (aac_sync_fib(sc, ContainerCommand, 0, fib,
 644             sizeof(struct aac_close_command)))
 645                 kprintf("FAILED.\n");
 646         else
 647                 kprintf("done\n");
 648 #if 0
 649         else {
 650                 fib->data[0] = 0;
 651                 /*
 652                  * XXX Issuing this command to the controller makes it shut down
 653                  * but also keeps it from coming back up without a reset of the
 654                  * PCI bus.  This is not desirable if you are just unloading the
 655                  * driver module with the intent to reload it later.
 656                  */
 657                 if (aac_sync_fib(sc, FsaHostShutdown, AAC_FIBSTATE_SHUTDOWN,
 658                     fib, 1)) {
 659                         kprintf("FAILED.\n");
 660                 } else {
 661                         kprintf("done.\n");
 662                 }
 663         }
 664 #endif
 665
 666         AAC_MASK_INTERRUPTS(sc);
 667         aac_release_sync_fib(sc);
 668         AAC_LOCK_RELEASE(&sc->aac_io_lock);
 669
 670         return(0);
 671 }
 672
 673 /*
 674  * Bring the controller to a quiescent state, ready for system suspend.
 675  */
 676 int
 677 aac_suspend(device_t dev)
 678 {
 679         struct aac_softc *sc;
 680
 681         debug_called(1);
 682
 683         sc = device_get_softc(dev);
 684
 685         sc->aac_state |= AAC_STATE_SUSPEND;
 686
 687         AAC_MASK_INTERRUPTS(sc);
 688         return(0);
 689 }
 690
 691 /*
 692  * Bring the controller back to a state ready for operation.
 693  */
 694 int
 695 aac_resume(device_t dev)
 696 {
 697         struct aac_softc *sc;
 698
 699         debug_called(1);
 700
 701         sc = device_get_softc(dev);
 702
 703         sc->aac_state &= ~AAC_STATE_SUSPEND;
 704         AAC_UNMASK_INTERRUPTS(sc);
 705         return(0);
 706 }
 707
 708 /*
 709  * Interrupt handler for NEW_COMM interface.
 710  */
 711 void
 712 aac_new_intr(void *arg)
 713 {
 714         struct aac_softc *sc;
 715         u_int32_t index, fast;
 716         struct aac_command *cm;
 717         struct aac_fib *fib;
 718         int i;
 719
 720         debug_called(2);
 721
 722         sc = (struct aac_softc *)arg;
 723
 724         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 725         while (1) {
 726                 index = AAC_GET_OUTB_QUEUE(sc);
 727                 if (index == 0xffffffff)
 728                         index = AAC_GET_OUTB_QUEUE(sc);
 729                 if (index == 0xffffffff)
 730                         break;
 731                 if (index & 2) {
 732                         if (index == 0xfffffffe) {
 733                                 /* XXX This means that the controller wants
 734                                  * more work.  Ignore it for now.
 735                                  */
 736                                 continue;
 737                         }
 738                         /* AIF */
 739                         fib = (struct aac_fib *)kmalloc(sizeof *fib, M_AACBUF,
 740                                    M_INTWAIT | M_ZERO);
 741                         index &= ~2;
 742                         for (i = 0; i < sizeof(struct aac_fib)/4; ++i)
 743                                 ((u_int32_t *)fib)[i] = AAC_GETREG4(sc, index + i*4);
 744                         aac_handle_aif(sc, fib);
 745                         kfree(fib, M_AACBUF);
 746
 747                         /*
 748                          * AIF memory is owned by the adapter, so let it
 749                          * know that we are done with it.
 750                          */
 751                         AAC_SET_OUTB_QUEUE(sc, index);
 752                         AAC_CLEAR_ISTATUS(sc, AAC_DB_RESPONSE_READY);
 753                 } else {
 754                         fast = index & 1;
 755                         cm = sc->aac_commands + (index >> 2);
 756                         fib = cm->cm_fib;
 757                         if (fast) {
 758                                 fib->Header.XferState |= AAC_FIBSTATE_DONEADAP;
 759                                 *((u_int32_t *)(fib->data)) = AAC_ERROR_NORMAL;
 760                         }
 761                         aac_remove_busy(cm);
 762                         aac_unmap_command(cm);
 763                         cm->cm_flags |= AAC_CMD_COMPLETED;
 764
 765                         /* is there a completion handler? */
 766                         if (cm->cm_complete != NULL) {
 767                                 cm->cm_complete(cm);
 768                         } else {
 769                                 /* assume that someone is sleeping on this
 770                                  * command
 771                                  */
 772                                 wakeup(cm);
 773                         }
 774                         sc->flags &= ~AAC_QUEUE_FRZN;
 775                 }
 776         }
 777         /* see if we can start some more I/O */
 778         if ((sc->flags & AAC_QUEUE_FRZN) == 0)
 779                 aac_startio(sc);
 780
 781         AAC_LOCK_RELEASE(&sc->aac_io_lock);
 782 }
 783
 784 void
 785 aac_fast_intr(void *arg)
 786 {
 787         struct aac_softc *sc;
 788         u_int16_t reason;
 789
 790         debug_called(2);
 791
 792         sc = (struct aac_softc *)arg;
 793
 794         /*
 795          * Read the status register directly.  This is faster than taking the
 796          * driver lock and reading the queues directly.  It also saves having
 797          * to turn parts of the driver lock into a spin mutex, which would be
 798          * ugly.
 799          */
 800         reason = AAC_GET_ISTATUS(sc);
 801         AAC_CLEAR_ISTATUS(sc, reason);
 802
 803         /* handle completion processing */
 804         if (reason & AAC_DB_RESPONSE_READY)
 805                 taskqueue_enqueue(taskqueue_swi, &sc->aac_task_complete);
 806
 807         /* controller wants to talk to us */
 808         if (reason & (AAC_DB_PRINTF | AAC_DB_COMMAND_READY)) {
 809                 /*
 810                  * XXX Make sure that we don't get fooled by strange messages
 811                  * that start with a NULL.
 812                  */
 813                 if ((reason & AAC_DB_PRINTF) &&
 814                         (sc->aac_common->ac_printf[0] == 0))
 815                         sc->aac_common->ac_printf[0] = 32;
 816
 817                 /*
 818                  * This might miss doing the actual wakeup.  However, the
 819                  * msleep that this is waking up has a timeout, so it will
 820                  * wake up eventually.  AIFs and printfs are low enough
 821                  * priority that they can handle hanging out for a few seconds
 822                  * if needed.
 823                  */
 824                 wakeup(sc->aifthread);
 825         }
 826 }
 827
 828 /*
 829  * Command Processing
 830  */
 831
 832 /*
 833  * Start as much queued I/O as possible on the controller
 834  */
 835 void
 836 aac_startio(struct aac_softc *sc)
 837 {
 838         struct aac_command *cm;
 839
 840         debug_called(2);
 841
 842         if (sc->flags & AAC_QUEUE_FRZN)
 843                 return;
 844
 845         for (;;) {
 846                 /*
 847                  * Try to get a command that's been put off for lack of
 848                  * resources
 849                  */
 850                 cm = aac_dequeue_ready(sc);
 851
 852                 /*
 853                  * Try to build a command off the bio queue (ignore error
 854                  * return)
 855                  */
 856                 if (cm == NULL)
 857                         aac_bio_command(sc, &cm);
 858
 859                 /* nothing to do? */
 860                 if (cm == NULL)
 861                         break;
 862
 863                 /*
 864                  * Try to give the command to the controller.  Any error is
 865                  * catastrophic since it means that bus_dmamap_load() failed.
 866                  */
 867                 if (aac_map_command(cm) != 0)
 868                         panic("aac: error mapping command %p\n", cm);
 869         }
 870 }
 871
 872 /*
 873  * Deliver a command to the controller; allocate controller resources at the
 874  * last moment when possible.
 875  */
 876 static int
 877 aac_map_command(struct aac_command *cm)
 878 {
 879         struct aac_softc *sc;
 880         int error;
 881
 882         debug_called(2);
 883
 884         sc = cm->cm_sc;
 885         error = 0;
 886
 887         /* don't map more than once */
 888         if (cm->cm_flags & AAC_CMD_MAPPED)
 889                 panic("aac: command %p already mapped", cm);
 890
 891         if (cm->cm_datalen != 0) {
 892                 error = bus_dmamap_load(sc->aac_buffer_dmat, cm->cm_datamap,
 893                                         cm->cm_data, cm->cm_datalen,
 894                                         aac_map_command_sg, cm, 0);
 895                 if (error == EINPROGRESS) {
 896                         debug(1, "freezing queue\n");
 897                         sc->flags |= AAC_QUEUE_FRZN;
 898                         error = 0;
 899                 }
 900         } else {
 901                 aac_map_command_sg(cm, NULL, 0, 0);
 902         }
 903         return (error);
 904 }
 905
 906 /*
 907  * Handle notification of one or more FIBs coming from the controller.
 908  */
 909 static void
 910 aac_command_thread(struct aac_softc *sc)
 911 {
 912         struct aac_fib *fib;
 913         u_int32_t fib_size;
 914         int size, retval;
 915
 916         debug_called(2);
 917
 918         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 919         sc->aifflags = AAC_AIFFLAGS_RUNNING;
 920
 921         while ((sc->aifflags & AAC_AIFFLAGS_EXIT) == 0) {
 922                 retval = 0;
 923                 if ((sc->aifflags & AAC_AIFFLAGS_PENDING) == 0) {
 924                         crit_enter();
 925                         tsleep_interlock(sc->aifthread);
 926                         AAC_LOCK_RELEASE(&sc->aac_io_lock);
 927                         retval = tsleep(sc->aifthread, 0,
 928                                         "aifthd", AAC_PERIODIC_INTERVAL * hz);
 929                         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 930                         crit_exit();
 931                 }
 932                 /*
 933                  * First see if any FIBs need to be allocated.  This needs
 934                  * to be called without the driver lock because contigmalloc
 935                  * will grab Giant, and would result in an LOR.
 936                  */
 937                 if ((sc->aifflags & AAC_AIFFLAGS_ALLOCFIBS) != 0) {
 938                         AAC_LOCK_RELEASE(&sc->aac_io_lock);
 939                         aac_alloc_commands(sc);
 940                         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
 941                         sc->aifflags &= ~AAC_AIFFLAGS_ALLOCFIBS;
 942                         aac_startio(sc);
 943                 }
 944
 945                 /*
 946                  * While we're here, check to see if any commands are stuck.
 947                  * This is pretty low-priority, so it's ok if it doesn't
 948                  * always fire.
 949                  */
 950                 if (retval == EWOULDBLOCK)
 951                         aac_timeout(sc);
 952
 953                 /* Check the hardware printf message buffer */
 954                 if (sc->aac_common->ac_printf[0] != 0)
 955                         aac_print_printf(sc);
 956
 957                 /* Also check to see if the adapter has a command for us. */
 958                 if (sc->flags & AAC_FLAGS_NEW_COMM)
 959                         continue;
 960                 for (;;) {
 961                         if (aac_dequeue_fib(sc, AAC_HOST_NORM_CMD_QUEUE,
 962                                             &fib_size, &fib))
 963                                 break;
 964
 965                         AAC_PRINT_FIB(sc, fib);
 966
 967                         switch (fib->Header.Command) {
 968                         case AifRequest:
 969                                 aac_handle_aif(sc, fib);
 970                                 break;
 971                         default:
 972                                 device_printf(sc->aac_dev, "unknown command "
 973                                               "from controller\n");
 974                                 break;
 975                         }
 976
 977                         if ((fib->Header.XferState == 0) ||
 978                             (fib->Header.StructType != AAC_FIBTYPE_TFIB)) {
 979                                 break;
 980                         }
 981
 982                         /* Return the AIF to the controller. */
 983                         if (fib->Header.XferState & AAC_FIBSTATE_FROMADAP) {
 984                                 fib->Header.XferState |= AAC_FIBSTATE_DONEHOST;
 985                                 *(AAC_FSAStatus*)fib->data = ST_OK;
 986
 987                                 /* XXX Compute the Size field? */
 988                                 size = fib->Header.Size;
 989                                 if (size > sizeof(struct aac_fib)) {
 990                                         size = sizeof(struct aac_fib);
 991                                         fib->Header.Size = size;
 992                                 }
 993                                 /*
 994                                  * Since we did not generate this command, it
 995                                  * cannot go through the normal
 996                                  * enqueue->startio chain.
 997                                  */
 998                                 aac_enqueue_response(sc,
 999                                                  AAC_ADAP_NORM_RESP_QUEUE,
1000                                                  fib);
1001                         }
1002                 }
1003         }
1004         sc->aifflags &= ~AAC_AIFFLAGS_RUNNING;
1005         AAC_LOCK_RELEASE(&sc->aac_io_lock);
1006         wakeup(sc->aac_dev);
1007
1008         kthread_exit();
1009 }
1010
1011 /*
1012  * Process completed commands.
1013  */
1014 static void
1015 aac_complete(void *context, int pending)
1016 {
1017         struct aac_softc *sc;
1018         struct aac_command *cm;
1019         struct aac_fib *fib;
1020         u_int32_t fib_size;
1021
1022         debug_called(2);
1023
1024         sc = (struct aac_softc *)context;
1025
1026         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
1027
1028         /* pull completed commands off the queue */
1029         for (;;) {
1030                 /* look for completed FIBs on our queue */
1031                 if (aac_dequeue_fib(sc, AAC_HOST_NORM_RESP_QUEUE, &fib_size,
1032                                                         &fib))
1033                         break;  /* nothing to do */
1034
1035                 /* get the command, unmap and queue for later processing */
1036                 cm = sc->aac_commands + fib->Header.SenderData;
1037                 if (cm == NULL) {
1038                         AAC_PRINT_FIB(sc, fib);
1039                         break;
1040                 }
1041                 aac_remove_busy(cm);
1042                 aac_unmap_command(cm);          /* XXX defer? */
1043                 cm->cm_flags |= AAC_CMD_COMPLETED;
1044
1045                 /* is there a completion handler? */
1046                 if (cm->cm_complete != NULL) {
1047                         cm->cm_complete(cm);
1048                 } else {
1049                         /* assume that someone is sleeping on this command */
1050                         wakeup(cm);
1051                 }
1052         }
1053
1054         /* see if we can start some more I/O */
1055         sc->flags &= ~AAC_QUEUE_FRZN;
1056         aac_startio(sc);
1057
1058         AAC_LOCK_RELEASE(&sc->aac_io_lock);
1059 }
1060
1061 /*
1062  * Handle a bio submitted from a disk device.
1063  */
1064 void
1065 aac_submit_bio(struct aac_disk *ad, struct bio *bio)
1066 {
1067         struct aac_softc *sc;
1068
1069         debug_called(2);
1070
1071         bio->bio_driver_info = ad;
1072         sc = ad->ad_controller;
1073
1074         /* queue the BIO and try to get some work done */
1075         aac_enqueue_bio(sc, bio);
1076         aac_startio(sc);
1077 }
1078
1079 /*
1080  * Get a bio and build a command to go with it.
1081  */
1082 static int
1083 aac_bio_command(struct aac_softc *sc, struct aac_command **cmp)
1084 {
1085         struct aac_command *cm;
1086         struct aac_fib *fib;
1087         struct aac_disk *ad;
1088         struct bio *bio;
1089         struct buf *bp;
1090
1091         debug_called(2);
1092
1093         /* get the resources we will need */
1094         cm = NULL;
1095         bio = NULL;
1096         if (aac_alloc_command(sc, &cm)) /* get a command */
1097                 goto fail;
1098         if ((bio = aac_dequeue_bio(sc)) == NULL)
1099                 goto fail;
1100
1101         /* fill out the command */
1102         bp = bio->bio_buf;
1103         cm->cm_data = (void *)bp->b_data;
1104         cm->cm_datalen = bp->b_bcount;
1105         cm->cm_complete = aac_bio_complete;
1106         cm->cm_private = bio;
1107         cm->cm_timestamp = time_second;
1108         cm->cm_queue = AAC_ADAP_NORM_CMD_QUEUE;
1109
1110         /* build the FIB */
1111         fib = cm->cm_fib;
1112         fib->Header.Size = sizeof(struct aac_fib_header);
1113         fib->Header.XferState =
1114                 AAC_FIBSTATE_HOSTOWNED   |
1115                 AAC_FIBSTATE_INITIALISED |
1116                 AAC_FIBSTATE_EMPTY       |
1117                 AAC_FIBSTATE_FROMHOST    |
1118                 AAC_FIBSTATE_REXPECTED   |
1119                 AAC_FIBSTATE_NORM        |
1120                 AAC_FIBSTATE_ASYNC       |
1121                 AAC_FIBSTATE_FAST_RESPONSE;
1122
1123         /* build the read/write request */
1124         ad = (struct aac_disk *)bio->bio_driver_info;
1125
1126         if (sc->flags & AAC_FLAGS_RAW_IO) {
1127                 struct aac_raw_io *raw;
1128                 raw = (struct aac_raw_io *)&fib->data[0];
1129                 fib->Header.Command = RawIo;
1130                 raw->BlockNumber = bio->bio_offset / AAC_BLOCK_SIZE;
1131                 raw->ByteCount = bp->b_bcount;
1132                 raw->ContainerId = ad->ad_container->co_mntobj.ObjectId;
1133                 raw->BpTotal = 0;
1134                 raw->BpComplete = 0;
1135                 fib->Header.Size += sizeof(struct aac_raw_io);
1136                 cm->cm_sgtable = (struct aac_sg_table *)&raw->SgMapRaw;
1137                 if (bp->b_cmd == BUF_CMD_READ) {
1138                         raw->Flags = 1;
1139                         cm->cm_flags |= AAC_CMD_DATAIN;
1140                 } else {
1141                         raw->Flags = 0;
1142                         cm->cm_flags |= AAC_CMD_DATAOUT;
1143                 }
1144         } else if ((sc->flags & AAC_FLAGS_SG_64BIT) == 0) {
1145                 fib->Header.Command = ContainerCommand;
1146                 if (bp->b_cmd == BUF_CMD_READ) {
1147                         struct aac_blockread *br;
1148                         br = (struct aac_blockread *)&fib->data[0];
1149                         br->Command = VM_CtBlockRead;
1150                         br->ContainerId = ad->ad_container->co_mntobj.ObjectId;
1151                         br->BlockNumber = bio->bio_offset / AAC_BLOCK_SIZE;
1152                         br->ByteCount = bp->b_bcount;
1153                         fib->Header.Size += sizeof(struct aac_blockread);
1154                         cm->cm_sgtable = &br->SgMap;
1155                         cm->cm_flags |= AAC_CMD_DATAIN;
1156                 } else {
1157                         struct aac_blockwrite *bw;
1158                         bw = (struct aac_blockwrite *)&fib->data[0];
1159                         bw->Command = VM_CtBlockWrite;
1160                         bw->ContainerId = ad->ad_container->co_mntobj.ObjectId;
1161                         bw->BlockNumber = bio->bio_offset / AAC_BLOCK_SIZE;
1162                         bw->ByteCount = bp->b_bcount;
1163                         bw->Stable = CUNSTABLE;
1164                         fib->Header.Size += sizeof(struct aac_blockwrite);
1165                         cm->cm_flags |= AAC_CMD_DATAOUT;
1166                         cm->cm_sgtable = &bw->SgMap;
1167                 }
1168         } else {
1169                 fib->Header.Command = ContainerCommand64;
1170                 if (bp->b_cmd == BUF_CMD_READ) {
1171                         struct aac_blockread64 *br;
1172                         br = (struct aac_blockread64 *)&fib->data[0];
1173                         br->Command = VM_CtHostRead64;
1174                         br->ContainerId = ad->ad_container->co_mntobj.ObjectId;
1175                         br->SectorCount = bp->b_bcount / AAC_BLOCK_SIZE;
1176                         br->BlockNumber = bio->bio_offset / AAC_BLOCK_SIZE;
1177                         br->Pad = 0;
1178                         br->Flags = 0;
1179                         fib->Header.Size += sizeof(struct aac_blockread64);
1180                         cm->cm_flags |= AAC_CMD_DATAOUT;
1181                         cm->cm_sgtable = (struct aac_sg_table *)&br->SgMap64;
1182                 } else {
1183                         struct aac_blockwrite64 *bw;
1184                         bw = (struct aac_blockwrite64 *)&fib->data[0];
1185                         bw->Command = VM_CtHostWrite64;
1186                         bw->ContainerId = ad->ad_container->co_mntobj.ObjectId;
1187                         bw->SectorCount = bp->b_bcount / AAC_BLOCK_SIZE;
1188                         bw->BlockNumber = bio->bio_offset / AAC_BLOCK_SIZE;
1189                         bw->Pad = 0;
1190                         bw->Flags = 0;
1191                         fib->Header.Size += sizeof(struct aac_blockwrite64);
1192                         cm->cm_flags |= AAC_CMD_DATAIN;
1193                         cm->cm_sgtable = (struct aac_sg_table *)&bw->SgMap64;
1194                 }
1195         }
1196
1197         *cmp = cm;
1198         return(0);
1199
1200 fail:
1201         if (bio != NULL)
1202                 aac_enqueue_bio(sc, bio);
1203         if (cm != NULL)
1204                 aac_release_command(cm);
1205         return(ENOMEM);
1206 }
1207
1208 /*
1209  * Handle a bio-instigated command that has been completed.
1210  */
1211 static void
1212 aac_bio_complete(struct aac_command *cm)
1213 {
1214         struct aac_blockread_response *brr;
1215         struct aac_blockwrite_response *bwr;
1216         struct bio *bio;
1217         struct buf *bp;
1218         const char *code;
1219         AAC_FSAStatus status;
1220
1221         /* fetch relevant status and then release the command */
1222         bio = (struct bio *)cm->cm_private;
1223         bp = bio->bio_buf;
1224         if (bp->b_cmd == BUF_CMD_READ) {
1225                 brr = (struct aac_blockread_response *)&cm->cm_fib->data[0];
1226                 status = brr->Status;
1227         } else {
1228                 bwr = (struct aac_blockwrite_response *)&cm->cm_fib->data[0];
1229                 status = bwr->Status;
1230         }
1231         aac_release_command(cm);
1232
1233         /* fix up the bio based on status */
1234         if (status == ST_OK) {
1235                 bp->b_resid = 0;
1236                 code = 0;
1237         } else {
1238                 bp->b_error = EIO;
1239                 bp->b_flags |= B_ERROR;
1240                 /* pass an error string out to the disk layer */
1241                 code = aac_describe_code(aac_command_status_table, status);
1242         }
1243         aac_biodone(bio, code);
1244 }
1245
1246 /*
1247  * Dump a block of data to the controller.  If the queue is full, tell the
1248  * caller to hold off and wait for the queue to drain.
1249  */
1250 int
1251 aac_dump_enqueue(struct aac_disk *ad, u_int64_t lba, void *data, int dumppages)
1252 {
1253         struct aac_softc *sc;
1254         struct aac_command *cm;
1255         struct aac_fib *fib;
1256         struct aac_blockwrite *bw;
1257
1258         sc = ad->ad_controller;
1259         cm = NULL;
1260
1261         KKASSERT(lba <= 0x100000000ULL);
1262
1263         if (aac_alloc_command(sc, &cm))
1264                 return (EBUSY);
1265
1266         /* fill out the command */
1267         cm->cm_data = data;
1268         cm->cm_datalen = dumppages * PAGE_SIZE;
1269         cm->cm_complete = NULL;
1270         cm->cm_private = NULL;
1271         cm->cm_timestamp = time_second;
1272         cm->cm_queue = AAC_ADAP_NORM_CMD_QUEUE;
1273
1274         /* build the FIB */
1275         fib = cm->cm_fib;
1276         fib->Header.XferState =
1277         AAC_FIBSTATE_HOSTOWNED   |
1278         AAC_FIBSTATE_INITIALISED |
1279         AAC_FIBSTATE_FROMHOST    |
1280         AAC_FIBSTATE_REXPECTED   |
1281         AAC_FIBSTATE_NORM;
1282         fib->Header.Command = ContainerCommand;
1283         fib->Header.Size = sizeof(struct aac_fib_header);
1284
1285         bw = (struct aac_blockwrite *)&fib->data[0];
1286         bw->Command = VM_CtBlockWrite;
1287         bw->ContainerId = ad->ad_container->co_mntobj.ObjectId;
1288         bw->BlockNumber = lba;
1289         bw->ByteCount = dumppages * PAGE_SIZE;
1290         bw->Stable = CUNSTABLE;         /* XXX what's appropriate here? */
1291         fib->Header.Size += sizeof(struct aac_blockwrite);
1292         cm->cm_flags |= AAC_CMD_DATAOUT;
1293         cm->cm_sgtable = &bw->SgMap;
1294
1295         return (aac_map_command(cm));
1296 }
1297
1298 /*
1299  * Wait for the card's queue to drain when dumping.  Also check for monitor
1300  * kprintf's
1301  */
1302 void
1303 aac_dump_complete(struct aac_softc *sc)
1304 {
1305         struct aac_fib *fib;
1306         struct aac_command *cm;
1307         u_int16_t reason;
1308         u_int32_t pi, ci, fib_size;
1309
1310         do {
1311                 reason = AAC_GET_ISTATUS(sc);
1312                 if (reason & AAC_DB_RESPONSE_READY) {
1313                         AAC_CLEAR_ISTATUS(sc, AAC_DB_RESPONSE_READY);
1314                         for (;;) {
1315                                 if (aac_dequeue_fib(sc,
1316                                                     AAC_HOST_NORM_RESP_QUEUE,
1317                                                     &fib_size, &fib))
1318                                         break;
1319                                 cm = (struct aac_command *)
1320                                         fib->Header.SenderData;
1321                                 if (cm == NULL)
1322                                         AAC_PRINT_FIB(sc, fib);
1323                                 else {
1324                                         aac_remove_busy(cm);
1325                                         aac_unmap_command(cm);
1326                                         aac_enqueue_complete(cm);
1327                                         aac_release_command(cm);
1328                                 }
1329                         }
1330                 }
1331                 if (reason & AAC_DB_PRINTF) {
1332                         AAC_CLEAR_ISTATUS(sc, AAC_DB_PRINTF);
1333                         aac_print_printf(sc);
1334                 }
1335                 pi = sc->aac_queues->qt_qindex[AAC_ADAP_NORM_CMD_QUEUE][
1336                         AAC_PRODUCER_INDEX];
1337                 ci = sc->aac_queues->qt_qindex[AAC_ADAP_NORM_CMD_QUEUE][
1338                         AAC_CONSUMER_INDEX];
1339         } while (ci != pi);
1340
1341         return;
1342 }
1343
1344 /*
1345  * Submit a command to the controller, return when it completes.
1346  * XXX This is very dangerous!  If the card has gone out to lunch, we could
1347  *     be stuck here forever.  At the same time, signals are not caught
1348  *     because there is a risk that a signal could wakeup the sleep before
1349  *     the card has a chance to complete the command.  Since there is no way
1350  *     to cancel a command that is in progress, we can't protect against the
1351  *     card completing a command late and spamming the command and data
1352  *     memory.  So, we are held hostage until the command completes.
1353  */
1354 static int
1355 aac_wait_command(struct aac_command *cm)
1356 {
1357         struct aac_softc *sc;
1358         int error;
1359
1360         debug_called(2);
1361
1362         sc = cm->cm_sc;
1363
1364         /* Put the command on the ready queue and get things going */
1365         cm->cm_queue = AAC_ADAP_NORM_CMD_QUEUE;
1366         aac_enqueue_ready(cm);
1367         aac_startio(sc);
1368         /* Lock is held */
1369         KKASSERT(lockstatus(&sc->aac_io_lock, curthread) != 0);
1370         crit_enter();
1371         tsleep_interlock(cm);
1372         AAC_LOCK_RELEASE(&sc->aac_io_lock);
1373         error = tsleep(cm, 0, "aacwait", 0);
1374         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
1375         crit_exit();
1376         return(error);
1377 }
1378
1379 /*
1380  *Command Buffer Management
1381  */
1382
1383 /*
1384  * Allocate a command.
1385  */
1386 int
1387 aac_alloc_command(struct aac_softc *sc, struct aac_command **cmp)
1388 {
1389         struct aac_command *cm;
1390
1391         debug_called(3);
1392
1393         if ((cm = aac_dequeue_free(sc)) == NULL) {
1394                 if (sc->total_fibs < sc->aac_max_fibs) {
1395                         sc->aifflags |= AAC_AIFFLAGS_ALLOCFIBS;
1396                         wakeup(sc->aifthread);
1397                 }
1398                 return (EBUSY);
1399         }
1400
1401         *cmp = cm;
1402         return(0);
1403 }
1404
1405 /*
1406  * Release a command back to the freelist.
1407  */
1408 void
1409 aac_release_command(struct aac_command *cm)
1410 {
1411         struct aac_event *event;
1412         struct aac_softc *sc;
1413
1414         debug_called(3);
1415
1416         /* (re)initialise the command/FIB */
1417         cm->cm_sgtable = NULL;
1418         cm->cm_flags = 0;
1419         cm->cm_complete = NULL;
1420         cm->cm_private = NULL;
1421         cm->cm_fib->Header.XferState = AAC_FIBSTATE_EMPTY;
1422         cm->cm_fib->Header.StructType = AAC_FIBTYPE_TFIB;
1423         cm->cm_fib->Header.Flags = 0;
1424         cm->cm_fib->Header.SenderSize = cm->cm_sc->aac_max_fib_size;
1425
1426         /*
1427          * These are duplicated in aac_start to cover the case where an
1428          * intermediate stage may have destroyed them.  They're left
1429          * initialised here for debugging purposes only.
1430          */
1431         cm->cm_fib->Header.ReceiverFibAddress = (u_int32_t)cm->cm_fibphys;
1432         cm->cm_fib->Header.SenderData = 0;
1433
1434         aac_enqueue_free(cm);
1435
1436         sc = cm->cm_sc;
1437         event = TAILQ_FIRST(&sc->aac_ev_cmfree);
1438         if (event != NULL) {
1439                 TAILQ_REMOVE(&sc->aac_ev_cmfree, event, ev_links);
1440                 event->ev_callback(sc, event, event->ev_arg);
1441         }
1442 }
1443
1444 /*
1445  * Map helper for command/FIB allocation.
1446  */
1447 static void
1448 aac_map_command_helper(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1449 {
1450         uint64_t        *fibphys;
1451
1452         fibphys = (uint64_t *)arg;
1453
1454         debug_called(3);
1455
1456         *fibphys = segs[0].ds_addr;
1457 }
1458
1459 /*
1460  * Allocate and initialise commands/FIBs for this adapter.
1461  */
1462 static int
1463 aac_alloc_commands(struct aac_softc *sc)
1464 {
1465         struct aac_command *cm;
1466         struct aac_fibmap *fm;
1467         uint64_t fibphys;
1468         int i, error;
1469
1470         debug_called(2);
1471
1472         if (sc->total_fibs + sc->aac_max_fibs_alloc > sc->aac_max_fibs)
1473                 return (ENOMEM);
1474
1475         fm = kmalloc(sizeof(struct aac_fibmap), M_AACBUF, M_INTWAIT | M_ZERO);
1476
1477         /* allocate the FIBs in DMAable memory and load them */
1478         if (bus_dmamem_alloc(sc->aac_fib_dmat, (void **)&fm->aac_fibs,
1479                              BUS_DMA_NOWAIT, &fm->aac_fibmap)) {
1480                 device_printf(sc->aac_dev,
1481                               "Not enough contiguous memory available.\n");
1482                 kfree(fm, M_AACBUF);
1483                 return (ENOMEM);
1484         }
1485
1486         /* Ignore errors since this doesn't bounce */
1487         bus_dmamap_load(sc->aac_fib_dmat, fm->aac_fibmap, fm->aac_fibs,
1488                         sc->aac_max_fibs_alloc * sc->aac_max_fib_size,
1489                         aac_map_command_helper, &fibphys, 0);
1490
1491         /* initialise constant fields in the command structure */
1492         bzero(fm->aac_fibs, sc->aac_max_fibs_alloc * sc->aac_max_fib_size);
1493         for (i = 0; i < sc->aac_max_fibs_alloc; i++) {
1494                 cm = sc->aac_commands + sc->total_fibs;
1495                 fm->aac_commands = cm;
1496                 cm->cm_sc = sc;
1497                 cm->cm_fib = (struct aac_fib *)
1498                         ((u_int8_t *)fm->aac_fibs + i*sc->aac_max_fib_size);
1499                 cm->cm_fibphys = fibphys + i*sc->aac_max_fib_size;
1500                 cm->cm_index = sc->total_fibs;
1501
1502                 if ((error = bus_dmamap_create(sc->aac_buffer_dmat, 0,
1503                                                &cm->cm_datamap)) != 0)
1504                         break;
1505                 AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
1506                 aac_release_command(cm);
1507                 sc->total_fibs++;
1508                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
1509         }
1510
1511         if (i > 0) {
1512                 AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
1513                 TAILQ_INSERT_TAIL(&sc->aac_fibmap_tqh, fm, fm_link);
1514                 debug(1, "total_fibs= %d\n", sc->total_fibs);
1515                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
1516                 return (0);
1517         }
1518
1519         bus_dmamap_unload(sc->aac_fib_dmat, fm->aac_fibmap);
1520         bus_dmamem_free(sc->aac_fib_dmat, fm->aac_fibs, fm->aac_fibmap);
1521         kfree(fm, M_AACBUF);
1522         return (ENOMEM);
1523 }
1524
1525 /*
1526  * Free FIBs owned by this adapter.
1527  */
1528 static void
1529 aac_free_commands(struct aac_softc *sc)
1530 {
1531         struct aac_fibmap *fm;
1532         struct aac_command *cm;
1533         int i;
1534
1535         debug_called(1);
1536
1537         while ((fm = TAILQ_FIRST(&sc->aac_fibmap_tqh)) != NULL) {
1538
1539                 TAILQ_REMOVE(&sc->aac_fibmap_tqh, fm, fm_link);
1540                 /*
1541                  * We check against total_fibs to handle partially
1542                  * allocated blocks.
1543                  */
1544                 for (i = 0; i < sc->aac_max_fibs_alloc && sc->total_fibs--; i++) {
1545                         cm = fm->aac_commands + i;
1546                         bus_dmamap_destroy(sc->aac_buffer_dmat, cm->cm_datamap);
1547                 }
1548                 bus_dmamap_unload(sc->aac_fib_dmat, fm->aac_fibmap);
1549                 bus_dmamem_free(sc->aac_fib_dmat, fm->aac_fibs, fm->aac_fibmap);
1550                 kfree(fm, M_AACBUF);
1551         }
1552 }
1553
1554 /*
1555  * Command-mapping helper function - populate this command's s/g table.
1556  */
1557 static void
1558 aac_map_command_sg(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1559 {
1560         struct aac_softc *sc;
1561         struct aac_command *cm;
1562         struct aac_fib *fib;
1563         int i;
1564
1565         debug_called(3);
1566
1567         cm = (struct aac_command *)arg;
1568         sc = cm->cm_sc;
1569         fib = cm->cm_fib;
1570
1571         /* copy into the FIB */
1572         if (cm->cm_sgtable != NULL) {
1573                 if (fib->Header.Command == RawIo) {
1574                         struct aac_sg_tableraw *sg;
1575                         sg = (struct aac_sg_tableraw *)cm->cm_sgtable;
1576                         sg->SgCount = nseg;
1577                         for (i = 0; i < nseg; i++) {
1578                                 sg->SgEntryRaw[i].SgAddress = segs[i].ds_addr;
1579                                 sg->SgEntryRaw[i].SgByteCount = segs[i].ds_len;
1580                                 sg->SgEntryRaw[i].Next = 0;
1581                                 sg->SgEntryRaw[i].Prev = 0;
1582                                 sg->SgEntryRaw[i].Flags = 0;
1583                         }
1584                         /* update the FIB size for the s/g count */
1585                         fib->Header.Size += nseg*sizeof(struct aac_sg_entryraw);
1586                 } else if ((cm->cm_sc->flags & AAC_FLAGS_SG_64BIT) == 0) {
1587                         struct aac_sg_table *sg;
1588                         sg = cm->cm_sgtable;
1589                         sg->SgCount = nseg;
1590                         for (i = 0; i < nseg; i++) {
1591                                 sg->SgEntry[i].SgAddress = segs[i].ds_addr;
1592                                 sg->SgEntry[i].SgByteCount = segs[i].ds_len;
1593                         }
1594                         /* update the FIB size for the s/g count */
1595                         fib->Header.Size += nseg*sizeof(struct aac_sg_entry);
1596                 } else {
1597                         struct aac_sg_table64 *sg;
1598                         sg = (struct aac_sg_table64 *)cm->cm_sgtable;
1599                         sg->SgCount = nseg;
1600                         for (i = 0; i < nseg; i++) {
1601                                 sg->SgEntry64[i].SgAddress = segs[i].ds_addr;
1602                                 sg->SgEntry64[i].SgByteCount = segs[i].ds_len;
1603                         }
1604                         /* update the FIB size for the s/g count */
1605                         fib->Header.Size += nseg*sizeof(struct aac_sg_entry64);
1606                 }
1607         }
1608
1609         /* Fix up the address values in the FIB.  Use the command array index
1610          * instead of a pointer since these fields are only 32 bits.  Shift
1611          * the SenderFibAddress over to make room for the fast response bit
1612          * and for the AIF bit
1613          */
1614         cm->cm_fib->Header.SenderFibAddress = (cm->cm_index << 2);
1615         cm->cm_fib->Header.ReceiverFibAddress = (u_int32_t)cm->cm_fibphys;
1616
1617         /* save a pointer to the command for speedy reverse-lookup */
1618         cm->cm_fib->Header.SenderData = cm->cm_index;
1619
1620         if (cm->cm_flags & AAC_CMD_DATAIN)
1621                 bus_dmamap_sync(sc->aac_buffer_dmat, cm->cm_datamap,
1622                                 BUS_DMASYNC_PREREAD);
1623         if (cm->cm_flags & AAC_CMD_DATAOUT)
1624                 bus_dmamap_sync(sc->aac_buffer_dmat, cm->cm_datamap,
1625                                 BUS_DMASYNC_PREWRITE);
1626         cm->cm_flags |= AAC_CMD_MAPPED;
1627
1628         if (sc->flags & AAC_FLAGS_NEW_COMM) {
1629                 int count = 10000000L;
1630                 while (AAC_SEND_COMMAND(sc, cm) != 0) {
1631                         if (--count == 0) {
1632                                 aac_unmap_command(cm);
1633                                 sc->flags |= AAC_QUEUE_FRZN;
1634                                 aac_requeue_ready(cm);
1635                         }
1636                         DELAY(5);                       /* wait 5 usec. */
1637                 }
1638         } else {
1639                 /* Put the FIB on the outbound queue */
1640                 if (aac_enqueue_fib(sc, cm->cm_queue, cm) == EBUSY) {
1641                         aac_unmap_command(cm);
1642                         sc->flags |= AAC_QUEUE_FRZN;
1643                         aac_requeue_ready(cm);
1644                 }
1645         }
1646 }
1647
1648 /*
1649  * Unmap a command from controller-visible space.
1650  */
1651 static void
1652 aac_unmap_command(struct aac_command *cm)
1653 {
1654         struct aac_softc *sc;
1655
1656         debug_called(2);
1657
1658         sc = cm->cm_sc;
1659
1660         if (!(cm->cm_flags & AAC_CMD_MAPPED))
1661                 return;
1662
1663         if (cm->cm_datalen != 0) {
1664                 if (cm->cm_flags & AAC_CMD_DATAIN)
1665                         bus_dmamap_sync(sc->aac_buffer_dmat, cm->cm_datamap,
1666                                         BUS_DMASYNC_POSTREAD);
1667                 if (cm->cm_flags & AAC_CMD_DATAOUT)
1668                         bus_dmamap_sync(sc->aac_buffer_dmat, cm->cm_datamap,
1669                                         BUS_DMASYNC_POSTWRITE);
1670
1671                 bus_dmamap_unload(sc->aac_buffer_dmat, cm->cm_datamap);
1672         }
1673         cm->cm_flags &= ~AAC_CMD_MAPPED;
1674 }
1675
1676 /*
1677  * Hardware Interface
1678  */
1679
1680 /*
1681  * Initialise the adapter.
1682  */
1683 static void
1684 aac_common_map(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1685 {
1686         struct aac_softc *sc;
1687
1688         debug_called(1);
1689
1690         sc = (struct aac_softc *)arg;
1691
1692         sc->aac_common_busaddr = segs[0].ds_addr;
1693 }
1694
1695 static int
1696 aac_check_firmware(struct aac_softc *sc)
1697 {
1698         u_int32_t major, minor, options = 0, atu_size = 0;
1699         int status;
1700
1701         debug_called(1);
1702
1703         /*
1704          * Retrieve the firmware version numbers.  Dell PERC2/QC cards with
1705          * firmware version 1.x are not compatible with this driver.
1706          */
1707         if (sc->flags & AAC_FLAGS_PERC2QC) {
1708                 if (aac_sync_command(sc, AAC_MONKER_GETKERNVER, 0, 0, 0, 0,
1709                                      NULL)) {
1710                         device_printf(sc->aac_dev,
1711                                       "Error reading firmware version\n");
1712                         return (EIO);
1713                 }
1714
1715                 /* These numbers are stored as ASCII! */
1716                 major = (AAC_GET_MAILBOX(sc, 1) & 0xff) - 0x30;
1717                 minor = (AAC_GET_MAILBOX(sc, 2) & 0xff) - 0x30;
1718                 if (major == 1) {
1719                         device_printf(sc->aac_dev,
1720                             "Firmware version %d.%d is not supported.\n",
1721                             major, minor);
1722                         return (EINVAL);
1723                 }
1724         }
1725
1726         /*
1727          * Retrieve the capabilities/supported options word so we know what
1728          * work-arounds to enable.  Some firmware revs don't support this
1729          * command.
1730          */
1731         if (aac_sync_command(sc, AAC_MONKER_GETINFO, 0, 0, 0, 0, &status)) {
1732                 if (status != AAC_SRB_STS_INVALID_REQUEST) {
1733                         device_printf(sc->aac_dev,
1734                              "RequestAdapterInfo failed\n");
1735                         return (EIO);
1736                 }
1737         } else {
1738                 options = AAC_GET_MAILBOX(sc, 1);
1739                 atu_size = AAC_GET_MAILBOX(sc, 2);
1740                 sc->supported_options = options;
1741
1742                 if ((options & AAC_SUPPORTED_4GB_WINDOW) != 0 &&
1743                     (sc->flags & AAC_FLAGS_NO4GB) == 0)
1744                         sc->flags |= AAC_FLAGS_4GB_WINDOW;
1745                 if (options & AAC_SUPPORTED_NONDASD)
1746                         sc->flags |= AAC_FLAGS_ENABLE_CAM;
1747                 if ((options & AAC_SUPPORTED_SGMAP_HOST64) != 0
1748                      && (sizeof(bus_addr_t) > 4)) {
1749                         device_printf(sc->aac_dev,
1750                             "Enabling 64-bit address support\n");
1751                         sc->flags |= AAC_FLAGS_SG_64BIT;
1752                 }
1753                 if ((options & AAC_SUPPORTED_NEW_COMM)
1754                  && sc->aac_if.aif_send_command)
1755                         sc->flags |= AAC_FLAGS_NEW_COMM;
1756                 if (options & AAC_SUPPORTED_64BIT_ARRAYSIZE)
1757                         sc->flags |= AAC_FLAGS_ARRAY_64BIT;
1758         }
1759
1760         /* Check for broken hardware that does a lower number of commands */
1761         sc->aac_max_fibs = (sc->flags & AAC_FLAGS_256FIBS ? 256:512);
1762
1763         /* Remap mem. resource, if required */
1764         if ((sc->flags & AAC_FLAGS_NEW_COMM) &&
1765                 atu_size > rman_get_size(sc->aac_regs_resource)) {
1766                 bus_release_resource(
1767                         sc->aac_dev, SYS_RES_MEMORY,
1768                         sc->aac_regs_rid, sc->aac_regs_resource);
1769                 sc->aac_regs_resource = bus_alloc_resource(
1770                         sc->aac_dev, SYS_RES_MEMORY, &sc->aac_regs_rid,
1771                         0ul, ~0ul, atu_size, RF_ACTIVE);
1772                 if (sc->aac_regs_resource == NULL) {
1773                         sc->aac_regs_resource = bus_alloc_resource_any(
1774                                 sc->aac_dev, SYS_RES_MEMORY,
1775                                 &sc->aac_regs_rid, RF_ACTIVE);
1776                         if (sc->aac_regs_resource == NULL) {
1777                                 device_printf(sc->aac_dev,
1778                                     "couldn't allocate register window\n");
1779                                 return (ENXIO);
1780                         }
1781                         sc->flags &= ~AAC_FLAGS_NEW_COMM;
1782                 }
1783                 sc->aac_btag = rman_get_bustag(sc->aac_regs_resource);
1784                 sc->aac_bhandle = rman_get_bushandle(sc->aac_regs_resource);
1785         }
1786
1787         /* Read preferred settings */
1788         sc->aac_max_fib_size = sizeof(struct aac_fib);
1789         sc->aac_max_sectors = 128;                              /* 64KB */
1790         if (sc->flags & AAC_FLAGS_SG_64BIT)
1791                 sc->aac_sg_tablesize = (AAC_FIB_DATASIZE
1792                  - sizeof(struct aac_blockwrite64)
1793                  + sizeof(struct aac_sg_table64))
1794                  / sizeof(struct aac_sg_table64);
1795         else
1796                 sc->aac_sg_tablesize = (AAC_FIB_DATASIZE
1797                  - sizeof(struct aac_blockwrite)
1798                  + sizeof(struct aac_sg_table))
1799                  / sizeof(struct aac_sg_table);
1800
1801         if (!aac_sync_command(sc, AAC_MONKER_GETCOMMPREF, 0, 0, 0, 0, NULL)) {
1802                 options = AAC_GET_MAILBOX(sc, 1);
1803                 sc->aac_max_fib_size = (options & 0xFFFF);
1804                 sc->aac_max_sectors = (options >> 16) << 1;
1805                 options = AAC_GET_MAILBOX(sc, 2);
1806                 sc->aac_sg_tablesize = (options >> 16);
1807                 options = AAC_GET_MAILBOX(sc, 3);
1808                 sc->aac_max_fibs = (options & 0xFFFF);
1809         }
1810         if (sc->aac_max_fib_size > PAGE_SIZE)
1811                 sc->aac_max_fib_size = PAGE_SIZE;
1812         sc->aac_max_fibs_alloc = PAGE_SIZE / sc->aac_max_fib_size;
1813
1814         return (0);
1815 }
1816
1817 static int
1818 aac_init(struct aac_softc *sc)
1819 {
1820         struct aac_adapter_init *ip;
1821         time_t then;
1822         u_int32_t code, qoffset;
1823         int error;
1824
1825         debug_called(1);
1826
1827         /*
1828          * First wait for the adapter to come ready.
1829          */
1830         then = time_second;
1831         do {
1832                 code = AAC_GET_FWSTATUS(sc);
1833                 if (code & AAC_SELF_TEST_FAILED) {
1834                         device_printf(sc->aac_dev, "FATAL: selftest failed\n");
1835                         return(ENXIO);
1836                 }
1837                 if (code & AAC_KERNEL_PANIC) {
1838                         device_printf(sc->aac_dev,
1839                                       "FATAL: controller kernel panic\n");
1840                         return(ENXIO);
1841                 }
1842                 if (time_second > (then + AAC_BOOT_TIMEOUT)) {
1843                         device_printf(sc->aac_dev,
1844                                       "FATAL: controller not coming ready, "
1845                                            "status %x\n", code);
1846                         return(ENXIO);
1847                 }
1848         } while (!(code & AAC_UP_AND_RUNNING));
1849
1850         error = ENOMEM;
1851         /*
1852          * Create DMA tag for mapping buffers into controller-addressable space.
1853          */
1854         if (bus_dma_tag_create(sc->aac_parent_dmat,     /* parent */
1855                                1, 0,                    /* algnmnt, boundary */
1856                                (sc->flags & AAC_FLAGS_SG_64BIT) ?
1857                                BUS_SPACE_MAXADDR :
1858                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1859                                BUS_SPACE_MAXADDR,       /* highaddr */
1860                                NULL, NULL,              /* filter, filterarg */
1861                                MAXBSIZE,                /* maxsize */
1862                                sc->aac_sg_tablesize,    /* nsegments */
1863                                MAXBSIZE,                /* maxsegsize */
1864                                BUS_DMA_ALLOCNOW,        /* flags */
1865                                &sc->aac_buffer_dmat)) {
1866                 device_printf(sc->aac_dev, "can't allocate buffer DMA tag\n");
1867                 goto out;
1868         }
1869
1870         /*
1871          * Create DMA tag for mapping FIBs into controller-addressable space..
1872          */
1873         if (bus_dma_tag_create(sc->aac_parent_dmat,     /* parent */
1874                                1, 0,                    /* algnmnt, boundary */
1875                                (sc->flags & AAC_FLAGS_4GB_WINDOW) ?
1876                                BUS_SPACE_MAXADDR_32BIT :
1877                                0x7fffffff,              /* lowaddr */
1878                                BUS_SPACE_MAXADDR,       /* highaddr */
1879                                NULL, NULL,              /* filter, filterarg */
1880                                sc->aac_max_fibs_alloc *
1881                                sc->aac_max_fib_size,    /* maxsize */
1882                                1,                       /* nsegments */
1883                                sc->aac_max_fibs_alloc *
1884                                sc->aac_max_fib_size,    /* maxsegsize */
1885                                0,                       /* flags */
1886                                &sc->aac_fib_dmat)) {
1887                 device_printf(sc->aac_dev, "can't allocate FIB DMA tag\n");
1888                 goto out;
1889         }
1890
1891         /*
1892          * Create DMA tag for the common structure and allocate it.
1893          */
1894         if (bus_dma_tag_create(sc->aac_parent_dmat,     /* parent */
1895                                1, 0,                    /* algnmnt, boundary */
1896                                (sc->flags & AAC_FLAGS_4GB_WINDOW) ?
1897                                BUS_SPACE_MAXADDR_32BIT :
1898                                0x7fffffff,              /* lowaddr */
1899                                BUS_SPACE_MAXADDR,       /* highaddr */
1900                                NULL, NULL,              /* filter, filterarg */
1901                                8192 + sizeof(struct aac_common), /* maxsize */
1902                                1,                       /* nsegments */
1903                                BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */
1904                                0,                       /* flags */
1905                                &sc->aac_common_dmat)) {
1906                 device_printf(sc->aac_dev,
1907                               "can't allocate common structure DMA tag\n");
1908                 goto out;
1909         }
1910         if (bus_dmamem_alloc(sc->aac_common_dmat, (void **)&sc->aac_common,
1911                              BUS_DMA_NOWAIT, &sc->aac_common_dmamap)) {
1912                 device_printf(sc->aac_dev, "can't allocate common structure\n");
1913                 goto out;
1914         }
1915         /*
1916          * Work around a bug in the 2120 and 2200 that cannot DMA commands
1917          * below address 8192 in physical memory.
1918          * XXX If the padding is not needed, can it be put to use instead
1919          * of ignored?
1920          */
1921         bus_dmamap_load(sc->aac_common_dmat, sc->aac_common_dmamap,
1922                         sc->aac_common, 8192 + sizeof(*sc->aac_common),
1923                         aac_common_map, sc, 0);
1924
1925         if (sc->aac_common_busaddr < 8192) {
1926                 sc->aac_common =
1927                     (struct aac_common *)((uint8_t *)sc->aac_common + 8192);
1928                 sc->aac_common_busaddr += 8192;
1929         }
1930         bzero(sc->aac_common, sizeof(*sc->aac_common));
1931
1932         /* Allocate some FIBs and associated command structs */
1933         TAILQ_INIT(&sc->aac_fibmap_tqh);
1934         sc->aac_commands = kmalloc(sc->aac_max_fibs * sizeof(struct aac_command),
1935                                   M_AACBUF, M_INTWAIT | M_ZERO);
1936         while (sc->total_fibs < AAC_PREALLOCATE_FIBS) {
1937                 if (aac_alloc_commands(sc) != 0)
1938                         break;
1939         }
1940         if (sc->total_fibs == 0)
1941                 goto out;
1942
1943         /*
1944          * Fill in the init structure.  This tells the adapter about the
1945          * physical location of various important shared data structures.
1946          */
1947         ip = &sc->aac_common->ac_init;
1948         ip->InitStructRevision = AAC_INIT_STRUCT_REVISION;
1949         if (sc->aac_max_fib_size > sizeof(struct aac_fib)) {
1950                 ip->InitStructRevision = AAC_INIT_STRUCT_REVISION_4;
1951                 sc->flags |= AAC_FLAGS_RAW_IO;
1952         }
1953         ip->MiniPortRevision = AAC_INIT_STRUCT_MINIPORT_REVISION;
1954
1955         ip->AdapterFibsPhysicalAddress = sc->aac_common_busaddr +
1956                                          offsetof(struct aac_common, ac_fibs);
1957         ip->AdapterFibsVirtualAddress = 0;
1958         ip->AdapterFibsSize = AAC_ADAPTER_FIBS * sizeof(struct aac_fib);
1959         ip->AdapterFibAlign = sizeof(struct aac_fib);
1960
1961         ip->PrintfBufferAddress = sc->aac_common_busaddr +
1962                                   offsetof(struct aac_common, ac_printf);
1963         ip->PrintfBufferSize = AAC_PRINTF_BUFSIZE;
1964
1965         /*
1966          * The adapter assumes that pages are 4K in size, except on some
1967          * broken firmware versions that do the page->byte conversion twice,
1968          * therefore 'assuming' that this value is in 16MB units (2^24).
1969          * Round up since the granularity is so high.
1970          */
1971         /* XXX why should the adapter care? */
1972         ip->HostPhysMemPages = ctob((int)Maxmem) / AAC_PAGE_SIZE;
1973         if (sc->flags & AAC_FLAGS_BROKEN_MEMMAP) {
1974                 ip->HostPhysMemPages =
1975                     (ip->HostPhysMemPages + AAC_PAGE_SIZE) / AAC_PAGE_SIZE;
1976         }
1977         ip->HostElapsedSeconds = time_second;   /* reset later if invalid */
1978
1979         ip->InitFlags = 0;
1980         if (sc->flags & AAC_FLAGS_NEW_COMM) {
1981                 ip->InitFlags = INITFLAGS_NEW_COMM_SUPPORTED;
1982                 device_printf(sc->aac_dev, "New comm. interface enabled\n");
1983         }
1984
1985         ip->MaxIoCommands = sc->aac_max_fibs;
1986         ip->MaxIoSize = sc->aac_max_sectors << 9;
1987         ip->MaxFibSize = sc->aac_max_fib_size;
1988
1989         /*
1990          * Initialise FIB queues.  Note that it appears that the layout of the
1991          * indexes and the segmentation of the entries may be mandated by the
1992          * adapter, which is only told about the base of the queue index fields.
1993          *
1994          * The initial values of the indices are assumed to inform the adapter
1995          * of the sizes of the respective queues, and theoretically it could
1996          * work out the entire layout of the queue structures from this.  We
1997          * take the easy route and just lay this area out like everyone else
1998          * does.
1999          *
2000          * The Linux driver uses a much more complex scheme whereby several
2001          * header records are kept for each queue.  We use a couple of generic
2002          * list manipulation functions which 'know' the size of each list by
2003          * virtue of a table.
2004          */
2005         qoffset = offsetof(struct aac_common, ac_qbuf) + AAC_QUEUE_ALIGN;
2006         qoffset &= ~(AAC_QUEUE_ALIGN - 1);
2007         sc->aac_queues =
2008             (struct aac_queue_table *)((uintptr_t)sc->aac_common + qoffset);
2009         ip->CommHeaderAddress = sc->aac_common_busaddr + qoffset;
2010
2011         sc->aac_queues->qt_qindex[AAC_HOST_NORM_CMD_QUEUE][AAC_PRODUCER_INDEX] =
2012                 AAC_HOST_NORM_CMD_ENTRIES;
2013         sc->aac_queues->qt_qindex[AAC_HOST_NORM_CMD_QUEUE][AAC_CONSUMER_INDEX] =
2014                 AAC_HOST_NORM_CMD_ENTRIES;
2015         sc->aac_queues->qt_qindex[AAC_HOST_HIGH_CMD_QUEUE][AAC_PRODUCER_INDEX] =
2016                 AAC_HOST_HIGH_CMD_ENTRIES;
2017         sc->aac_queues->qt_qindex[AAC_HOST_HIGH_CMD_QUEUE][AAC_CONSUMER_INDEX] =
2018                 AAC_HOST_HIGH_CMD_ENTRIES;
2019         sc->aac_queues->qt_qindex[AAC_ADAP_NORM_CMD_QUEUE][AAC_PRODUCER_INDEX] =
2020                 AAC_ADAP_NORM_CMD_ENTRIES;
2021         sc->aac_queues->qt_qindex[AAC_ADAP_NORM_CMD_QUEUE][AAC_CONSUMER_INDEX] =
2022                 AAC_ADAP_NORM_CMD_ENTRIES;
2023         sc->aac_queues->qt_qindex[AAC_ADAP_HIGH_CMD_QUEUE][AAC_PRODUCER_INDEX] =
2024                 AAC_ADAP_HIGH_CMD_ENTRIES;
2025         sc->aac_queues->qt_qindex[AAC_ADAP_HIGH_CMD_QUEUE][AAC_CONSUMER_INDEX] =
2026                 AAC_ADAP_HIGH_CMD_ENTRIES;
2027         sc->aac_queues->qt_qindex[AAC_HOST_NORM_RESP_QUEUE][AAC_PRODUCER_INDEX]=
2028                 AAC_HOST_NORM_RESP_ENTRIES;
2029         sc->aac_queues->qt_qindex[AAC_HOST_NORM_RESP_QUEUE][AAC_CONSUMER_INDEX]=
2030                 AAC_HOST_NORM_RESP_ENTRIES;
2031         sc->aac_queues->qt_qindex[AAC_HOST_HIGH_RESP_QUEUE][AAC_PRODUCER_INDEX]=
2032                 AAC_HOST_HIGH_RESP_ENTRIES;
2033         sc->aac_queues->qt_qindex[AAC_HOST_HIGH_RESP_QUEUE][AAC_CONSUMER_INDEX]=
2034                 AAC_HOST_HIGH_RESP_ENTRIES;
2035         sc->aac_queues->qt_qindex[AAC_ADAP_NORM_RESP_QUEUE][AAC_PRODUCER_INDEX]=
2036                 AAC_ADAP_NORM_RESP_ENTRIES;
2037         sc->aac_queues->qt_qindex[AAC_ADAP_NORM_RESP_QUEUE][AAC_CONSUMER_INDEX]=
2038                 AAC_ADAP_NORM_RESP_ENTRIES;
2039         sc->aac_queues->qt_qindex[AAC_ADAP_HIGH_RESP_QUEUE][AAC_PRODUCER_INDEX]=
2040                 AAC_ADAP_HIGH_RESP_ENTRIES;
2041         sc->aac_queues->qt_qindex[AAC_ADAP_HIGH_RESP_QUEUE][AAC_CONSUMER_INDEX]=
2042                 AAC_ADAP_HIGH_RESP_ENTRIES;
2043         sc->aac_qentries[AAC_HOST_NORM_CMD_QUEUE] =
2044                 &sc->aac_queues->qt_HostNormCmdQueue[0];
2045         sc->aac_qentries[AAC_HOST_HIGH_CMD_QUEUE] =
2046                 &sc->aac_queues->qt_HostHighCmdQueue[0];
2047         sc->aac_qentries[AAC_ADAP_NORM_CMD_QUEUE] =
2048                 &sc->aac_queues->qt_AdapNormCmdQueue[0];
2049         sc->aac_qentries[AAC_ADAP_HIGH_CMD_QUEUE] =
2050                 &sc->aac_queues->qt_AdapHighCmdQueue[0];
2051         sc->aac_qentries[AAC_HOST_NORM_RESP_QUEUE] =
2052                 &sc->aac_queues->qt_HostNormRespQueue[0];
2053         sc->aac_qentries[AAC_HOST_HIGH_RESP_QUEUE] =
2054                 &sc->aac_queues->qt_HostHighRespQueue[0];
2055         sc->aac_qentries[AAC_ADAP_NORM_RESP_QUEUE] =
2056                 &sc->aac_queues->qt_AdapNormRespQueue[0];
2057         sc->aac_qentries[AAC_ADAP_HIGH_RESP_QUEUE] =
2058                 &sc->aac_queues->qt_AdapHighRespQueue[0];
2059
2060         /*
2061          * Do controller-type-specific initialisation
2062          */
2063         switch (sc->aac_hwif) {
2064         case AAC_HWIF_I960RX:
2065                 AAC_SETREG4(sc, AAC_RX_ODBR, ~0);
2066                 break;
2067         case AAC_HWIF_RKT:
2068                 AAC_SETREG4(sc, AAC_RKT_ODBR, ~0);
2069                 break;
2070         default:
2071                 break;
2072         }
2073
2074         /*
2075          * Give the init structure to the controller.
2076          */
2077         if (aac_sync_command(sc, AAC_MONKER_INITSTRUCT,
2078                              sc->aac_common_busaddr +
2079                              offsetof(struct aac_common, ac_init), 0, 0, 0,
2080                              NULL)) {
2081                 device_printf(sc->aac_dev,
2082                               "error establishing init structure\n");
2083                 error = EIO;
2084                 goto out;
2085         }
2086
2087         error = 0;
2088 out:
2089         return(error);
2090 }
2091
2092 /*
2093  * Send a synchronous command to the controller and wait for a result.
2094  * Indicate if the controller completed the command with an error status.
2095  */
2096 static int
2097 aac_sync_command(struct aac_softc *sc, u_int32_t command,
2098                  u_int32_t arg0, u_int32_t arg1, u_int32_t arg2, u_int32_t arg3,
2099                  u_int32_t *sp)
2100 {
2101         time_t then;
2102         u_int32_t status;
2103
2104         debug_called(3);
2105
2106         /* populate the mailbox */
2107         AAC_SET_MAILBOX(sc, command, arg0, arg1, arg2, arg3);
2108
2109         /* ensure the sync command doorbell flag is cleared */
2110         AAC_CLEAR_ISTATUS(sc, AAC_DB_SYNC_COMMAND);
2111
2112         /* then set it to signal the adapter */
2113         AAC_QNOTIFY(sc, AAC_DB_SYNC_COMMAND);
2114
2115         /* spin waiting for the command to complete */
2116         then = time_second;
2117         do {
2118                 if (time_second > (then + AAC_IMMEDIATE_TIMEOUT)) {
2119                         debug(1, "timed out");
2120                         return(EIO);
2121                 }
2122         } while (!(AAC_GET_ISTATUS(sc) & AAC_DB_SYNC_COMMAND));
2123
2124         /* clear the completion flag */
2125         AAC_CLEAR_ISTATUS(sc, AAC_DB_SYNC_COMMAND);
2126
2127         /* get the command status */
2128         status = AAC_GET_MAILBOX(sc, 0);
2129         if (sp != NULL)
2130                 *sp = status;
2131
2132         if (status != AAC_SRB_STS_SUCCESS)
2133                 return (-1);
2134         return(0);
2135 }
2136
2137 int
2138 aac_sync_fib(struct aac_softc *sc, u_int32_t command, u_int32_t xferstate,
2139                  struct aac_fib *fib, u_int16_t datasize)
2140 {
2141         debug_called(3);
2142         KKASSERT(lockstatus(&sc->aac_io_lock, curthread) != 0);
2143
2144         if (datasize > AAC_FIB_DATASIZE)
2145                 return(EINVAL);
2146
2147         /*
2148          * Set up the sync FIB
2149          */
2150         fib->Header.XferState = AAC_FIBSTATE_HOSTOWNED |
2151                                 AAC_FIBSTATE_INITIALISED |
2152                                 AAC_FIBSTATE_EMPTY;
2153         fib->Header.XferState |= xferstate;
2154         fib->Header.Command = command;
2155         fib->Header.StructType = AAC_FIBTYPE_TFIB;
2156         fib->Header.Size = sizeof(struct aac_fib) + datasize;
2157         fib->Header.SenderSize = sizeof(struct aac_fib);
2158         fib->Header.SenderFibAddress = 0;       /* Not needed */
2159         fib->Header.ReceiverFibAddress = sc->aac_common_busaddr +
2160                                          offsetof(struct aac_common,
2161                                                   ac_sync_fib);
2162
2163         /*
2164          * Give the FIB to the controller, wait for a response.
2165          */
2166         if (aac_sync_command(sc, AAC_MONKER_SYNCFIB,
2167                              fib->Header.ReceiverFibAddress, 0, 0, 0, NULL)) {
2168                 debug(2, "IO error");
2169                 return(EIO);
2170         }
2171
2172         return (0);
2173 }
2174
2175 /*
2176  * Adapter-space FIB queue manipulation
2177  *
2178  * Note that the queue implementation here is a little funky; neither the PI or
2179  * CI will ever be zero.  This behaviour is a controller feature.
2180  */
2181 static struct {
2182         int             size;
2183         int             notify;
2184 } aac_qinfo[] = {
2185         {AAC_HOST_NORM_CMD_ENTRIES, AAC_DB_COMMAND_NOT_FULL},
2186         {AAC_HOST_HIGH_CMD_ENTRIES, 0},
2187         {AAC_ADAP_NORM_CMD_ENTRIES, AAC_DB_COMMAND_READY},
2188         {AAC_ADAP_HIGH_CMD_ENTRIES, 0},
2189         {AAC_HOST_NORM_RESP_ENTRIES, AAC_DB_RESPONSE_NOT_FULL},
2190         {AAC_HOST_HIGH_RESP_ENTRIES, 0},
2191         {AAC_ADAP_NORM_RESP_ENTRIES, AAC_DB_RESPONSE_READY},
2192         {AAC_ADAP_HIGH_RESP_ENTRIES, 0}
2193 };
2194
2195 /*
2196  * Atomically insert an entry into the nominated queue, returns 0 on success or
2197  * EBUSY if the queue is full.
2198  *
2199  * Note: it would be more efficient to defer notifying the controller in
2200  *       the case where we may be inserting several entries in rapid succession,
2201  *       but implementing this usefully may be difficult (it would involve a
2202  *       separate queue/notify interface).
2203  */
2204 static int
2205 aac_enqueue_fib(struct aac_softc *sc, int queue, struct aac_command *cm)
2206 {
2207         u_int32_t pi, ci;
2208         int error;
2209         u_int32_t fib_size;
2210         u_int32_t fib_addr;
2211
2212         debug_called(3);
2213
2214         fib_size = cm->cm_fib->Header.Size;
2215         fib_addr = cm->cm_fib->Header.ReceiverFibAddress;
2216
2217         /* get the producer/consumer indices */
2218         pi = sc->aac_queues->qt_qindex[queue][AAC_PRODUCER_INDEX];
2219         ci = sc->aac_queues->qt_qindex[queue][AAC_CONSUMER_INDEX];
2220
2221         /* wrap the queue? */
2222         if (pi >= aac_qinfo[queue].size)
2223                 pi = 0;
2224
2225         /* check for queue full */
2226         if ((pi + 1) == ci) {
2227                 error = EBUSY;
2228                 goto out;
2229         }
2230         /*
2231          * To avoid a race with its completion interrupt, place this command on
2232          * the busy queue prior to advertising it to the controller.
2233          */
2234         aac_enqueue_busy(cm);
2235
2236
2237
2238         /* populate queue entry */
2239         (sc->aac_qentries[queue] + pi)->aq_fib_size = fib_size;
2240         (sc->aac_qentries[queue] + pi)->aq_fib_addr = fib_addr;
2241
2242         /* update producer index */
2243         sc->aac_queues->qt_qindex[queue][AAC_PRODUCER_INDEX] = pi + 1;
2244
2245         /* notify the adapter if we know how */
2246         if (aac_qinfo[queue].notify != 0)
2247                 AAC_QNOTIFY(sc, aac_qinfo[queue].notify);
2248
2249         error = 0;
2250
2251 out:
2252         return(error);
2253 }
2254
2255 /*
2256  * Atomically remove one entry from the nominated queue, returns 0 on
2257  * success or ENOENT if the queue is empty.
2258  */
2259 static int
2260 aac_dequeue_fib(struct aac_softc *sc, int queue, u_int32_t *fib_size,
2261                 struct aac_fib **fib_addr)
2262 {
2263         u_int32_t pi, ci;
2264         u_int32_t fib_index;
2265         int error;
2266         int notify;
2267
2268         debug_called(3);
2269
2270         /* get the producer/consumer indices */
2271         pi = sc->aac_queues->qt_qindex[queue][AAC_PRODUCER_INDEX];
2272         ci = sc->aac_queues->qt_qindex[queue][AAC_CONSUMER_INDEX];
2273
2274         /* check for queue empty */
2275         if (ci == pi) {
2276                 error = ENOENT;
2277                 goto out;
2278         }
2279
2280         /* wrap the pi so the following test works */
2281         if (pi >= aac_qinfo[queue].size)
2282                 pi = 0;
2283
2284         notify = 0;
2285         if (ci == pi + 1)
2286                 notify++;
2287
2288         /* wrap the queue? */
2289         if (ci >= aac_qinfo[queue].size)
2290                 ci = 0;
2291
2292         /* fetch the entry */
2293         *fib_size = (sc->aac_qentries[queue] + ci)->aq_fib_size;
2294
2295         switch (queue) {
2296         case AAC_HOST_NORM_CMD_QUEUE:
2297         case AAC_HOST_HIGH_CMD_QUEUE:
2298                 /*
2299                  * The aq_fib_addr is only 32 bits wide so it can't be counted
2300                  * on to hold an address.  For AIF's, the adapter assumes
2301                  * that it's giving us an address into the array of AIF fibs.
2302                  * Therefore, we have to convert it to an index.
2303                  */
2304                 fib_index = (sc->aac_qentries[queue] + ci)->aq_fib_addr /
2305                         sizeof(struct aac_fib);
2306                 *fib_addr = &sc->aac_common->ac_fibs[fib_index];
2307                 break;
2308
2309         case AAC_HOST_NORM_RESP_QUEUE:
2310         case AAC_HOST_HIGH_RESP_QUEUE:
2311         {
2312                 struct aac_command *cm;
2313
2314                 /*
2315                  * As above, an index is used instead of an actual address.
2316                  * Gotta shift the index to account for the fast response
2317                  * bit.  No other correction is needed since this value was
2318                  * originally provided by the driver via the SenderFibAddress
2319                  * field.
2320                  */
2321                 fib_index = (sc->aac_qentries[queue] + ci)->aq_fib_addr;
2322                 cm = sc->aac_commands + (fib_index >> 2);
2323                 *fib_addr = cm->cm_fib;
2324
2325                 /*
2326                  * Is this a fast response? If it is, update the fib fields in
2327                  * local memory since the whole fib isn't DMA'd back up.
2328                  */
2329                 if (fib_index & 0x01) {
2330                         (*fib_addr)->Header.XferState |= AAC_FIBSTATE_DONEADAP;
2331                         *((u_int32_t*)((*fib_addr)->data)) = AAC_ERROR_NORMAL;
2332                 }
2333                 break;
2334         }
2335         default:
2336                 panic("Invalid queue in aac_dequeue_fib()");
2337                 break;
2338         }
2339
2340         /* update consumer index */
2341         sc->aac_queues->qt_qindex[queue][AAC_CONSUMER_INDEX] = ci + 1;
2342
2343         /* if we have made the queue un-full, notify the adapter */
2344         if (notify && (aac_qinfo[queue].notify != 0))
2345                 AAC_QNOTIFY(sc, aac_qinfo[queue].notify);
2346         error = 0;
2347
2348 out:
2349         return(error);
2350 }
2351
2352 /*
2353  * Put our response to an Adapter Initialed Fib on the response queue
2354  */
2355 static int
2356 aac_enqueue_response(struct aac_softc *sc, int queue, struct aac_fib *fib)
2357 {
2358         u_int32_t pi, ci;
2359         int error;
2360         u_int32_t fib_size;
2361         u_int32_t fib_addr;
2362
2363         debug_called(1);
2364
2365         /* Tell the adapter where the FIB is */
2366         fib_size = fib->Header.Size;
2367         fib_addr = fib->Header.SenderFibAddress;
2368         fib->Header.ReceiverFibAddress = fib_addr;
2369
2370         /* get the producer/consumer indices */
2371         pi = sc->aac_queues->qt_qindex[queue][AAC_PRODUCER_INDEX];
2372         ci = sc->aac_queues->qt_qindex[queue][AAC_CONSUMER_INDEX];
2373
2374         /* wrap the queue? */
2375         if (pi >= aac_qinfo[queue].size)
2376                 pi = 0;
2377
2378         /* check for queue full */
2379         if ((pi + 1) == ci) {
2380                 error = EBUSY;
2381                 goto out;
2382         }
2383
2384         /* populate queue entry */
2385         (sc->aac_qentries[queue] + pi)->aq_fib_size = fib_size;
2386         (sc->aac_qentries[queue] + pi)->aq_fib_addr = fib_addr;
2387
2388         /* update producer index */
2389         sc->aac_queues->qt_qindex[queue][AAC_PRODUCER_INDEX] = pi + 1;
2390
2391         /* notify the adapter if we know how */
2392         if (aac_qinfo[queue].notify != 0)
2393                 AAC_QNOTIFY(sc, aac_qinfo[queue].notify);
2394
2395         error = 0;
2396
2397 out:
2398         return(error);
2399 }
2400
2401 /*
2402  * Check for commands that have been outstanding for a suspiciously long time,
2403  * and complain about them.
2404  */
2405 static void
2406 aac_timeout(void *xsc)
2407 {
2408         struct aac_softc *sc = xsc;
2409         struct aac_command *cm;
2410         time_t deadline;
2411         int timedout, code;
2412         /*
2413          * Traverse the busy command list, bitch about late commands once
2414          * only.
2415          */
2416         timedout = 0;
2417         deadline = time_second - AAC_CMD_TIMEOUT;
2418         TAILQ_FOREACH(cm, &sc->aac_busy, cm_link) {
2419                 if ((cm->cm_timestamp  < deadline)
2420                         /* && !(cm->cm_flags & AAC_CMD_TIMEDOUT) */) {
2421                         cm->cm_flags |= AAC_CMD_TIMEDOUT;
2422                         device_printf(sc->aac_dev,
2423                                       "COMMAND %p TIMEOUT AFTER %d SECONDS\n",
2424                                       cm, (int)(time_second-cm->cm_timestamp));
2425                         AAC_PRINT_FIB(sc, cm->cm_fib);
2426                         timedout++;
2427                 }
2428         }
2429         if (timedout) {
2430                 code = AAC_GET_FWSTATUS(sc);
2431                 if (code != AAC_UP_AND_RUNNING) {
2432                         device_printf(sc->aac_dev, "WARNING! Controller is no "
2433                                       "longer running! code= 0x%x\n", code);
2434
2435                 }
2436         }
2437 }
2438
2439 /*
2440  * Interface Function Vectors
2441  */
2442
2443 /*
2444  * Read the current firmware status word.
2445  */
2446 static int
2447 aac_sa_get_fwstatus(struct aac_softc *sc)
2448 {
2449         debug_called(3);
2450
2451         return(AAC_GETREG4(sc, AAC_SA_FWSTATUS));
2452 }
2453
2454 static int
2455 aac_rx_get_fwstatus(struct aac_softc *sc)
2456 {
2457         debug_called(3);
2458
2459         return(AAC_GETREG4(sc, AAC_RX_FWSTATUS));
2460 }
2461
2462 static int
2463 aac_fa_get_fwstatus(struct aac_softc *sc)
2464 {
2465         int val;
2466
2467         debug_called(3);
2468
2469         val = AAC_GETREG4(sc, AAC_FA_FWSTATUS);
2470         return (val);
2471 }
2472
2473 static int
2474 aac_rkt_get_fwstatus(struct aac_softc *sc)
2475 {
2476         debug_called(3);
2477
2478         return(AAC_GETREG4(sc, AAC_RKT_FWSTATUS));
2479 }
2480
2481 /*
2482  * Notify the controller of a change in a given queue
2483  */
2484
2485 static void
2486 aac_sa_qnotify(struct aac_softc *sc, int qbit)
2487 {
2488         debug_called(3);
2489
2490         AAC_SETREG2(sc, AAC_SA_DOORBELL1_SET, qbit);
2491 }
2492
2493 static void
2494 aac_rx_qnotify(struct aac_softc *sc, int qbit)
2495 {
2496         debug_called(3);
2497
2498         AAC_SETREG4(sc, AAC_RX_IDBR, qbit);
2499 }
2500
2501 static void
2502 aac_fa_qnotify(struct aac_softc *sc, int qbit)
2503 {
2504         debug_called(3);
2505
2506         AAC_SETREG2(sc, AAC_FA_DOORBELL1, qbit);
2507         AAC_FA_HACK(sc);
2508 }
2509
2510 static void
2511 aac_rkt_qnotify(struct aac_softc *sc, int qbit)
2512 {
2513         debug_called(3);
2514
2515         AAC_SETREG4(sc, AAC_RKT_IDBR, qbit);
2516 }
2517
2518 /*
2519  * Get the interrupt reason bits
2520  */
2521 static int
2522 aac_sa_get_istatus(struct aac_softc *sc)
2523 {
2524         debug_called(3);
2525
2526         return(AAC_GETREG2(sc, AAC_SA_DOORBELL0));
2527 }
2528
2529 static int
2530 aac_rx_get_istatus(struct aac_softc *sc)
2531 {
2532         debug_called(3);
2533
2534         return(AAC_GETREG4(sc, AAC_RX_ODBR));
2535 }
2536
2537 static int
2538 aac_fa_get_istatus(struct aac_softc *sc)
2539 {
2540         int val;
2541
2542         debug_called(3);
2543
2544         val = AAC_GETREG2(sc, AAC_FA_DOORBELL0);
2545         return (val);
2546 }
2547
2548 static int
2549 aac_rkt_get_istatus(struct aac_softc *sc)
2550 {
2551         debug_called(3);
2552
2553         return(AAC_GETREG4(sc, AAC_RKT_ODBR));
2554 }
2555
2556 /*
2557  * Clear some interrupt reason bits
2558  */
2559 static void
2560 aac_sa_clear_istatus(struct aac_softc *sc, int mask)
2561 {
2562         debug_called(3);
2563
2564         AAC_SETREG2(sc, AAC_SA_DOORBELL0_CLEAR, mask);
2565 }
2566
2567 static void
2568 aac_rx_clear_istatus(struct aac_softc *sc, int mask)
2569 {
2570         debug_called(3);
2571
2572         AAC_SETREG4(sc, AAC_RX_ODBR, mask);
2573 }
2574
2575 static void
2576 aac_fa_clear_istatus(struct aac_softc *sc, int mask)
2577 {
2578         debug_called(3);
2579
2580         AAC_SETREG2(sc, AAC_FA_DOORBELL0_CLEAR, mask);
2581         AAC_FA_HACK(sc);
2582 }
2583
2584 static void
2585 aac_rkt_clear_istatus(struct aac_softc *sc, int mask)
2586 {
2587         debug_called(3);
2588
2589         AAC_SETREG4(sc, AAC_RKT_ODBR, mask);
2590 }
2591
2592 /*
2593  * Populate the mailbox and set the command word
2594  */
2595 static void
2596 aac_sa_set_mailbox(struct aac_softc *sc, u_int32_t command,
2597                 u_int32_t arg0, u_int32_t arg1, u_int32_t arg2, u_int32_t arg3)
2598 {
2599         debug_called(4);
2600
2601         AAC_SETREG4(sc, AAC_SA_MAILBOX, command);
2602         AAC_SETREG4(sc, AAC_SA_MAILBOX + 4, arg0);
2603         AAC_SETREG4(sc, AAC_SA_MAILBOX + 8, arg1);
2604         AAC_SETREG4(sc, AAC_SA_MAILBOX + 12, arg2);
2605         AAC_SETREG4(sc, AAC_SA_MAILBOX + 16, arg3);
2606 }
2607
2608 static void
2609 aac_rx_set_mailbox(struct aac_softc *sc, u_int32_t command,
2610                 u_int32_t arg0, u_int32_t arg1, u_int32_t arg2, u_int32_t arg3)
2611 {
2612         debug_called(4);
2613
2614         AAC_SETREG4(sc, AAC_RX_MAILBOX, command);
2615         AAC_SETREG4(sc, AAC_RX_MAILBOX + 4, arg0);
2616         AAC_SETREG4(sc, AAC_RX_MAILBOX + 8, arg1);
2617         AAC_SETREG4(sc, AAC_RX_MAILBOX + 12, arg2);
2618         AAC_SETREG4(sc, AAC_RX_MAILBOX + 16, arg3);
2619 }
2620
2621 static void
2622 aac_fa_set_mailbox(struct aac_softc *sc, u_int32_t command,
2623                 u_int32_t arg0, u_int32_t arg1, u_int32_t arg2, u_int32_t arg3)
2624 {
2625         debug_called(4);
2626
2627         AAC_SETREG4(sc, AAC_FA_MAILBOX, command);
2628         AAC_FA_HACK(sc);
2629         AAC_SETREG4(sc, AAC_FA_MAILBOX + 4, arg0);
2630         AAC_FA_HACK(sc);
2631         AAC_SETREG4(sc, AAC_FA_MAILBOX + 8, arg1);
2632         AAC_FA_HACK(sc);
2633         AAC_SETREG4(sc, AAC_FA_MAILBOX + 12, arg2);
2634         AAC_FA_HACK(sc);
2635         AAC_SETREG4(sc, AAC_FA_MAILBOX + 16, arg3);
2636         AAC_FA_HACK(sc);
2637 }
2638
2639 static void
2640 aac_rkt_set_mailbox(struct aac_softc *sc, u_int32_t command, u_int32_t arg0,
2641                     u_int32_t arg1, u_int32_t arg2, u_int32_t arg3)
2642 {
2643         debug_called(4);
2644
2645         AAC_SETREG4(sc, AAC_RKT_MAILBOX, command);
2646         AAC_SETREG4(sc, AAC_RKT_MAILBOX + 4, arg0);
2647         AAC_SETREG4(sc, AAC_RKT_MAILBOX + 8, arg1);
2648         AAC_SETREG4(sc, AAC_RKT_MAILBOX + 12, arg2);
2649         AAC_SETREG4(sc, AAC_RKT_MAILBOX + 16, arg3);
2650 }
2651
2652 /*
2653  * Fetch the immediate command status word
2654  */
2655 static int
2656 aac_sa_get_mailbox(struct aac_softc *sc, int mb)
2657 {
2658         debug_called(4);
2659
2660         return(AAC_GETREG4(sc, AAC_SA_MAILBOX + (mb * 4)));
2661 }
2662
2663 static int
2664 aac_rx_get_mailbox(struct aac_softc *sc, int mb)
2665 {
2666         debug_called(4);
2667
2668         return(AAC_GETREG4(sc, AAC_RX_MAILBOX + (mb * 4)));
2669 }
2670
2671 static int
2672 aac_fa_get_mailbox(struct aac_softc *sc, int mb)
2673 {
2674         int val;
2675
2676         debug_called(4);
2677
2678         val = AAC_GETREG4(sc, AAC_FA_MAILBOX + (mb * 4));
2679         return (val);
2680 }
2681
2682 static int
2683 aac_rkt_get_mailbox(struct aac_softc *sc, int mb)
2684 {
2685         debug_called(4);
2686
2687         return(AAC_GETREG4(sc, AAC_RKT_MAILBOX + (mb * 4)));
2688 }
2689
2690 /*
2691  * Set/clear interrupt masks
2692  */
2693 static void
2694 aac_sa_set_interrupts(struct aac_softc *sc, int enable)
2695 {
2696         debug(2, "%sable interrupts", enable ? "en" : "dis");
2697
2698         if (enable) {
2699                 AAC_SETREG2((sc), AAC_SA_MASK0_CLEAR, AAC_DB_INTERRUPTS);
2700         } else {
2701                 AAC_SETREG2((sc), AAC_SA_MASK0_SET, ~0);
2702         }
2703 }
2704
2705 static void
2706 aac_rx_set_interrupts(struct aac_softc *sc, int enable)
2707 {
2708         debug(2, "%sable interrupts", enable ? "en" : "dis");
2709
2710         if (enable) {
2711                 if (sc->flags & AAC_FLAGS_NEW_COMM)
2712                         AAC_SETREG4(sc, AAC_RX_OIMR, ~AAC_DB_INT_NEW_COMM);
2713                 else
2714                         AAC_SETREG4(sc, AAC_RX_OIMR, ~AAC_DB_INTERRUPTS);
2715         } else {
2716                 AAC_SETREG4(sc, AAC_RX_OIMR, ~0);
2717         }
2718 }
2719
2720 static void
2721 aac_fa_set_interrupts(struct aac_softc *sc, int enable)
2722 {
2723         debug(2, "%sable interrupts", enable ? "en" : "dis");
2724
2725         if (enable) {
2726                 AAC_SETREG2((sc), AAC_FA_MASK0_CLEAR, AAC_DB_INTERRUPTS);
2727                 AAC_FA_HACK(sc);
2728         } else {
2729                 AAC_SETREG2((sc), AAC_FA_MASK0, ~0);
2730                 AAC_FA_HACK(sc);
2731         }
2732 }
2733
2734 static void
2735 aac_rkt_set_interrupts(struct aac_softc *sc, int enable)
2736 {
2737         debug(2, "%sable interrupts", enable ? "en" : "dis");
2738
2739         if (enable) {
2740                 if (sc->flags & AAC_FLAGS_NEW_COMM)
2741                         AAC_SETREG4(sc, AAC_RKT_OIMR, ~AAC_DB_INT_NEW_COMM);
2742                 else
2743                         AAC_SETREG4(sc, AAC_RKT_OIMR, ~AAC_DB_INTERRUPTS);
2744         } else {
2745                 AAC_SETREG4(sc, AAC_RKT_OIMR, ~0);
2746         }
2747 }
2748
2749 /*
2750  * New comm. interface: Send command functions
2751  */
2752 static int
2753 aac_rx_send_command(struct aac_softc *sc, struct aac_command *cm)
2754 {
2755         u_int32_t index, device;
2756
2757         debug(2, "send command (new comm.)");
2758
2759         index = AAC_GETREG4(sc, AAC_RX_IQUE);
2760         if (index == 0xffffffffL)
2761                 index = AAC_GETREG4(sc, AAC_RX_IQUE);
2762         if (index == 0xffffffffL)
2763                 return index;
2764         aac_enqueue_busy(cm);
2765         device = index;
2766         AAC_SETREG4(sc, device, (u_int32_t)(cm->cm_fibphys & 0xffffffffUL));
2767         device += 4;
2768         AAC_SETREG4(sc, device, (u_int32_t)(cm->cm_fibphys >> 32));
2769         device += 4;
2770         AAC_SETREG4(sc, device, cm->cm_fib->Header.Size);
2771         AAC_SETREG4(sc, AAC_RX_IQUE, index);
2772         return 0;
2773 }
2774
2775 static int
2776 aac_rkt_send_command(struct aac_softc *sc, struct aac_command *cm)
2777 {
2778         u_int32_t index, device;
2779
2780         debug(2, "send command (new comm.)");
2781
2782         index = AAC_GETREG4(sc, AAC_RKT_IQUE);
2783         if (index == 0xffffffffL)
2784                 index = AAC_GETREG4(sc, AAC_RKT_IQUE);
2785         if (index == 0xffffffffL)
2786                 return index;
2787         aac_enqueue_busy(cm);
2788         device = index;
2789         AAC_SETREG4(sc, device, (u_int32_t)(cm->cm_fibphys & 0xffffffffUL));
2790         device += 4;
2791         AAC_SETREG4(sc, device, (u_int32_t)(cm->cm_fibphys >> 32));
2792         device += 4;
2793         AAC_SETREG4(sc, device, cm->cm_fib->Header.Size);
2794         AAC_SETREG4(sc, AAC_RKT_IQUE, index);
2795         return 0;
2796 }
2797
2798 /*
2799  * New comm. interface: get, set outbound queue index
2800  */
2801 static int
2802 aac_rx_get_outb_queue(struct aac_softc *sc)
2803 {
2804         debug_called(3);
2805
2806         return(AAC_GETREG4(sc, AAC_RX_OQUE));
2807 }
2808
2809 static int
2810 aac_rkt_get_outb_queue(struct aac_softc *sc)
2811 {
2812         debug_called(3);
2813
2814         return(AAC_GETREG4(sc, AAC_RKT_OQUE));
2815 }
2816
2817 static void
2818 aac_rx_set_outb_queue(struct aac_softc *sc, int index)
2819 {
2820         debug_called(3);
2821
2822         AAC_SETREG4(sc, AAC_RX_OQUE, index);
2823 }
2824
2825 static void
2826 aac_rkt_set_outb_queue(struct aac_softc *sc, int index)
2827 {
2828         debug_called(3);
2829
2830         AAC_SETREG4(sc, AAC_RKT_OQUE, index);
2831 }
2832
2833 /*
2834  * Debugging and Diagnostics
2835  */
2836
2837 /*
2838  * Print some information about the controller.
2839  */
2840 static void
2841 aac_describe_controller(struct aac_softc *sc)
2842 {
2843         struct aac_fib *fib;
2844         struct aac_adapter_info *info;
2845
2846         debug_called(2);
2847
2848         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
2849         aac_alloc_sync_fib(sc, &fib);
2850
2851         fib->data[0] = 0;
2852         if (aac_sync_fib(sc, RequestAdapterInfo, 0, fib, 1)) {
2853                 device_printf(sc->aac_dev, "RequestAdapterInfo failed\n");
2854                 aac_release_sync_fib(sc);
2855                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
2856                 return;
2857         }
2858
2859         /* save the kernel revision structure for later use */
2860         info = (struct aac_adapter_info *)&fib->data[0];
2861         sc->aac_revision = info->KernelRevision;
2862
2863         device_printf(sc->aac_dev, "Adaptec Raid Controller %d.%d.%d-%d\n",
2864                 AAC_DRIVER_VERSION >> 24,
2865                 (AAC_DRIVER_VERSION >> 16) & 0xFF,
2866                 AAC_DRIVER_VERSION & 0xFF,
2867                 AAC_DRIVER_BUILD);
2868
2869         if (bootverbose) {
2870                 device_printf(sc->aac_dev, "%s %dMHz, %dMB memory "
2871                     "(%dMB cache, %dMB execution), %s\n",
2872                     aac_describe_code(aac_cpu_variant, info->CpuVariant),
2873                     info->ClockSpeed, info->TotalMem / (1024 * 1024),
2874                     info->BufferMem / (1024 * 1024),
2875                     info->ExecutionMem / (1024 * 1024),
2876                     aac_describe_code(aac_battery_platform,
2877                     info->batteryPlatform));
2878
2879                 device_printf(sc->aac_dev,
2880                     "Kernel %d.%d-%d, Build %d, S/N %6X\n",
2881                     info->KernelRevision.external.comp.major,
2882                     info->KernelRevision.external.comp.minor,
2883                     info->KernelRevision.external.comp.dash,
2884                     info->KernelRevision.buildNumber,
2885                     (u_int32_t)(info->SerialNumber & 0xffffff));
2886
2887                 device_printf(sc->aac_dev, "Supported Options=%b\n",
2888                               sc->supported_options,
2889                               "\20"
2890                               "\1SNAPSHOT"
2891                               "\2CLUSTERS"
2892                               "\3WCACHE"
2893                               "\4DATA64"
2894                               "\5HOSTTIME"
2895                               "\6RAID50"
2896                               "\7WINDOW4GB"
2897                               "\10SCSIUPGD"
2898                               "\11SOFTERR"
2899                               "\12NORECOND"
2900                               "\13SGMAP64"
2901                               "\14ALARM"
2902                               "\15NONDASD"
2903                               "\16SCSIMGT"
2904                               "\17RAIDSCSI"
2905                               "\21ADPTINFO"
2906                               "\22NEWCOMM"
2907                               "\23ARRAY64BIT"
2908                               "\24HEATSENSOR");
2909         }
2910         aac_release_sync_fib(sc);
2911         AAC_LOCK_RELEASE(&sc->aac_io_lock);
2912 }
2913
2914 /*
2915  * Look up a text description of a numeric error code and return a pointer to
2916  * same.
2917  */
2918 static char *
2919 aac_describe_code(struct aac_code_lookup *table, u_int32_t code)
2920 {
2921         int i;
2922
2923         for (i = 0; table[i].string != NULL; i++)
2924                 if (table[i].code == code)
2925                         return(table[i].string);
2926         return(table[i + 1].string);
2927 }
2928
2929 /*
2930  * Management Interface
2931  */
2932
2933 static int
2934 aac_open(struct dev_open_args *ap)
2935 {
2936         cdev_t dev = ap->a_head.a_dev;
2937         struct aac_softc *sc;
2938
2939         debug_called(2);
2940
2941         sc = dev->si_drv1;
2942
2943         /* Check to make sure the device isn't already open */
2944         if (sc->aac_state & AAC_STATE_OPEN) {
2945                 return EBUSY;
2946         }
2947         sc->aac_state |= AAC_STATE_OPEN;
2948
2949         return 0;
2950 }
2951
2952 static int
2953 aac_close(struct dev_close_args *ap)
2954 {
2955         cdev_t dev = ap->a_head.a_dev;
2956         struct aac_softc *sc;
2957
2958         debug_called(2);
2959
2960         sc = dev->si_drv1;
2961
2962         /* Mark this unit as no longer open  */
2963         sc->aac_state &= ~AAC_STATE_OPEN;
2964
2965         return 0;
2966 }
2967
2968 static int
2969 aac_ioctl(struct dev_ioctl_args *ap)
2970 {
2971         cdev_t dev = ap->a_head.a_dev;
2972         caddr_t arg = ap->a_data;
2973         struct aac_softc *sc = dev->si_drv1;
2974         int error = 0;
2975         uint32_t cookie;
2976
2977         debug_called(2);
2978
2979         if (ap->a_cmd == AACIO_STATS) {
2980                 union aac_statrequest *as = (union aac_statrequest *)arg;
2981
2982                 switch (as->as_item) {
2983                 case AACQ_FREE:
2984                 case AACQ_BIO:
2985                 case AACQ_READY:
2986                 case AACQ_BUSY:
2987                 case AACQ_COMPLETE:
2988                         bcopy(&sc->aac_qstat[as->as_item], &as->as_qstat,
2989                               sizeof(struct aac_qstat));
2990                         break;
2991                 default:
2992                         error = ENOENT;
2993                         break;
2994                 }
2995                 return(error);
2996         }
2997
2998         arg = *(caddr_t *)arg;
2999
3000         switch (ap->a_cmd) {
3001         /* AACIO_STATS already handled above */
3002         case FSACTL_SENDFIB:
3003                 debug(1, "FSACTL_SENDFIB");
3004                 error = aac_ioctl_sendfib(sc, arg);
3005                 break;
3006         case FSACTL_AIF_THREAD:
3007                 debug(1, "FSACTL_AIF_THREAD");
3008                 error = EINVAL;
3009                 break;
3010         case FSACTL_OPEN_GET_ADAPTER_FIB:
3011                 debug(1, "FSACTL_OPEN_GET_ADAPTER_FIB");
3012                 /*
3013                  * Pass the caller out an AdapterFibContext.
3014                  *
3015                  * Note that because we only support one opener, we
3016                  * basically ignore this.  Set the caller's context to a magic
3017                  * number just in case.
3018                  *
3019                  * The Linux code hands the driver a pointer into kernel space,
3020                  * and then trusts it when the caller hands it back.  Aiee!
3021                  * Here, we give it the proc pointer of the per-adapter aif
3022                  * thread. It's only used as a sanity check in other calls.
3023                  */
3024                 cookie = (uint32_t)(uintptr_t)sc->aifthread;
3025                 error = copyout(&cookie, arg, sizeof(cookie));
3026                 break;
3027         case FSACTL_GET_NEXT_ADAPTER_FIB:
3028                 debug(1, "FSACTL_GET_NEXT_ADAPTER_FIB");
3029                 error = aac_getnext_aif(sc, arg);
3030                 break;
3031         case FSACTL_CLOSE_GET_ADAPTER_FIB:
3032                 debug(1, "FSACTL_CLOSE_GET_ADAPTER_FIB");
3033                 /* don't do anything here */
3034                 break;
3035         case FSACTL_MINIPORT_REV_CHECK:
3036                 debug(1, "FSACTL_MINIPORT_REV_CHECK");
3037                 error = aac_rev_check(sc, arg);
3038                 break;
3039         case FSACTL_QUERY_DISK:
3040                 debug(1, "FSACTL_QUERY_DISK");
3041                 error = aac_query_disk(sc, arg);
3042                 break;
3043         case FSACTL_DELETE_DISK:
3044                 /*
3045                  * We don't trust the underland to tell us when to delete a
3046                  * container, rather we rely on an AIF coming from the
3047                  * controller
3048                  */
3049                 error = 0;
3050                 break;
3051         case FSACTL_GET_PCI_INFO:
3052                 arg = *(caddr_t*)arg;
3053         case FSACTL_LNX_GET_PCI_INFO:
3054                 debug(1, "FSACTL_GET_PCI_INFO");
3055                 error = aac_get_pci_info(sc, arg);
3056                 break;
3057         default:
3058                 debug(1, "unsupported cmd 0x%lx\n", ap->a_cmd);
3059                 error = EINVAL;
3060                 break;
3061         }
3062         return(error);
3063 }
3064
3065 static int
3066 aac_poll(struct dev_poll_args *ap)
3067 {
3068         cdev_t dev = ap->a_head.a_dev;
3069         struct aac_softc *sc;
3070         int revents;
3071
3072         sc = dev->si_drv1;
3073         revents = 0;
3074
3075         AAC_LOCK_ACQUIRE(&sc->aac_aifq_lock);
3076         if ((ap->a_events & (POLLRDNORM | POLLIN)) != 0) {
3077                 if (sc->aac_aifq_tail != sc->aac_aifq_head)
3078                         revents |= ap->a_events & (POLLIN | POLLRDNORM);
3079         }
3080         AAC_LOCK_RELEASE(&sc->aac_aifq_lock);
3081
3082         if (revents == 0) {
3083                 if (ap->a_events & (POLLIN | POLLRDNORM))
3084                         selrecord(curthread, &sc->rcv_select);
3085         }
3086         ap->a_events = revents;
3087         return (0);
3088 }
3089
3090 static void
3091 aac_ioctl_event(struct aac_softc *sc, struct aac_event *event, void *arg)
3092 {
3093
3094         switch (event->ev_type) {
3095         case AAC_EVENT_CMFREE:
3096                 AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3097                 if (aac_alloc_command(sc, (struct aac_command **)arg)) {
3098                         aac_add_event(sc, event);
3099                         AAC_LOCK_RELEASE(&sc->aac_io_lock);
3100                         return;
3101                 }
3102                 kfree(event, M_AACBUF);
3103                 wakeup(arg);
3104                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
3105                 break;
3106         default:
3107                 break;
3108         }
3109 }
3110
3111 /*
3112  * Send a FIB supplied from userspace
3113  */
3114 static int
3115 aac_ioctl_sendfib(struct aac_softc *sc, caddr_t ufib)
3116 {
3117         struct aac_command *cm;
3118         int size, error;
3119
3120         debug_called(2);
3121
3122         cm = NULL;
3123
3124         /*
3125          * Get a command
3126          */
3127         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3128         if (aac_alloc_command(sc, &cm)) {
3129                 struct aac_event *event;
3130
3131                 event = kmalloc(sizeof(struct aac_event), M_AACBUF,
3132                     M_INTWAIT | M_ZERO);
3133                 event->ev_type = AAC_EVENT_CMFREE;
3134                 event->ev_callback = aac_ioctl_event;
3135                 event->ev_arg = &cm;
3136                 aac_add_event(sc, event);
3137                 crit_enter();
3138                 tsleep_interlock(&cm);
3139                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
3140                 tsleep(&cm, 0, "sendfib", 0);
3141                 AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3142                 crit_exit();
3143         }
3144         AAC_LOCK_RELEASE(&sc->aac_io_lock);
3145
3146         /*
3147          * Fetch the FIB header, then re-copy to get data as well.
3148          */
3149         if ((error = copyin(ufib, cm->cm_fib,
3150                             sizeof(struct aac_fib_header))) != 0)
3151                 goto out;
3152         size = cm->cm_fib->Header.Size + sizeof(struct aac_fib_header);
3153         if (size > sizeof(struct aac_fib)) {
3154                 device_printf(sc->aac_dev, "incoming FIB oversized (%d > %zd)\n",
3155                               size, sizeof(struct aac_fib));
3156                 size = sizeof(struct aac_fib);
3157         }
3158         if ((error = copyin(ufib, cm->cm_fib, size)) != 0)
3159                 goto out;
3160         cm->cm_fib->Header.Size = size;
3161         cm->cm_timestamp = time_second;
3162
3163         /*
3164          * Pass the FIB to the controller, wait for it to complete.
3165          */
3166         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3167         if ((error = aac_wait_command(cm)) != 0) {
3168                 device_printf(sc->aac_dev,
3169                               "aac_wait_command return %d\n", error);
3170                 goto out;
3171         }
3172         AAC_LOCK_RELEASE(&sc->aac_io_lock);
3173
3174         /*
3175          * Copy the FIB and data back out to the caller.
3176          */
3177         size = cm->cm_fib->Header.Size;
3178         if (size > sizeof(struct aac_fib)) {
3179                 device_printf(sc->aac_dev, "outbound FIB oversized (%d > %zd)\n",
3180                               size, sizeof(struct aac_fib));
3181                 size = sizeof(struct aac_fib);
3182         }
3183         error = copyout(cm->cm_fib, ufib, size);
3184         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3185
3186 out:
3187         if (cm != NULL) {
3188                 aac_release_command(cm);
3189         }
3190
3191         AAC_LOCK_RELEASE(&sc->aac_io_lock);
3192         return(error);
3193 }
3194
3195 /*
3196  * Handle an AIF sent to us by the controller; queue it for later reference.
3197  * If the queue fills up, then drop the older entries.
3198  */
3199 static void
3200 aac_handle_aif(struct aac_softc *sc, struct aac_fib *fib)
3201 {
3202         struct aac_aif_command *aif;
3203         struct aac_container *co, *co_next;
3204         struct aac_mntinfo *mi;
3205         struct aac_mntinforesp *mir = NULL;
3206         u_int16_t rsize;
3207         int next, found;
3208         int count = 0, added = 0, i = 0;
3209
3210         debug_called(2);
3211
3212         aif = (struct aac_aif_command*)&fib->data[0];
3213         aac_print_aif(sc, aif);
3214
3215         /* Is it an event that we should care about? */
3216         switch (aif->command) {
3217         case AifCmdEventNotify:
3218                 switch (aif->data.EN.type) {
3219                 case AifEnAddContainer:
3220                 case AifEnDeleteContainer:
3221                         /*
3222                          * A container was added or deleted, but the message
3223                          * doesn't tell us anything else!  Re-enumerate the
3224                          * containers and sort things out.
3225                          */
3226                         aac_alloc_sync_fib(sc, &fib);
3227                         mi = (struct aac_mntinfo *)&fib->data[0];
3228                         do {
3229                                 /*
3230                                  * Ask the controller for its containers one at
3231                                  * a time.
3232                                  * XXX What if the controller's list changes
3233                                  * midway through this enumaration?
3234                                  * XXX This should be done async.
3235                                  */
3236                                 bzero(mi, sizeof(struct aac_mntinfo));
3237                                 mi->Command = VM_NameServe;
3238                                 mi->MntType = FT_FILESYS;
3239                                 mi->MntCount = i;
3240                                 rsize = sizeof(mir);
3241                                 if (aac_sync_fib(sc, ContainerCommand, 0, fib,
3242                                                  sizeof(struct aac_mntinfo))) {
3243                                         device_printf(sc->aac_dev,
3244                                             "Error probing container %d\n", i);
3245
3246                                         continue;
3247                                 }
3248                                 mir = (struct aac_mntinforesp *)&fib->data[0];
3249                                 /* XXX Need to check if count changed */
3250                                 count = mir->MntRespCount;
3251
3252                                 /*
3253                                  * Check the container against our list.
3254                                  * co->co_found was already set to 0 in a
3255                                  * previous run.
3256                                  */
3257                                 if ((mir->Status == ST_OK) &&
3258                                     (mir->MntTable[0].VolType != CT_NONE)) {
3259                                         found = 0;
3260                                         TAILQ_FOREACH(co,
3261                                                       &sc->aac_container_tqh,
3262                                                       co_link) {
3263                                                 if (co->co_mntobj.ObjectId ==
3264                                                     mir->MntTable[0].ObjectId) {
3265                                                         co->co_found = 1;
3266                                                         found = 1;
3267                                                         break;
3268                                                 }
3269                                         }
3270                                         /*
3271                                          * If the container matched, continue
3272                                          * in the list.
3273                                          */
3274                                         if (found) {
3275                                                 i++;
3276                                                 continue;
3277                                         }
3278
3279                                         /*
3280                                          * This is a new container.  Do all the
3281                                          * appropriate things to set it up.
3282                                          */
3283                                         aac_add_container(sc, mir, 1);
3284                                         added = 1;
3285                                 }
3286                                 i++;
3287                         } while ((i < count) && (i < AAC_MAX_CONTAINERS));
3288                         aac_release_sync_fib(sc);
3289
3290                         /*
3291                          * Go through our list of containers and see which ones
3292                          * were not marked 'found'.  Since the controller didn't
3293                          * list them they must have been deleted.  Do the
3294                          * appropriate steps to destroy the device.  Also reset
3295                          * the co->co_found field.
3296                          */
3297                         co = TAILQ_FIRST(&sc->aac_container_tqh);
3298                         while (co != NULL) {
3299                                 if (co->co_found == 0) {
3300                                         AAC_LOCK_RELEASE(&sc->aac_io_lock);
3301                                         get_mplock();
3302                                         device_delete_child(sc->aac_dev,
3303                                                             co->co_disk);
3304                                         rel_mplock();
3305                                         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3306                                         co_next = TAILQ_NEXT(co, co_link);
3307                                         AAC_LOCK_ACQUIRE(&sc->
3308                                                         aac_container_lock);
3309                                         TAILQ_REMOVE(&sc->aac_container_tqh, co,
3310                                                      co_link);
3311                                         AAC_LOCK_RELEASE(&sc->
3312                                                          aac_container_lock);
3313                                         kfree(co, M_AACBUF);
3314                                         co = co_next;
3315                                 } else {
3316                                         co->co_found = 0;
3317                                         co = TAILQ_NEXT(co, co_link);
3318                                 }
3319                         }
3320
3321                         /* Attach the newly created containers */
3322                         if (added) {
3323                                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
3324                                 get_mplock();
3325                                 bus_generic_attach(sc->aac_dev);
3326                                 rel_mplock();
3327                                 AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3328                         }
3329
3330                         break;
3331
3332                 default:
3333                         break;
3334                 }
3335
3336         default:
3337                 break;
3338         }
3339
3340         /* Copy the AIF data to the AIF queue for ioctl retrieval */
3341         AAC_LOCK_ACQUIRE(&sc->aac_aifq_lock);
3342         next = (sc->aac_aifq_head + 1) % AAC_AIFQ_LENGTH;
3343         if (next != sc->aac_aifq_tail) {
3344                 bcopy(aif, &sc->aac_aifq[next], sizeof(struct aac_aif_command));
3345                 sc->aac_aifq_head = next;
3346
3347                 /* On the off chance that someone is sleeping for an aif... */
3348                 if (sc->aac_state & AAC_STATE_AIF_SLEEPER)
3349                         wakeup(sc->aac_aifq);
3350                 /* token may have been lost */
3351                 /* Wakeup any poll()ers */
3352                 selwakeup(&sc->rcv_select);
3353                 /* token may have been lost */
3354         }
3355         AAC_LOCK_RELEASE(&sc->aac_aifq_lock);
3356
3357         return;
3358 }
3359
3360 /*
3361  * Return the Revision of the driver to userspace and check to see if the
3362  * userspace app is possibly compatible.  This is extremely bogus since
3363  * our driver doesn't follow Adaptec's versioning system.  Cheat by just
3364  * returning what the card reported.
3365  */
3366 static int
3367 aac_rev_check(struct aac_softc *sc, caddr_t udata)
3368 {
3369         struct aac_rev_check rev_check;
3370         struct aac_rev_check_resp rev_check_resp;
3371         int error = 0;
3372
3373         debug_called(2);
3374
3375         /*
3376          * Copyin the revision struct from userspace
3377          */
3378         if ((error = copyin(udata, (caddr_t)&rev_check,
3379                         sizeof(struct aac_rev_check))) != 0) {
3380                 return error;
3381         }
3382
3383         debug(2, "Userland revision= %d\n",
3384               rev_check.callingRevision.buildNumber);
3385
3386         /*
3387          * Doctor up the response struct.
3388          */
3389         rev_check_resp.possiblyCompatible = 1;
3390         rev_check_resp.adapterSWRevision.external.ul =
3391             sc->aac_revision.external.ul;
3392         rev_check_resp.adapterSWRevision.buildNumber =
3393             sc->aac_revision.buildNumber;
3394
3395         return(copyout((caddr_t)&rev_check_resp, udata,
3396                         sizeof(struct aac_rev_check_resp)));
3397 }
3398
3399 /*
3400  * Pass the caller the next AIF in their queue
3401  */
3402 static int
3403 aac_getnext_aif(struct aac_softc *sc, caddr_t arg)
3404 {
3405         struct get_adapter_fib_ioctl agf;
3406         int error;
3407
3408         debug_called(2);
3409
3410         if ((error = copyin(arg, &agf, sizeof(agf))) == 0) {
3411
3412                 /*
3413                  * Check the magic number that we gave the caller.
3414                  */
3415                 if (agf.AdapterFibContext != (int)(uintptr_t)sc->aifthread) {
3416                         error = EFAULT;
3417                 } else {
3418
3419                         error = aac_return_aif(sc, agf.AifFib);
3420
3421                         if ((error == EAGAIN) && (agf.Wait)) {
3422                                 sc->aac_state |= AAC_STATE_AIF_SLEEPER;
3423                                 while (error == EAGAIN) {
3424                                         error = tsleep(sc->aac_aifq,
3425                                                        PCATCH, "aacaif", 0);
3426                                         if (error == 0)
3427                                                 error = aac_return_aif(sc,
3428                                                     agf.AifFib);
3429                                 }
3430                                 sc->aac_state &= ~AAC_STATE_AIF_SLEEPER;
3431                         }
3432                 }
3433         }
3434         return(error);
3435 }
3436
3437 /*
3438  * Hand the next AIF off the top of the queue out to userspace.
3439  *
3440  * YYY token could be lost during copyout
3441  */
3442 static int
3443 aac_return_aif(struct aac_softc *sc, caddr_t uptr)
3444 {
3445         int next, error;
3446
3447         debug_called(2);
3448
3449         AAC_LOCK_ACQUIRE(&sc->aac_aifq_lock);
3450         if (sc->aac_aifq_tail == sc->aac_aifq_head) {
3451                 AAC_LOCK_RELEASE(&sc->aac_aifq_lock);
3452                 return (EAGAIN);
3453         }
3454
3455         next = (sc->aac_aifq_tail + 1) % AAC_AIFQ_LENGTH;
3456         error = copyout(&sc->aac_aifq[next], uptr,
3457                         sizeof(struct aac_aif_command));
3458         if (error)
3459                 device_printf(sc->aac_dev,
3460                     "aac_return_aif: copyout returned %d\n", error);
3461         else
3462                 sc->aac_aifq_tail = next;
3463
3464         AAC_LOCK_RELEASE(&sc->aac_aifq_lock);
3465         return(error);
3466 }
3467
3468 static int
3469 aac_get_pci_info(struct aac_softc *sc, caddr_t uptr)
3470 {
3471         struct aac_pci_info {
3472                 u_int32_t bus;
3473                 u_int32_t slot;
3474         } pciinf;
3475         int error;
3476
3477         debug_called(2);
3478
3479         pciinf.bus = pci_get_bus(sc->aac_dev);
3480         pciinf.slot = pci_get_slot(sc->aac_dev);
3481
3482         error = copyout((caddr_t)&pciinf, uptr,
3483                         sizeof(struct aac_pci_info));
3484
3485         return (error);
3486 }
3487
3488 /*
3489  * Give the userland some information about the container.  The AAC arch
3490  * expects the driver to be a SCSI passthrough type driver, so it expects
3491  * the containers to have b:t:l numbers.  Fake it.
3492  */
3493 static int
3494 aac_query_disk(struct aac_softc *sc, caddr_t uptr)
3495 {
3496         struct aac_query_disk query_disk;
3497         struct aac_container *co;
3498         struct aac_disk *disk;
3499         int error, id;
3500
3501         debug_called(2);
3502
3503         disk = NULL;
3504
3505         error = copyin(uptr, (caddr_t)&query_disk,
3506                        sizeof(struct aac_query_disk));
3507         if (error)
3508                 return (error);
3509
3510         id = query_disk.ContainerNumber;
3511         if (id == -1)
3512                 return (EINVAL);
3513
3514         AAC_LOCK_ACQUIRE(&sc->aac_container_lock);
3515         TAILQ_FOREACH(co, &sc->aac_container_tqh, co_link) {
3516                 if (co->co_mntobj.ObjectId == id)
3517                         break;
3518                 }
3519
3520         if (co == NULL) {
3521                         query_disk.Valid = 0;
3522                         query_disk.Locked = 0;
3523                         query_disk.Deleted = 1;         /* XXX is this right? */
3524         } else {
3525                 disk = device_get_softc(co->co_disk);
3526                 query_disk.Valid = 1;
3527                 query_disk.Locked =
3528                     (disk->ad_flags & AAC_DISK_OPEN) ? 1 : 0;
3529                 query_disk.Deleted = 0;
3530                 query_disk.Bus = device_get_unit(sc->aac_dev);
3531                 query_disk.Target = disk->unit;
3532                 query_disk.Lun = 0;
3533                 query_disk.UnMapped = 0;
3534                 bcopy(disk->ad_dev_t->si_name,
3535                       &query_disk.diskDeviceName[0], 10);
3536         }
3537         AAC_LOCK_RELEASE(&sc->aac_container_lock);
3538
3539         error = copyout((caddr_t)&query_disk, uptr,
3540                         sizeof(struct aac_query_disk));
3541
3542         return (error);
3543 }
3544
3545 static void
3546 aac_get_bus_info(struct aac_softc *sc)
3547 {
3548         struct aac_fib *fib;
3549         struct aac_ctcfg *c_cmd;
3550         struct aac_ctcfg_resp *c_resp;
3551         struct aac_vmioctl *vmi;
3552         struct aac_vmi_businf_resp *vmi_resp;
3553         struct aac_getbusinf businfo;
3554         struct aac_sim *caminf;
3555         device_t child;
3556         int i, found, error;
3557
3558         AAC_LOCK_ACQUIRE(&sc->aac_io_lock);
3559         aac_alloc_sync_fib(sc, &fib);
3560         c_cmd = (struct aac_ctcfg *)&fib->data[0];
3561         bzero(c_cmd, sizeof(struct aac_ctcfg));
3562
3563         c_cmd->Command = VM_ContainerConfig;
3564         c_cmd->cmd = CT_GET_SCSI_METHOD;
3565         c_cmd->param = 0;
3566
3567         error = aac_sync_fib(sc, ContainerCommand, 0, fib,
3568             sizeof(struct aac_ctcfg));
3569         if (error) {
3570                 device_printf(sc->aac_dev, "Error %d sending "
3571                     "VM_ContainerConfig command\n", error);
3572                 aac_release_sync_fib(sc);
3573                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
3574                 return;
3575         }
3576
3577         c_resp = (struct aac_ctcfg_resp *)&fib->data[0];
3578         if (c_resp->Status != ST_OK) {
3579                 device_printf(sc->aac_dev, "VM_ContainerConfig returned 0x%x\n",
3580                     c_resp->Status);
3581                 aac_release_sync_fib(sc);
3582                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
3583                 return;
3584         }
3585
3586         sc->scsi_method_id = c_resp->param;
3587
3588         vmi = (struct aac_vmioctl *)&fib->data[0];
3589         bzero(vmi, sizeof(struct aac_vmioctl));
3590
3591         vmi->Command = VM_Ioctl;
3592         vmi->ObjType = FT_DRIVE;
3593         vmi->MethId = sc->scsi_method_id;
3594         vmi->ObjId = 0;
3595         vmi->IoctlCmd = GetBusInfo;
3596
3597         error = aac_sync_fib(sc, ContainerCommand, 0, fib,
3598             sizeof(struct aac_vmioctl));
3599         if (error) {
3600                 device_printf(sc->aac_dev, "Error %d sending VMIoctl command\n",
3601                     error);
3602                 aac_release_sync_fib(sc);
3603                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
3604                 return;
3605         }
3606
3607         vmi_resp = (struct aac_vmi_businf_resp *)&fib->data[0];
3608         if (vmi_resp->Status != ST_OK) {
3609                 debug(1, "VM_Ioctl returned %d\n", vmi_resp->Status);
3610                 aac_release_sync_fib(sc);
3611                 AAC_LOCK_RELEASE(&sc->aac_io_lock);
3612                 return;
3613         }
3614
3615         bcopy(&vmi_resp->BusInf, &businfo, sizeof(struct aac_getbusinf));
3616         aac_release_sync_fib(sc);
3617         AAC_LOCK_RELEASE(&sc->aac_io_lock);
3618
3619         found = 0;
3620         for (i = 0; i < businfo.BusCount; i++) {
3621                 if (businfo.BusValid[i] != AAC_BUS_VALID)
3622                         continue;
3623
3624                 caminf = (struct aac_sim *)kmalloc(sizeof(struct aac_sim),
3625                     M_AACBUF, M_INTWAIT | M_ZERO);
3626
3627                 child = device_add_child(sc->aac_dev, "aacp", -1);
3628                 if (child == NULL) {
3629                         device_printf(sc->aac_dev,
3630                             "device_add_child failed for passthrough bus %d\n",
3631                             i);
3632                         kfree(caminf, M_AACBUF);
3633                         break;
3634                 };
3635
3636                 caminf->TargetsPerBus = businfo.TargetsPerBus;
3637                 caminf->BusNumber = i;
3638                 caminf->InitiatorBusId = businfo.InitiatorBusId[i];
3639                 caminf->aac_sc = sc;
3640                 caminf->sim_dev = child;
3641
3642                 device_set_ivars(child, caminf);
3643                 device_set_desc(child, "SCSI Passthrough Bus");
3644                 TAILQ_INSERT_TAIL(&sc->aac_sim_tqh, caminf, sim_link);
3645
3646                 found = 1;
3647         }
3648
3649         if (found)
3650                 bus_generic_attach(sc->aac_dev);
3651
3652         return;
3653 }