<sys/time.h>: Add 3rd arg to timespecadd()/sub() and make them public.
[dragonfly.git] / sys / dev / disk / isp / isp_freebsd.c
1 /*-
2  * Copyright (c) 1997-2009 by Matthew Jacob
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice immediately at the beginning of the file, without modification,
10  *    this list of conditions, and the following disclaimer.
11  * 2. The name of the author may not be used to endorse or promote products
12  *    derived from this software without specific prior written permission.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
18  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD: src/sys/dev/isp/isp_freebsd.c,v 1.176 2011/11/06 00:44:40 mjacob Exp $
27  */
28
29 /*
30  * Platform (FreeBSD) dependent common attachment code for Qlogic adapters.
31  */
32 #include <dev/disk/isp/isp_freebsd.h>
33 #include <sys/unistd.h>
34 #include <sys/kthread.h>
35 #include <sys/conf.h>
36 #include <sys/module.h>
37 #include <dev/disk/isp/isp_ioctl.h>
38 #include <bus/cam/cam_periph.h>
39 #include <bus/cam/cam_xpt_periph.h>
40 #include <sys/device.h>
41
42 #define THREAD_CREATE   kthread_create
43
44 MODULE_VERSION(isp, 1);
45 MODULE_DEPEND(isp, cam, 1, 1, 1);
46 int isp_announced = 0;
47 int isp_fabric_hysteresis = 5;
48 int isp_loop_down_limit = 60;   /* default loop down limit */
49 int isp_change_is_bad = 0;      /* "changed" devices are bad */
50 int isp_quickboot_time = 7;     /* don't wait more than N secs for loop up */
51 int isp_gone_device_time = 30;  /* grace time before reporting device lost */
52 int isp_autoconfig = 1;         /* automatically attach/detach devices */
53 static const char *roles[4] = {
54     "(none)", "Target", "Initiator", "Target/Initiator"
55 };
56 static const char prom3[] = "Chan %d PortID 0x%06x Departed from Target %u because of %s";
57 #ifdef ISP_TARGET_MODE
58 static const char rqo[] = "%s: Request Queue Overflow\n";
59 #endif
60
61 static void isp_freeze_loopdown(ispsoftc_t *, int, char *);
62 static d_ioctl_t ispioctl;
63 static void isp_intr_enable(void *);
64 static void isp_cam_async(void *, uint32_t, struct cam_path *, void *);
65 static void isp_poll(struct cam_sim *);
66 static timeout_t isp_watchdog;
67 static timeout_t isp_gdt;
68 static task_fn_t isp_gdt_task;
69 static timeout_t isp_ldt;
70 static task_fn_t isp_ldt_task;
71 static void isp_kthread(void *);
72 static void isp_action(struct cam_sim *, union ccb *);
73 #ifdef  ISP_INTERNAL_TARGET
74 static void isp_target_thread_pi(void *);
75 static void isp_target_thread_fc(void *);
76 #endif
77 static void isp_timer(void *);
78
79 static struct dev_ops isp_ops = {
80         { "isp", 0, D_MPSAFE },
81         .d_ioctl =      ispioctl,
82 };
83
84 static int
85 isp_attach_chan(ispsoftc_t *isp, struct cam_devq *devq, int chan)
86 {
87         struct ccb_setasync *csa;
88         struct cam_sim *sim;
89         struct cam_path *path;
90
91         /*
92          * Construct our SIM entry.
93          */
94         sim = cam_sim_alloc(isp_action, isp_poll, "isp", isp, device_get_unit(isp->isp_dev), &isp->isp_osinfo.lock, isp->isp_maxcmds, isp->isp_maxcmds, devq);
95         cam_simq_release(devq);
96
97         if (sim == NULL) {
98                 return (ENOMEM);
99         }
100
101         ISP_LOCK(isp);
102         if (xpt_bus_register(sim, chan) != CAM_SUCCESS) {
103                 ISP_UNLOCK(isp);
104                 cam_sim_free(sim);
105                 return (EIO);
106         }
107         ISP_UNLOCK(isp);
108         if (xpt_create_path_unlocked(&path, NULL, cam_sim_path(sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
109                 ISP_LOCK(isp);
110                 xpt_bus_deregister(cam_sim_path(sim));
111                 ISP_UNLOCK(isp);
112                 cam_sim_free(sim);
113                 return (ENXIO);
114         }
115         csa = &xpt_alloc_ccb()->csa;
116         xpt_setup_ccb(&csa->ccb_h, path, 5);
117         csa->ccb_h.func_code = XPT_SASYNC_CB;
118         csa->event_enable = AC_LOST_DEVICE;
119         csa->callback = isp_cam_async;
120         csa->callback_arg = sim;
121         xpt_action((union ccb *)csa);
122         xpt_free_ccb(&csa->ccb_h);
123
124         if (IS_SCSI(isp)) {
125                 struct isp_spi *spi = ISP_SPI_PC(isp, chan);
126                 spi->sim = sim;
127                 spi->path = path;
128 #ifdef  ISP_INTERNAL_TARGET
129                 ISP_SET_PC(isp, chan, proc_active, 1);
130                 if (THREAD_CREATE(isp_target_thread_pi, spi, &spi->target_proc, "%s: isp_test_tgt%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) {
131                         ISP_SET_PC(isp, chan, proc_active, 0);
132                         isp_prt(isp, ISP_LOGERR, "cannot create test target thread");
133                 }
134 #endif
135         } else {
136                 fcparam *fcp = FCPARAM(isp, chan);
137                 struct isp_fc *fc = ISP_FC_PC(isp, chan);
138
139                 ISP_LOCK(isp);
140                 fc->sim = sim;
141                 fc->path = path;
142                 fc->isp = isp;
143                 fc->ready = 1;
144
145                 callout_init(&fc->ldt);
146                 callout_init(&fc->gdt);
147                 TASK_INIT(&fc->ltask, 1, isp_ldt_task, fc);
148                 TASK_INIT(&fc->gtask, 1, isp_gdt_task, fc);
149
150                 /*
151                  * We start by being "loop down" if we have an initiator role
152                  */
153                 if (fcp->role & ISP_ROLE_INITIATOR) {
154                         isp_freeze_loopdown(isp, chan, "isp_attach");
155                         callout_reset(&fc->ldt, isp_quickboot_time * hz, isp_ldt, fc);
156                         isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "Starting Initial Loop Down Timer @ %lu", (unsigned long) time_second);
157                 }
158                 ISP_UNLOCK(isp);
159                 if (THREAD_CREATE(isp_kthread, fc, &fc->kthread, "%s: fc_thrd%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) {
160                         xpt_free_path(fc->path);
161                         ISP_LOCK(isp);
162                         if (callout_active(&fc->ldt)) {
163                                 callout_stop(&fc->ldt);
164                         }
165                         xpt_bus_deregister(cam_sim_path(fc->sim));
166                         ISP_UNLOCK(isp);
167                         cam_sim_free(fc->sim);
168                         return (ENOMEM);
169                 }
170 #ifdef  ISP_INTERNAL_TARGET
171                 ISP_SET_PC(isp, chan, proc_active, 1);
172                 if (THREAD_CREATE(isp_target_thread_fc, fc, &fc->target_proc, "%s: isp_test_tgt%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) {
173                         ISP_SET_PC(isp, chan, proc_active, 0);
174                         isp_prt(isp, ISP_LOGERR, "cannot create test target thread");
175                 }
176 #endif
177                 if (chan == 0) {
178                         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(isp->isp_osinfo.dev);
179                         struct sysctl_oid *tree = device_get_sysctl_tree(isp->isp_osinfo.dev);
180                         SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwnn", CTLFLAG_RD, &FCPARAM(isp, 0)->isp_wwnn, 0, "World Wide Node Name");
181                         SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwpn", CTLFLAG_RD, &FCPARAM(isp, 0)->isp_wwpn, 0, "World Wide Port Name");
182                         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "loop_down_limit", CTLFLAG_RW, &ISP_FC_PC(isp, 0)->loop_down_limit, 0, "Loop Down Limit");
183                         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "gone_device_time", CTLFLAG_RW, &ISP_FC_PC(isp, 0)->gone_device_time, 0, "Gone Device Time");
184                 }
185         }
186         return (0);
187 }
188
189 int
190 isp_attach(ispsoftc_t *isp)
191 {
192         const char *nu = device_get_nameunit(isp->isp_osinfo.dev);
193         int du = device_get_unit(isp->isp_dev);
194         int chan;
195
196         isp->isp_osinfo.ehook.ich_func = isp_intr_enable;
197         isp->isp_osinfo.ehook.ich_arg = isp;
198         isp->isp_osinfo.ehook.ich_desc = "isp";
199         /*
200          * Haha. Set this first, because if we're loaded as a module isp_intr_enable
201          * will be called right awawy, which will clear isp_osinfo.ehook_active,
202          * which would be unwise to then set again later.
203          */
204         isp->isp_osinfo.ehook_active = 1;
205         if (config_intrhook_establish(&isp->isp_osinfo.ehook) != 0) {
206                 isp_prt(isp, ISP_LOGERR, "could not establish interrupt enable hook");
207                 return (-EIO);
208         }
209
210         /*
211          * Create the device queue for our SIM(s).
212          */
213         isp->isp_osinfo.devq = cam_simq_alloc(isp->isp_maxcmds);
214         if (isp->isp_osinfo.devq == NULL) {
215                 config_intrhook_disestablish(&isp->isp_osinfo.ehook);
216                 return (EIO);
217         }
218
219         for (chan = 0; chan < isp->isp_nchan; chan++) {
220                 if (isp_attach_chan(isp, isp->isp_osinfo.devq, chan)) {
221                         goto unwind;
222                 }
223         }
224
225         callout_init(&isp->isp_osinfo.tmo);
226         callout_reset(&isp->isp_osinfo.tmo, hz, isp_timer, isp);
227         isp->isp_osinfo.timer_active = 1;
228
229         isp->isp_osinfo.cdev = make_dev(&isp_ops, du, UID_ROOT, GID_OPERATOR, 0600, "%s", nu);
230         if (isp->isp_osinfo.cdev) {
231                 isp->isp_osinfo.cdev->si_drv1 = isp;
232         }
233         return (0);
234
235 unwind:
236         while (--chan >= 0) {
237                 struct cam_sim *sim;
238                 struct cam_path *path;
239                 if (IS_FC(isp)) {
240                         sim = ISP_FC_PC(isp, chan)->sim;
241                         path = ISP_FC_PC(isp, chan)->path;
242                 } else {
243                         sim = ISP_SPI_PC(isp, chan)->sim;
244                         path = ISP_SPI_PC(isp, chan)->path;
245                 }
246                 xpt_free_path(path);
247                 ISP_LOCK(isp);
248                 xpt_bus_deregister(cam_sim_path(sim));
249                 ISP_UNLOCK(isp);
250                 cam_sim_free(sim);
251         }
252         if (isp->isp_osinfo.ehook_active) {
253                 config_intrhook_disestablish(&isp->isp_osinfo.ehook);
254                 isp->isp_osinfo.ehook_active = 0;
255         }
256         if (isp->isp_osinfo.cdev) {
257                 destroy_dev(isp->isp_osinfo.cdev);
258                 isp->isp_osinfo.cdev = NULL;
259         }
260         isp->isp_osinfo.devq = NULL;
261         return (-1);
262 }
263
264 int
265 isp_detach(ispsoftc_t *isp)
266 {
267         struct cam_sim *sim;
268         struct cam_path *path;
269         struct ccb_setasync *csa;
270         int chan;
271
272         ISP_LOCK(isp);
273         for (chan = isp->isp_nchan - 1; chan >= 0; chan -= 1) {
274                 if (IS_FC(isp)) {
275                         sim = ISP_FC_PC(isp, chan)->sim;
276                         path = ISP_FC_PC(isp, chan)->path;
277                 } else {
278                         sim = ISP_SPI_PC(isp, chan)->sim;
279                         path = ISP_SPI_PC(isp, chan)->path;
280                 }
281                 if (sim->refcount > 2) {
282                         ISP_UNLOCK(isp);
283                         return (EBUSY);
284                 }
285         }
286         if (isp->isp_osinfo.timer_active) {
287                 callout_stop(&isp->isp_osinfo.tmo);
288                 isp->isp_osinfo.timer_active = 0;
289         }
290         for (chan = isp->isp_nchan - 1; chan >= 0; chan -= 1) {
291                 if (IS_FC(isp)) {
292                         sim = ISP_FC_PC(isp, chan)->sim;
293                         path = ISP_FC_PC(isp, chan)->path;
294                 } else {
295                         sim = ISP_SPI_PC(isp, chan)->sim;
296                         path = ISP_SPI_PC(isp, chan)->path;
297                 }
298                 csa = &xpt_alloc_ccb()->csa;
299                 xpt_setup_ccb(&csa->ccb_h, path, 5);
300                 csa->ccb_h.func_code = XPT_SASYNC_CB;
301                 csa->event_enable = 0;
302                 csa->callback = isp_cam_async;
303                 csa->callback_arg = sim;
304                 xpt_action((union ccb *)csa);
305                 xpt_free_ccb(&csa->ccb_h);
306                 xpt_free_path(path);
307                 xpt_bus_deregister(cam_sim_path(sim));
308                 cam_sim_free(sim);
309         }
310         ISP_UNLOCK(isp);
311         if (isp->isp_osinfo.cdev) {
312                 destroy_dev(isp->isp_osinfo.cdev);
313                 isp->isp_osinfo.cdev = NULL;
314         }
315         if (isp->isp_osinfo.ehook_active) {
316                 config_intrhook_disestablish(&isp->isp_osinfo.ehook);
317                 isp->isp_osinfo.ehook_active = 0;
318         }
319         if (isp->isp_osinfo.devq != NULL) {
320                 isp->isp_osinfo.devq = NULL;
321         }
322         return (0);
323 }
324
325 static void
326 isp_freeze_loopdown(ispsoftc_t *isp, int chan, char *msg)
327 {
328         if (IS_FC(isp)) {
329                 struct isp_fc *fc = ISP_FC_PC(isp, chan);
330                 if (fc->simqfrozen == 0) {
331                         isp_prt(isp, ISP_LOGDEBUG0, "%s: freeze simq (loopdown) chan %d", msg, chan);
332                         fc->simqfrozen = SIMQFRZ_LOOPDOWN;
333                         xpt_freeze_simq(fc->sim, 1);
334                 } else {
335                         isp_prt(isp, ISP_LOGDEBUG0, "%s: mark frozen (loopdown) chan %d", msg, chan);
336                         fc->simqfrozen |= SIMQFRZ_LOOPDOWN;
337                 }
338         }
339 }
340
341 static void
342 isp_unfreeze_loopdown(ispsoftc_t *isp, int chan)
343 {
344         if (IS_FC(isp)) {
345                 struct isp_fc *fc = ISP_FC_PC(isp, chan);
346                 int wasfrozen = fc->simqfrozen & SIMQFRZ_LOOPDOWN;
347                 fc->simqfrozen &= ~SIMQFRZ_LOOPDOWN;
348                 if (wasfrozen && fc->simqfrozen == 0) {
349                         isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "%s: Chan %d releasing simq", __func__, chan);
350                         xpt_release_simq(fc->sim, 1);
351                 }
352         }
353 }
354
355
356 static int
357 ispioctl(struct dev_ioctl_args *ap)
358 {
359         cdev_t dev = ap->a_head.a_dev;
360         caddr_t addr = ap->a_data;
361         u_long c = ap->a_cmd;
362         ispsoftc_t *isp;
363         int nr, chan, retval = ENOTTY;
364
365         isp = dev->si_drv1;
366
367         switch (c) {
368         case ISP_SDBLEV:
369         {
370                 int olddblev = isp->isp_dblev;
371                 isp->isp_dblev = *(int *)addr;
372                 *(int *)addr = olddblev;
373                 retval = 0;
374                 break;
375         }
376         case ISP_GETROLE:
377                 chan = *(int *)addr;
378                 if (chan < 0 || chan >= isp->isp_nchan) {
379                         retval = -ENXIO;
380                         break;
381                 }
382                 if (IS_FC(isp)) {
383                         *(int *)addr = FCPARAM(isp, chan)->role;
384                 } else {
385                         *(int *)addr = SDPARAM(isp, chan)->role;
386                 }
387                 retval = 0;
388                 break;
389         case ISP_SETROLE:
390                 nr = *(int *)addr;
391                 chan = nr >> 8;
392                 if (chan < 0 || chan >= isp->isp_nchan) {
393                         retval = -ENXIO;
394                         break;
395                 }
396                 nr &= 0xff;
397                 if (nr & ~(ISP_ROLE_INITIATOR|ISP_ROLE_TARGET)) {
398                         retval = EINVAL;
399                         break;
400                 }
401                 if (IS_FC(isp)) {
402                         /*
403                          * We don't really support dual role at present on FC cards.
404                          *
405                          * We should, but a bunch of things are currently broken,
406                          * so don't allow it.
407                          */
408                         if (nr == ISP_ROLE_BOTH) {
409                                 isp_prt(isp, ISP_LOGERR, "cannot support dual role at present");
410                                 retval = EINVAL;
411                                 break;
412                         }
413                         *(int *)addr = FCPARAM(isp, chan)->role;
414 #ifdef  ISP_INTERNAL_TARGET
415                         ISP_LOCK(isp);
416                         retval = isp_fc_change_role(isp, chan, nr);
417                         ISP_UNLOCK(isp);
418 #else
419                         FCPARAM(isp, chan)->role = nr;
420 #endif
421                 } else {
422                         *(int *)addr = SDPARAM(isp, chan)->role;
423                         SDPARAM(isp, chan)->role = nr;
424                 }
425                 retval = 0;
426                 break;
427
428         case ISP_RESETHBA:
429                 ISP_LOCK(isp);
430 #ifdef  ISP_TARGET_MODE
431                 isp_del_all_wwn_entries(isp, ISP_NOCHAN);
432 #endif
433                 isp_reinit(isp, 0);
434                 ISP_UNLOCK(isp);
435                 retval = 0;
436                 break;
437
438         case ISP_RESCAN:
439                 if (IS_FC(isp)) {
440                         chan = *(int *)addr;
441                         if (chan < 0 || chan >= isp->isp_nchan) {
442                                 retval = -ENXIO;
443                                 break;
444                         }
445                         ISP_LOCK(isp);
446                         if (isp_fc_runstate(isp, chan, 5 * 1000000)) {
447                                 retval = EIO;
448                         } else {
449                                 retval = 0;
450                         }
451                         ISP_UNLOCK(isp);
452                 }
453                 break;
454
455         case ISP_FC_LIP:
456                 if (IS_FC(isp)) {
457                         chan = *(int *)addr;
458                         if (chan < 0 || chan >= isp->isp_nchan) {
459                                 retval = -ENXIO;
460                                 break;
461                         }
462                         ISP_LOCK(isp);
463                         if (isp_control(isp, ISPCTL_SEND_LIP, chan)) {
464                                 retval = EIO;
465                         } else {
466                                 retval = 0;
467                         }
468                         ISP_UNLOCK(isp);
469                 }
470                 break;
471         case ISP_FC_GETDINFO:
472         {
473                 struct isp_fc_device *ifc = (struct isp_fc_device *) addr;
474                 fcportdb_t *lp;
475
476                 if (IS_SCSI(isp)) {
477                         break;
478                 }
479                 if (ifc->loopid >= MAX_FC_TARG) {
480                         retval = EINVAL;
481                         break;
482                 }
483                 lp = &FCPARAM(isp, ifc->chan)->portdb[ifc->loopid];
484                 if (lp->state == FC_PORTDB_STATE_VALID || lp->target_mode) {
485                         ifc->role = lp->roles;
486                         ifc->loopid = lp->handle;
487                         ifc->portid = lp->portid;
488                         ifc->node_wwn = lp->node_wwn;
489                         ifc->port_wwn = lp->port_wwn;
490                         retval = 0;
491                 } else {
492                         retval = ENODEV;
493                 }
494                 break;
495         }
496         case ISP_GET_STATS:
497         {
498                 isp_stats_t *sp = (isp_stats_t *) addr;
499
500                 ISP_MEMZERO(sp, sizeof (*sp));
501                 sp->isp_stat_version = ISP_STATS_VERSION;
502                 sp->isp_type = isp->isp_type;
503                 sp->isp_revision = isp->isp_revision;
504                 ISP_LOCK(isp);
505                 sp->isp_stats[ISP_INTCNT] = isp->isp_intcnt;
506                 sp->isp_stats[ISP_INTBOGUS] = isp->isp_intbogus;
507                 sp->isp_stats[ISP_INTMBOXC] = isp->isp_intmboxc;
508                 sp->isp_stats[ISP_INGOASYNC] = isp->isp_intoasync;
509                 sp->isp_stats[ISP_RSLTCCMPLT] = isp->isp_rsltccmplt;
510                 sp->isp_stats[ISP_FPHCCMCPLT] = isp->isp_fphccmplt;
511                 sp->isp_stats[ISP_RSCCHIWAT] = isp->isp_rscchiwater;
512                 sp->isp_stats[ISP_FPCCHIWAT] = isp->isp_fpcchiwater;
513                 ISP_UNLOCK(isp);
514                 retval = 0;
515                 break;
516         }
517         case ISP_CLR_STATS:
518                 ISP_LOCK(isp);
519                 isp->isp_intcnt = 0;
520                 isp->isp_intbogus = 0;
521                 isp->isp_intmboxc = 0;
522                 isp->isp_intoasync = 0;
523                 isp->isp_rsltccmplt = 0;
524                 isp->isp_fphccmplt = 0;
525                 isp->isp_rscchiwater = 0;
526                 isp->isp_fpcchiwater = 0;
527                 ISP_UNLOCK(isp);
528                 retval = 0;
529                 break;
530         case ISP_FC_GETHINFO:
531         {
532                 struct isp_hba_device *hba = (struct isp_hba_device *) addr;
533                 int chan = hba->fc_channel;
534
535                 if (chan < 0 || chan >= isp->isp_nchan) {
536                         retval = ENXIO;
537                         break;
538                 }
539                 hba->fc_fw_major = ISP_FW_MAJORX(isp->isp_fwrev);
540                 hba->fc_fw_minor = ISP_FW_MINORX(isp->isp_fwrev);
541                 hba->fc_fw_micro = ISP_FW_MICROX(isp->isp_fwrev);
542                 hba->fc_nchannels = isp->isp_nchan;
543                 if (IS_FC(isp)) {
544                         hba->fc_nports = MAX_FC_TARG;
545                         hba->fc_speed = FCPARAM(isp, hba->fc_channel)->isp_gbspeed;
546                         hba->fc_topology = FCPARAM(isp, chan)->isp_topo + 1;
547                         hba->fc_loopid = FCPARAM(isp, chan)->isp_loopid;
548                         hba->nvram_node_wwn = FCPARAM(isp, chan)->isp_wwnn_nvram;
549                         hba->nvram_port_wwn = FCPARAM(isp, chan)->isp_wwpn_nvram;
550                         hba->active_node_wwn = FCPARAM(isp, chan)->isp_wwnn;
551                         hba->active_port_wwn = FCPARAM(isp, chan)->isp_wwpn;
552                 } else {
553                         hba->fc_nports = MAX_TARGETS;
554                         hba->fc_speed = 0;
555                         hba->fc_topology = 0;
556                         hba->nvram_node_wwn = 0ull;
557                         hba->nvram_port_wwn = 0ull;
558                         hba->active_node_wwn = 0ull;
559                         hba->active_port_wwn = 0ull;
560                 }
561                 retval = 0;
562                 break;
563         }
564         case ISP_TSK_MGMT:
565         {
566                 int needmarker;
567                 struct isp_fc_tsk_mgmt *fct = (struct isp_fc_tsk_mgmt *) addr;
568                 uint16_t loopid;
569                 mbreg_t mbs;
570
571                 if (IS_SCSI(isp)) {
572                         break;
573                 }
574
575                 chan = fct->chan;
576                 if (chan < 0 || chan >= isp->isp_nchan) {
577                         retval = -ENXIO;
578                         break;
579                 }
580
581                 needmarker = retval = 0;
582                 loopid = fct->loopid;
583                 ISP_LOCK(isp);
584                 if (IS_24XX(isp)) {
585                         uint8_t local[QENTRY_LEN];
586                         isp24xx_tmf_t *tmf;
587                         isp24xx_statusreq_t *sp;
588                         fcparam *fcp = FCPARAM(isp, chan);
589                         fcportdb_t *lp;
590                         int i;
591
592                         for (i = 0; i < MAX_FC_TARG; i++) {
593                                 lp = &fcp->portdb[i];
594                                 if (lp->handle == loopid) {
595                                         break;
596                                 }
597                         }
598                         if (i == MAX_FC_TARG) {
599                                 retval = ENXIO;
600                                 ISP_UNLOCK(isp);
601                                 break;
602                         }
603                         /* XXX VALIDATE LP XXX */
604                         tmf = (isp24xx_tmf_t *) local;
605                         ISP_MEMZERO(tmf, QENTRY_LEN);
606                         tmf->tmf_header.rqs_entry_type = RQSTYPE_TSK_MGMT;
607                         tmf->tmf_header.rqs_entry_count = 1;
608                         tmf->tmf_nphdl = lp->handle;
609                         tmf->tmf_delay = 2;
610                         tmf->tmf_timeout = 2;
611                         tmf->tmf_tidlo = lp->portid;
612                         tmf->tmf_tidhi = lp->portid >> 16;
613                         tmf->tmf_vpidx = ISP_GET_VPIDX(isp, chan);
614                         tmf->tmf_lun[1] = fct->lun & 0xff;
615                         if (fct->lun >= 256) {
616                                 tmf->tmf_lun[0] = 0x40 | (fct->lun >> 8);
617                         }
618                         switch (fct->action) {
619                         case IPT_CLEAR_ACA:
620                                 tmf->tmf_flags = ISP24XX_TMF_CLEAR_ACA;
621                                 break;
622                         case IPT_TARGET_RESET:
623                                 tmf->tmf_flags = ISP24XX_TMF_TARGET_RESET;
624                                 needmarker = 1;
625                                 break;
626                         case IPT_LUN_RESET:
627                                 tmf->tmf_flags = ISP24XX_TMF_LUN_RESET;
628                                 needmarker = 1;
629                                 break;
630                         case IPT_CLEAR_TASK_SET:
631                                 tmf->tmf_flags = ISP24XX_TMF_CLEAR_TASK_SET;
632                                 needmarker = 1;
633                                 break;
634                         case IPT_ABORT_TASK_SET:
635                                 tmf->tmf_flags = ISP24XX_TMF_ABORT_TASK_SET;
636                                 needmarker = 1;
637                                 break;
638                         default:
639                                 retval = EINVAL;
640                                 break;
641                         }
642                         if (retval) {
643                                 ISP_UNLOCK(isp);
644                                 break;
645                         }
646                         MBSINIT(&mbs, MBOX_EXEC_COMMAND_IOCB_A64, MBLOGALL, 5000000);
647                         mbs.param[1] = QENTRY_LEN;
648                         mbs.param[2] = DMA_WD1(fcp->isp_scdma);
649                         mbs.param[3] = DMA_WD0(fcp->isp_scdma);
650                         mbs.param[6] = DMA_WD3(fcp->isp_scdma);
651                         mbs.param[7] = DMA_WD2(fcp->isp_scdma);
652
653                         if (FC_SCRATCH_ACQUIRE(isp, chan)) {
654                                 ISP_UNLOCK(isp);
655                                 retval = ENOMEM;
656                                 break;
657                         }
658                         isp_put_24xx_tmf(isp, tmf, fcp->isp_scratch);
659                         MEMORYBARRIER(isp, SYNC_SFORDEV, 0, QENTRY_LEN, chan);
660                         sp = (isp24xx_statusreq_t *) local;
661                         sp->req_completion_status = 1;
662                         retval = isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs);
663                         MEMORYBARRIER(isp, SYNC_SFORCPU, QENTRY_LEN, QENTRY_LEN, chan);
664                         isp_get_24xx_response(isp, &((isp24xx_statusreq_t *)fcp->isp_scratch)[1], sp);
665                         FC_SCRATCH_RELEASE(isp, chan);
666                         if (retval || sp->req_completion_status != 0) {
667                                 FC_SCRATCH_RELEASE(isp, chan);
668                                 retval = EIO;
669                         }
670                         if (retval == 0) {
671                                 if (needmarker) {
672                                         fcp->sendmarker = 1;
673                                 }
674                         }
675                 } else {
676                         MBSINIT(&mbs, 0, MBLOGALL, 0);
677                         if (ISP_CAP_2KLOGIN(isp) == 0) {
678                                 loopid <<= 8;
679                         }
680                         switch (fct->action) {
681                         case IPT_CLEAR_ACA:
682                                 mbs.param[0] = MBOX_CLEAR_ACA;
683                                 mbs.param[1] = loopid;
684                                 mbs.param[2] = fct->lun;
685                                 break;
686                         case IPT_TARGET_RESET:
687                                 mbs.param[0] = MBOX_TARGET_RESET;
688                                 mbs.param[1] = loopid;
689                                 needmarker = 1;
690                                 break;
691                         case IPT_LUN_RESET:
692                                 mbs.param[0] = MBOX_LUN_RESET;
693                                 mbs.param[1] = loopid;
694                                 mbs.param[2] = fct->lun;
695                                 needmarker = 1;
696                                 break;
697                         case IPT_CLEAR_TASK_SET:
698                                 mbs.param[0] = MBOX_CLEAR_TASK_SET;
699                                 mbs.param[1] = loopid;
700                                 mbs.param[2] = fct->lun;
701                                 needmarker = 1;
702                                 break;
703                         case IPT_ABORT_TASK_SET:
704                                 mbs.param[0] = MBOX_ABORT_TASK_SET;
705                                 mbs.param[1] = loopid;
706                                 mbs.param[2] = fct->lun;
707                                 needmarker = 1;
708                                 break;
709                         default:
710                                 retval = EINVAL;
711                                 break;
712                         }
713                         if (retval == 0) {
714                                 if (needmarker) {
715                                         FCPARAM(isp, chan)->sendmarker = 1;
716                                 }
717                                 retval = isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs);
718                                 if (retval) {
719                                         retval = EIO;
720                                 }
721                         }
722                 }
723                 ISP_UNLOCK(isp);
724                 break;
725         }
726         default:
727                 break;
728         }
729         return (retval);
730 }
731
732 static void
733 isp_intr_enable(void *arg)
734 {
735         int chan;
736         ispsoftc_t *isp = arg;
737         ISP_LOCK(isp);
738         for (chan = 0; chan < isp->isp_nchan; chan++) {
739                 if (IS_FC(isp)) {
740                         if (FCPARAM(isp, chan)->role != ISP_ROLE_NONE) {
741                                 ISP_ENABLE_INTS(isp);
742                                 break;
743                         }
744                 } else {
745                         if (SDPARAM(isp, chan)->role != ISP_ROLE_NONE) {
746                                 ISP_ENABLE_INTS(isp);
747                                 break;
748                         }
749                 }
750         }
751         isp->isp_osinfo.ehook_active = 0;
752         ISP_UNLOCK(isp);
753         /* Release our hook so that the boot can continue. */
754         config_intrhook_disestablish(&isp->isp_osinfo.ehook);
755 }
756
757 /*
758  * Local Inlines
759  */
760
761 static ISP_INLINE int isp_get_pcmd(ispsoftc_t *, union ccb *);
762 static ISP_INLINE void isp_free_pcmd(ispsoftc_t *, union ccb *);
763
764 static ISP_INLINE int
765 isp_get_pcmd(ispsoftc_t *isp, union ccb *ccb)
766 {
767         ISP_PCMD(ccb) = isp->isp_osinfo.pcmd_free;
768         if (ISP_PCMD(ccb) == NULL) {
769                 return (-1);
770         }
771         isp->isp_osinfo.pcmd_free = ((struct isp_pcmd *)ISP_PCMD(ccb))->next;
772         return (0);
773 }
774
775 static ISP_INLINE void
776 isp_free_pcmd(ispsoftc_t *isp, union ccb *ccb)
777 {
778         ((struct isp_pcmd *)ISP_PCMD(ccb))->next = isp->isp_osinfo.pcmd_free;
779         isp->isp_osinfo.pcmd_free = ISP_PCMD(ccb);
780         ISP_PCMD(ccb) = NULL;
781 }
782 /*
783  * Put the target mode functions here, because some are inlines
784  */
785
786 #ifdef  ISP_TARGET_MODE
787 static ISP_INLINE int is_lun_enabled(ispsoftc_t *, int, lun_id_t);
788 static ISP_INLINE tstate_t *get_lun_statep(ispsoftc_t *, int, lun_id_t);
789 static ISP_INLINE tstate_t *get_lun_statep_from_tag(ispsoftc_t *, int, uint32_t);
790 static ISP_INLINE void rls_lun_statep(ispsoftc_t *, tstate_t *);
791 static ISP_INLINE inot_private_data_t *get_ntp_from_tagdata(ispsoftc_t *, uint32_t, uint32_t, tstate_t **);
792 static ISP_INLINE atio_private_data_t *isp_get_atpd(ispsoftc_t *, tstate_t *, uint32_t);
793 static ISP_INLINE void isp_put_atpd(ispsoftc_t *, tstate_t *, atio_private_data_t *);
794 static ISP_INLINE inot_private_data_t *isp_get_ntpd(ispsoftc_t *, tstate_t *);
795 static ISP_INLINE inot_private_data_t *isp_find_ntpd(ispsoftc_t *, tstate_t *, uint32_t, uint32_t);
796 static ISP_INLINE void isp_put_ntpd(ispsoftc_t *, tstate_t *, inot_private_data_t *);
797 static cam_status create_lun_state(ispsoftc_t *, int, struct cam_path *, tstate_t **);
798 static void destroy_lun_state(ispsoftc_t *, tstate_t *);
799 static void isp_enable_lun(ispsoftc_t *, union ccb *);
800 static void isp_enable_deferred_luns(ispsoftc_t *, int);
801 static cam_status isp_enable_deferred(ispsoftc_t *, int, lun_id_t);
802 static void isp_disable_lun(ispsoftc_t *, union ccb *);
803 static int isp_enable_target_mode(ispsoftc_t *, int);
804 static void isp_ledone(ispsoftc_t *, lun_entry_t *);
805 static timeout_t isp_refire_putback_atio;
806 static void isp_complete_ctio(union ccb *);
807 static void isp_target_putback_atio(union ccb *);
808 static void isp_target_start_ctio(ispsoftc_t *, union ccb *);
809 static void isp_handle_platform_atio(ispsoftc_t *, at_entry_t *);
810 static void isp_handle_platform_atio2(ispsoftc_t *, at2_entry_t *);
811 static void isp_handle_platform_atio7(ispsoftc_t *, at7_entry_t *);
812 static void isp_handle_platform_ctio(ispsoftc_t *, void *);
813 static void isp_handle_platform_notify_scsi(ispsoftc_t *, in_entry_t *);
814 static void isp_handle_platform_notify_fc(ispsoftc_t *, in_fcentry_t *);
815 static void isp_handle_platform_notify_24xx(ispsoftc_t *, in_fcentry_24xx_t *);
816 static int isp_handle_platform_target_notify_ack(ispsoftc_t *, isp_notify_t *);
817 static void isp_handle_platform_target_tmf(ispsoftc_t *, isp_notify_t *);
818 static void isp_target_mark_aborted(ispsoftc_t *, union ccb *);
819 static void isp_target_mark_aborted_early(ispsoftc_t *, tstate_t *, uint32_t);
820
821 static ISP_INLINE int
822 is_lun_enabled(ispsoftc_t *isp, int bus, lun_id_t lun)
823 {
824         tstate_t *tptr;
825         struct tslist *lhp;
826
827         ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp);
828         SLIST_FOREACH(tptr, lhp, next) {
829                 if (xpt_path_lun_id(tptr->owner) == lun) {
830                         return (1);
831                 }
832         }
833         return (0);
834 }
835
836 static void
837 dump_tstates(ispsoftc_t *isp, int bus)
838 {
839         int i, j;
840         struct tslist *lhp;
841         tstate_t *tptr = NULL;
842
843         if (bus >= isp->isp_nchan) {
844                 return;
845         }
846         for (i = 0; i < LUN_HASH_SIZE; i++) {
847                 ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
848                 j = 0;
849                 SLIST_FOREACH(tptr, lhp, next) {
850                         xpt_print(tptr->owner, "[%d, %d] atio_cnt=%d inot_cnt=%d\n", i, j, tptr->atio_count, tptr->inot_count);
851                         j++;
852                 }
853         }
854 }
855
856 static ISP_INLINE tstate_t *
857 get_lun_statep(ispsoftc_t *isp, int bus, lun_id_t lun)
858 {
859         tstate_t *tptr = NULL;
860         struct tslist *lhp;
861         int i;
862
863         if (bus < isp->isp_nchan) {
864                 for (i = 0; i < LUN_HASH_SIZE; i++) {
865                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
866                         SLIST_FOREACH(tptr, lhp, next) {
867                                 if (xpt_path_lun_id(tptr->owner) == lun) {
868                                         tptr->hold++;
869                                         return (tptr);
870                                 }
871                         }
872                 }
873         }
874         return (NULL);
875 }
876
877 static ISP_INLINE tstate_t *
878 get_lun_statep_from_tag(ispsoftc_t *isp, int bus, uint32_t tagval)
879 {
880         tstate_t *tptr = NULL;
881         atio_private_data_t *atp;
882         struct tslist *lhp;
883         int i;
884
885         if (bus < isp->isp_nchan && tagval != 0) {
886                 for (i = 0; i < LUN_HASH_SIZE; i++) {
887                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
888                         SLIST_FOREACH(tptr, lhp, next) {
889                                 atp = isp_get_atpd(isp, tptr, tagval);
890                                 if (atp && atp->tag == tagval) {
891                                         tptr->hold++;
892                                         return (tptr);
893                                 }
894                         }
895                 }
896         }
897         return (NULL);
898 }
899
900 static ISP_INLINE inot_private_data_t *
901 get_ntp_from_tagdata(ispsoftc_t *isp, uint32_t tag_id, uint32_t seq_id, tstate_t **rslt)
902 {
903         inot_private_data_t *ntp;
904         tstate_t *tptr;
905         struct tslist *lhp;
906         int bus, i;
907
908         for (bus = 0; bus < isp->isp_nchan; bus++) {
909                 for (i = 0; i < LUN_HASH_SIZE; i++) {
910                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
911                         SLIST_FOREACH(tptr, lhp, next) {
912                                 ntp = isp_find_ntpd(isp, tptr, tag_id, seq_id);
913                                 if (ntp) {
914                                         *rslt = tptr;
915                                         tptr->hold++;
916                                         return (ntp);
917                                 }
918                         }
919                 }
920         }
921         return (NULL);
922 }
923 static ISP_INLINE void
924 rls_lun_statep(ispsoftc_t *isp, tstate_t *tptr)
925 {
926         KASSERT((tptr->hold), ("tptr not held"));
927         tptr->hold--;
928 }
929
930 static void
931 isp_tmcmd_restart(ispsoftc_t *isp)
932 {
933         inot_private_data_t *ntp;
934         tstate_t *tptr;
935         struct tslist *lhp;
936         int bus, i;
937
938         for (bus = 0; bus < isp->isp_nchan; bus++) {
939                 for (i = 0; i < LUN_HASH_SIZE; i++) {
940                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
941                         SLIST_FOREACH(tptr, lhp, next) {
942                                 inot_private_data_t *restart_queue = tptr->restart_queue;
943                                 tptr->restart_queue = NULL;
944                                 while (restart_queue) {
945                                         ntp = restart_queue;
946                                         restart_queue = ntp->rd.nt.nt_hba;
947                                         if (IS_24XX(isp)) {
948                                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at7_entry_t *)ntp->rd.data)->at_rxid);
949                                                 isp_handle_platform_atio7(isp, (at7_entry_t *) ntp->rd.data);
950                                         } else {
951                                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at2_entry_t *)ntp->rd.data)->at_rxid);
952                                                 isp_handle_platform_atio2(isp, (at2_entry_t *) ntp->rd.data);
953                                         }
954                                         isp_put_ntpd(isp, tptr, ntp);
955                                         if (tptr->restart_queue && restart_queue != NULL) {
956                                                 ntp = tptr->restart_queue;
957                                                 tptr->restart_queue = restart_queue;
958                                                 while (restart_queue->rd.nt.nt_hba) {
959                                                         restart_queue = restart_queue->rd.nt.nt_hba;
960                                                 }
961                                                 restart_queue->rd.nt.nt_hba = ntp;
962                                                 break;
963                                         }
964                                 }
965                         }
966                 }
967         }
968 }
969
970 static ISP_INLINE atio_private_data_t *
971 isp_get_atpd(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag)
972 {
973         atio_private_data_t *atp;
974
975         if (tag == 0) {
976                 atp = tptr->atfree;
977                 if (atp) {
978                         tptr->atfree = atp->next;
979                 }
980                 return (atp);
981         }
982         for (atp = tptr->atpool; atp < &tptr->atpool[ATPDPSIZE]; atp++) {
983                 if (atp->tag == tag) {
984                         return (atp);
985                 }
986         }
987         return (NULL);
988 }
989
990 static ISP_INLINE void
991 isp_put_atpd(ispsoftc_t *isp, tstate_t *tptr, atio_private_data_t *atp)
992 {
993         atp->tag = 0;
994         atp->dead = 0;
995         atp->next = tptr->atfree;
996         tptr->atfree = atp;
997 }
998
999 static void
1000 isp_dump_atpd(ispsoftc_t *isp, tstate_t *tptr)
1001 {
1002         atio_private_data_t *atp;
1003         const char *states[8] = { "Free", "ATIO", "CAM", "CTIO", "LAST_CTIO", "PDON", "?6", "7" };
1004
1005         for (atp = tptr->atpool; atp < &tptr->atpool[ATPDPSIZE]; atp++) {
1006                 if (atp->tag == 0) {
1007                         continue;
1008                 }
1009                 xpt_print(tptr->owner, "ATP: [0x%x] origdlen %u bytes_xfrd %u last_xfr %u lun %u nphdl 0x%04x s_id 0x%06x d_id 0x%06x oxid 0x%04x state %s\n",
1010                     atp->tag, atp->orig_datalen, atp->bytes_xfered, atp->last_xframt, atp->lun, atp->nphdl, atp->sid, atp->portid, atp->oxid, states[atp->state & 0x7]);
1011         }
1012 }
1013
1014
1015 static ISP_INLINE inot_private_data_t *
1016 isp_get_ntpd(ispsoftc_t *isp, tstate_t *tptr)
1017 {
1018         inot_private_data_t *ntp;
1019         ntp = tptr->ntfree;
1020         if (ntp) {
1021                 tptr->ntfree = ntp->next;
1022         }
1023         return (ntp);
1024 }
1025
1026 static ISP_INLINE inot_private_data_t *
1027 isp_find_ntpd(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag_id, uint32_t seq_id)
1028 {
1029         inot_private_data_t *ntp;
1030         for (ntp = tptr->ntpool; ntp < &tptr->ntpool[ATPDPSIZE]; ntp++) {
1031                 if (ntp->rd.tag_id == tag_id && ntp->rd.seq_id == seq_id) {
1032                         return (ntp);
1033                 }
1034         }
1035         return (NULL);
1036 }
1037
1038 static ISP_INLINE void
1039 isp_put_ntpd(ispsoftc_t *isp, tstate_t *tptr, inot_private_data_t *ntp)
1040 {
1041         ntp->rd.tag_id = ntp->rd.seq_id = 0;
1042         ntp->next = tptr->ntfree;
1043         tptr->ntfree = ntp;
1044 }
1045
1046 static cam_status
1047 create_lun_state(ispsoftc_t *isp, int bus, struct cam_path *path, tstate_t **rslt)
1048 {
1049         cam_status status;
1050         lun_id_t lun;
1051         struct tslist *lhp;
1052         tstate_t *tptr;
1053         int i;
1054
1055         lun = xpt_path_lun_id(path);
1056         if (lun != CAM_LUN_WILDCARD) {
1057                 if (lun >= ISP_MAX_LUNS(isp)) {
1058                         return (CAM_LUN_INVALID);
1059                 }
1060         }
1061         if (is_lun_enabled(isp, bus, lun)) {
1062                 return (CAM_LUN_ALRDY_ENA);
1063         }
1064         tptr = kmalloc(sizeof (tstate_t), M_DEVBUF, M_WAITOK | M_ZERO);
1065         status = xpt_create_path(&tptr->owner, NULL, xpt_path_path_id(path), xpt_path_target_id(path), lun);
1066         if (status != CAM_REQ_CMP) {
1067                 kfree(tptr, M_DEVBUF);
1068                 return (status);
1069         }
1070         SLIST_INIT(&tptr->atios);
1071         SLIST_INIT(&tptr->inots);
1072         for (i = 0; i < ATPDPSIZE-1; i++) {
1073                 tptr->atpool[i].next = &tptr->atpool[i+1];
1074                 tptr->ntpool[i].next = &tptr->ntpool[i+1];
1075         }
1076         tptr->atfree = tptr->atpool;
1077         tptr->ntfree = tptr->ntpool;
1078         tptr->hold = 1;
1079         ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(xpt_path_lun_id(tptr->owner))], lhp);
1080         SLIST_INSERT_HEAD(lhp, tptr, next);
1081         *rslt = tptr;
1082         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, path, "created tstate\n");
1083         return (CAM_REQ_CMP);
1084 }
1085
1086 static ISP_INLINE void
1087 destroy_lun_state(ispsoftc_t *isp, tstate_t *tptr)
1088 {
1089         struct tslist *lhp;
1090         KASSERT((tptr->hold == 0), ("tptr still held"));
1091         ISP_GET_PC_ADDR(isp, xpt_path_path_id(tptr->owner), lun_hash[LUN_HASH_FUNC(xpt_path_lun_id(tptr->owner))], lhp);
1092         SLIST_REMOVE(lhp, tptr, tstate, next);
1093         xpt_free_path(tptr->owner);
1094         kfree(tptr, M_DEVBUF);
1095 }
1096
1097 /*
1098  * Enable a lun.
1099  */
1100 static void
1101 isp_enable_lun(ispsoftc_t *isp, union ccb *ccb)
1102 {
1103         tstate_t *tptr = NULL;
1104         int bus, tm_enabled, target_role;
1105         target_id_t target;
1106         lun_id_t lun;
1107
1108         /*
1109          * We only support either a wildcard target/lun or a target ID of zero and a non-wildcard lun
1110          */
1111         bus = XS_CHANNEL(ccb);
1112         target = ccb->ccb_h.target_id;
1113         lun = ccb->ccb_h.target_lun;
1114         if (target != CAM_TARGET_WILDCARD && target != 0) {
1115                 ccb->ccb_h.status = CAM_TID_INVALID;
1116                 xpt_done(ccb);
1117                 return;
1118         }
1119         if (target == CAM_TARGET_WILDCARD && lun != CAM_LUN_WILDCARD) {
1120                 ccb->ccb_h.status = CAM_LUN_INVALID;
1121                 xpt_done(ccb);
1122                 return;
1123         }
1124
1125         if (target != CAM_TARGET_WILDCARD && lun == CAM_LUN_WILDCARD) {
1126                 ccb->ccb_h.status = CAM_LUN_INVALID;
1127                 xpt_done(ccb);
1128                 return;
1129         }
1130         if (isp->isp_dblev & ISP_LOGTDEBUG0) {
1131                 xpt_print(ccb->ccb_h.path, "enabling lun 0x%x on channel %d\n", lun, bus);
1132         }
1133
1134         /*
1135          * Wait until we're not busy with the lun enables subsystem
1136          */
1137         while (isp->isp_osinfo.tmbusy) {
1138                 isp->isp_osinfo.tmwanted = 1;
1139                 mtx_sleep(isp, &isp->isp_lock, 0, "want_isp_enable_lun", 0);
1140         }
1141         isp->isp_osinfo.tmbusy = 1;
1142
1143         /*
1144          * This is as a good a place as any to check f/w capabilities.
1145          */
1146
1147         if (IS_FC(isp)) {
1148                 if (ISP_CAP_TMODE(isp) == 0) {
1149                         xpt_print(ccb->ccb_h.path, "firmware does not support target mode\n");
1150                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
1151                         goto done;
1152                 }
1153                 /*
1154                  * We *could* handle non-SCCLUN f/w, but we'd have to
1155                  * dork with our already fragile enable/disable code.
1156                  */
1157                 if (ISP_CAP_SCCFW(isp) == 0) {
1158                         xpt_print(ccb->ccb_h.path, "firmware not SCCLUN capable\n");
1159                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
1160                         goto done;
1161                 }
1162
1163                 target_role = (FCPARAM(isp, bus)->role & ISP_ROLE_TARGET) != 0;
1164
1165         } else {
1166                 target_role = (SDPARAM(isp, bus)->role & ISP_ROLE_TARGET) != 0;
1167         }
1168
1169         /*
1170          * Create the state pointer.
1171          * It should not already exist.
1172          */
1173         tptr = get_lun_statep(isp, bus, lun);
1174         if (tptr) {
1175                 ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
1176                 goto done;
1177         }
1178         ccb->ccb_h.status = create_lun_state(isp, bus, ccb->ccb_h.path, &tptr);
1179         if (ccb->ccb_h.status != CAM_REQ_CMP) {
1180                 goto done;
1181         }
1182
1183         /*
1184          * We have a tricky maneuver to perform here.
1185          *
1186          * If target mode isn't already enabled here,
1187          * *and* our current role includes target mode,
1188          * we enable target mode here.
1189          *
1190          */
1191         ISP_GET_PC(isp, bus, tm_enabled, tm_enabled);
1192         if (tm_enabled == 0 && target_role != 0) {
1193                 if (isp_enable_target_mode(isp, bus)) {
1194                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1195                         destroy_lun_state(isp, tptr);
1196                         tptr = NULL;
1197                         goto done;
1198                 }
1199                 tm_enabled = 1;
1200         }
1201
1202         /*
1203          * Now check to see whether this bus is in target mode already.
1204          *
1205          * If not, a later role change into target mode will finish the job.
1206          */
1207         if (tm_enabled == 0) {
1208                 ISP_SET_PC(isp, bus, tm_enable_defer, 1);
1209                 ccb->ccb_h.status = CAM_REQ_CMP;
1210                 xpt_print(ccb->ccb_h.path, "Target Mode Not Enabled Yet- Lun Enables Deferred\n");
1211                 goto done;
1212         }
1213
1214         /*
1215          * Enable the lun.
1216          */
1217         ccb->ccb_h.status = isp_enable_deferred(isp, bus, lun);
1218
1219 done:
1220         if (ccb->ccb_h.status != CAM_REQ_CMP && tptr) {
1221                 destroy_lun_state(isp, tptr);
1222                 tptr = NULL;
1223         }
1224         if (tptr) {
1225                 rls_lun_statep(isp, tptr);
1226         }
1227         isp->isp_osinfo.tmbusy = 0;
1228         if (isp->isp_osinfo.tmwanted) {
1229                 isp->isp_osinfo.tmwanted = 0;
1230                 wakeup(isp);
1231         }
1232         xpt_done(ccb);
1233 }
1234
1235 static void
1236 isp_enable_deferred_luns(ispsoftc_t *isp, int bus)
1237 {
1238         /*
1239          * XXX: not entirely implemented yet
1240          */
1241         (void) isp_enable_deferred(isp, bus, 0);
1242 }
1243
1244 static uint32_t
1245 isp_enable_deferred(ispsoftc_t *isp, int bus, lun_id_t lun)
1246 {
1247         cam_status status;
1248
1249         isp_prt(isp, ISP_LOGTINFO, "%s: bus %d lun %u", __func__, bus, lun);
1250         if (IS_24XX(isp) || (IS_FC(isp) && ISP_FC_PC(isp, bus)->tm_luns_enabled)) {
1251                 status = CAM_REQ_CMP;
1252         } else {
1253                 int cmd_cnt, not_cnt;
1254
1255                 if (IS_23XX(isp)) {
1256                         cmd_cnt = DFLT_CMND_CNT;
1257                         not_cnt = DFLT_INOT_CNT;
1258                 } else {
1259                         cmd_cnt = 64;
1260                         not_cnt = 8;
1261                 }
1262                 status = CAM_REQ_INPROG;
1263                 isp->isp_osinfo.rptr = &status;
1264                 if (isp_lun_cmd(isp, RQSTYPE_ENABLE_LUN, bus, lun, DFLT_CMND_CNT, DFLT_INOT_CNT)) {
1265                         status = CAM_RESRC_UNAVAIL;
1266                 } else {
1267                         mtx_sleep(&status, &isp->isp_lock, PRIBIO, "isp_enable_deferred", 0);
1268                 }
1269                 isp->isp_osinfo.rptr = NULL;
1270         }
1271
1272         if (status == CAM_REQ_CMP) {
1273                 ISP_SET_PC(isp, bus, tm_luns_enabled, 1);
1274                 isp_prt(isp, ISP_LOGTINFO, "bus %d lun %u now enabled for target mode", bus, lun);
1275         }
1276         return (status);
1277 }
1278
1279 static void
1280 isp_disable_lun(ispsoftc_t *isp, union ccb *ccb)
1281 {
1282         tstate_t *tptr = NULL;
1283         int bus;
1284         cam_status status;
1285         target_id_t target;
1286         lun_id_t lun;
1287
1288         bus = XS_CHANNEL(ccb);
1289         target = ccb->ccb_h.target_id;
1290         lun = ccb->ccb_h.target_lun;
1291         if (target != CAM_TARGET_WILDCARD && target != 0) {
1292                 ccb->ccb_h.status = CAM_TID_INVALID;
1293                 xpt_done(ccb);
1294                 return;
1295         }
1296         if (target == CAM_TARGET_WILDCARD && lun != CAM_LUN_WILDCARD) {
1297                 ccb->ccb_h.status = CAM_LUN_INVALID;
1298                 xpt_done(ccb);
1299                 return;
1300         }
1301
1302         if (target != CAM_TARGET_WILDCARD && lun == CAM_LUN_WILDCARD) {
1303                 ccb->ccb_h.status = CAM_LUN_INVALID;
1304                 xpt_done(ccb);
1305                 return;
1306         }
1307         if (isp->isp_dblev & ISP_LOGTDEBUG0) {
1308                 xpt_print(ccb->ccb_h.path, "enabling lun 0x%x on channel %d\n", lun, bus);
1309         }
1310
1311         /*
1312          * See if we're busy disabling a lun now.
1313          */
1314         while (isp->isp_osinfo.tmbusy) {
1315                 isp->isp_osinfo.tmwanted = 1;
1316                 mtx_sleep(isp, &isp->isp_lock, PRIBIO, "want_isp_disable_lun", 0);
1317         }
1318         isp->isp_osinfo.tmbusy = 1;
1319
1320         /*
1321          * Find the state pointer.
1322          */
1323         if ((tptr = get_lun_statep(isp, bus, lun)) == NULL) {
1324                 ccb->ccb_h.status = CAM_PATH_INVALID;
1325                 goto done;
1326         }
1327
1328         /*
1329          * If we're a 24XX card, we're done.
1330          */
1331         if (IS_24XX(isp)) {
1332                 status = CAM_REQ_CMP;
1333                 goto done;
1334         }
1335
1336         /*
1337          * For SCC FW, we only deal with lun zero.
1338          */
1339         if (IS_FC(isp)) {
1340                 lun = 0;
1341         }
1342
1343         isp->isp_osinfo.rptr = &status;
1344         status = CAM_REQ_INPROG;
1345         if (isp_lun_cmd(isp, RQSTYPE_ENABLE_LUN, bus, lun, 0, 0)) {
1346                 status = CAM_RESRC_UNAVAIL;
1347         } else {
1348                 mtx_sleep(ccb, &isp->isp_lock, PRIBIO, "isp_disable_lun", 0);
1349         }
1350 done:
1351         if (status == CAM_REQ_CMP) {
1352                 xpt_print(ccb->ccb_h.path, "now disabled for target mode\n");
1353         }
1354         if (tptr) {
1355                 destroy_lun_state(isp, tptr);
1356         }
1357         isp->isp_osinfo.rptr = NULL;
1358         isp->isp_osinfo.tmbusy = 0;
1359         if (isp->isp_osinfo.tmwanted) {
1360                 isp->isp_osinfo.tmwanted = 0;
1361                 wakeup(isp);
1362         }
1363         xpt_done(ccb);
1364 }
1365
1366 static int
1367 isp_enable_target_mode(ispsoftc_t *isp, int bus)
1368 {
1369         int ct;
1370
1371         ISP_GET_PC(isp, bus, tm_enabled, ct);
1372         if (ct != 0) {
1373                 return (0);
1374         }
1375
1376         if (IS_SCSI(isp)) {
1377                 mbreg_t mbs;
1378
1379                 MBSINIT(&mbs, MBOX_ENABLE_TARGET_MODE, MBLOGALL, 0);
1380                 mbs.param[0] = MBOX_ENABLE_TARGET_MODE;
1381                 mbs.param[1] = ENABLE_TARGET_FLAG|ENABLE_TQING_FLAG;
1382                 mbs.param[2] = bus << 7;
1383                 if (isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs) < 0 || mbs.param[0] != MBOX_COMMAND_COMPLETE) {
1384                         isp_prt(isp, ISP_LOGERR, "Unable to add Target Role to Bus %d", bus);
1385                         return (EIO);
1386                 }
1387                 SDPARAM(isp, bus)->role |= ISP_ROLE_TARGET;
1388         }
1389         ISP_SET_PC(isp, bus, tm_enabled, 1);
1390         isp_prt(isp, ISP_LOGINFO, "Target Role added to Bus %d", bus);
1391         return (0);
1392 }
1393
1394 #ifdef  NEEDED
1395 static int
1396 isp_disable_target_mode(ispsoftc_t *isp, int bus)
1397 {
1398         int ct;
1399
1400         ISP_GET_PC(isp, bus, tm_enabled, ct);
1401         if (ct == 0) {
1402                 return (0);
1403         }
1404
1405         if (IS_SCSI(isp)) {
1406                 mbreg_t mbs;
1407
1408                 MBSINIT(&mbs, MBOX_ENABLE_TARGET_MODE, MBLOGALL, 0);
1409                 mbs.param[2] = bus << 7;
1410                 if (isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs) < 0 || mbs.param[0] != MBOX_COMMAND_COMPLETE) {
1411                         isp_prt(isp, ISP_LOGERR, "Unable to subtract Target Role to Bus %d", bus);
1412                         return (EIO);
1413                 }
1414                 SDPARAM(isp, bus)->role &= ~ISP_ROLE_TARGET;
1415         }
1416         ISP_SET_PC(isp, bus, tm_enabled, 0);
1417         isp_prt(isp, ISP_LOGINFO, "Target Role subtracted from Bus %d", bus);
1418         return (0);
1419 }
1420 #endif
1421
1422 static void
1423 isp_ledone(ispsoftc_t *isp, lun_entry_t *lep)
1424 {
1425         uint32_t *rptr;
1426
1427         rptr = isp->isp_osinfo.rptr;
1428         if (lep->le_status != LUN_OK) {
1429                 isp_prt(isp, ISP_LOGERR, "ENABLE/MODIFY LUN returned 0x%x", lep->le_status);
1430                 if (rptr) {
1431                         *rptr = CAM_REQ_CMP_ERR;
1432                         wakeup_one(rptr);
1433                 }
1434         } else {
1435                 if (rptr) {
1436                         *rptr = CAM_REQ_CMP;
1437                         wakeup_one(rptr);
1438                 }
1439         }
1440 }
1441
1442 static void
1443 isp_target_start_ctio(ispsoftc_t *isp, union ccb *ccb)
1444 {
1445         void *qe;
1446         tstate_t *tptr;
1447         atio_private_data_t *atp;
1448         struct ccb_scsiio *cso = &ccb->csio;
1449         uint32_t dmaresult, handle;
1450         uint8_t local[QENTRY_LEN];
1451
1452         /*
1453          * Do some sanity checks.
1454          */
1455         if (cso->dxfer_len == 0) {
1456                 if ((ccb->ccb_h.flags & CAM_SEND_STATUS) == 0) {
1457                         xpt_print(ccb->ccb_h.path, "a data transfer length of zero but no status to send is wrong\n");
1458                         ccb->ccb_h.status = CAM_REQ_INVALID;
1459                         xpt_done(ccb);
1460                         return;
1461                 }
1462         }
1463
1464         tptr = get_lun_statep(isp, XS_CHANNEL(ccb), XS_LUN(ccb));
1465         if (tptr == NULL) {
1466                 tptr = get_lun_statep(isp, XS_CHANNEL(ccb), CAM_LUN_WILDCARD);
1467                 if (tptr == NULL) {
1468                         xpt_print(ccb->ccb_h.path, "%s: [0x%x] cannot find tstate pointer in %s\n", __func__, cso->tag_id);
1469                         dump_tstates(isp, XS_CHANNEL(ccb));
1470                         ccb->ccb_h.status = CAM_DEV_NOT_THERE;
1471                         xpt_done(ccb);
1472                         return;
1473                 }
1474         }
1475
1476         atp = isp_get_atpd(isp, tptr, cso->tag_id);
1477         if (atp == NULL) {
1478                 xpt_print(ccb->ccb_h.path, "%s: [0x%x] cannot find private data adjunct\n", __func__, cso->tag_id);
1479                 isp_dump_atpd(isp, tptr);
1480                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1481                 xpt_done(ccb);
1482                 return;
1483         }
1484         if (atp->dead) {
1485                 xpt_print(ccb->ccb_h.path, "%s: [0x%x] stopping sending a CTIO for a dead command\n", __func__, cso->tag_id);
1486                 ccb->ccb_h.status = CAM_REQ_ABORTED;
1487                 xpt_done(ccb);
1488                 return;
1489         }
1490
1491         /*
1492          * Check to make sure we're still in target mode.
1493          */
1494         if ((FCPARAM(isp, XS_CHANNEL(ccb))->role & ISP_ROLE_TARGET) == 0) {
1495                 xpt_print(ccb->ccb_h.path, "%s: [0x%x] stopping sending a CTIO because we're no longer in target mode\n", __func__, cso->tag_id);
1496                 ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1497                 xpt_done(ccb);
1498                 return;
1499         }
1500
1501         /*
1502          * Get some resources
1503          */
1504         if (isp_get_pcmd(isp, ccb)) {
1505                 rls_lun_statep(isp, tptr);
1506                 xpt_print(ccb->ccb_h.path, "out of PCMDs\n");
1507                 cam_freeze_devq(ccb->ccb_h.path);
1508                 cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 250, 0);
1509                 ccb->ccb_h.status = CAM_REQUEUE_REQ;
1510                 xpt_done(ccb);
1511                 return;
1512         }
1513         qe = isp_getrqentry(isp);
1514         if (qe == NULL) {
1515                 xpt_print(ccb->ccb_h.path, rqo, __func__);
1516                 cam_freeze_devq(ccb->ccb_h.path);
1517                 cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 250, 0);
1518                 ccb->ccb_h.status = CAM_REQUEUE_REQ;
1519                 goto out;
1520         }
1521         memset(local, 0, QENTRY_LEN);
1522
1523         /*
1524          * We're either moving data or completing a command here.
1525          */
1526         if (IS_24XX(isp)) {
1527                 ct7_entry_t *cto = (ct7_entry_t *) local;
1528
1529                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
1530                 cto->ct_header.rqs_entry_count = 1;
1531                 cto->ct_header.rqs_seqno = 1;
1532                 cto->ct_nphdl = atp->nphdl;
1533                 cto->ct_rxid = atp->tag;
1534                 cto->ct_iid_lo = atp->portid;
1535                 cto->ct_iid_hi = atp->portid >> 16;
1536                 cto->ct_oxid = atp->oxid;
1537                 cto->ct_vpidx = ISP_GET_VPIDX(isp, XS_CHANNEL(ccb));
1538                 cto->ct_scsi_status = cso->scsi_status;
1539                 cto->ct_timeout = 120;
1540                 cto->ct_flags = atp->tattr << CT7_TASK_ATTR_SHIFT;
1541                 if (ccb->ccb_h.flags & CAM_SEND_STATUS) {
1542                         cto->ct_flags |= CT7_SENDSTATUS;
1543                 }
1544                 if (cso->dxfer_len == 0) {
1545                         cto->ct_flags |= CT7_FLAG_MODE1 | CT7_NO_DATA;
1546                         if ((ccb->ccb_h.flags & CAM_SEND_SENSE) != 0) {
1547                                 int m = min(cso->sense_len, sizeof (struct scsi_sense_data));
1548                                 cto->rsp.m1.ct_resplen = cto->ct_senselen = min(m, MAXRESPLEN_24XX);
1549                                 memcpy(cto->rsp.m1.ct_resp, &cso->sense_data, cto->ct_senselen);
1550                                 cto->ct_scsi_status |= (FCP_SNSLEN_VALID << 8);
1551                         }
1552                 } else {
1553                         cto->ct_flags |= CT7_FLAG_MODE0;
1554                         if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1555                                 cto->ct_flags |= CT7_DATA_IN;
1556                         } else {
1557                                 cto->ct_flags |= CT7_DATA_OUT;
1558                         }
1559                         cto->rsp.m0.reloff = atp->bytes_xfered;
1560                         /*
1561                          * Don't overrun the limits placed on us
1562                          */
1563                         if (atp->bytes_xfered + cso->dxfer_len > atp->orig_datalen) {
1564                                 cso->dxfer_len = atp->orig_datalen - atp->bytes_xfered;
1565                         }
1566                         atp->last_xframt = cso->dxfer_len;
1567                         cto->rsp.m0.ct_xfrlen = cso->dxfer_len;
1568                 }
1569                 if (cto->ct_flags & CT7_SENDSTATUS) {
1570                         int lvl = (cso->scsi_status)? ISP_LOGTINFO : ISP_LOGTDEBUG0;
1571                         cto->ct_resid = atp->orig_datalen - (atp->bytes_xfered + cso->dxfer_len);
1572                         if (cto->ct_resid < 0) {
1573                                 cto->ct_scsi_status |= (FCP_RESID_OVERFLOW << 8);
1574                         } else if (cto->ct_resid > 0) {
1575                                 cto->ct_scsi_status |= (FCP_RESID_UNDERFLOW << 8);
1576                         }
1577                         atp->state = ATPD_STATE_LAST_CTIO;
1578                         ISP_PATH_PRT(isp, lvl, cso->ccb_h.path, "%s: CTIO7[%x] CDB0=%x scsi status %x flags %x resid %d xfrlen %u offset %u\n", __func__, cto->ct_rxid,
1579                             atp->cdb0, cto->ct_scsi_status, cto->ct_flags, cto->ct_resid, cso->dxfer_len, atp->bytes_xfered);
1580                 } else {
1581                         cto->ct_resid = 0;
1582                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, cso->ccb_h.path, "%s: CTIO7[%x] flags %x xfrlen %u offset %u\n", __func__, cto->ct_rxid, cto->ct_flags,
1583                             cso->dxfer_len, atp->bytes_xfered);
1584                         atp->state = ATPD_STATE_CTIO;
1585                 }
1586         } else if (IS_FC(isp)) {
1587                 ct2_entry_t *cto = (ct2_entry_t *) local;
1588
1589                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO2;
1590                 cto->ct_header.rqs_entry_count = 1;
1591                 cto->ct_header.rqs_seqno = 1;
1592                 if (ISP_CAP_2KLOGIN(isp) == 0) {
1593                         ((ct2e_entry_t *)cto)->ct_iid = cso->init_id;
1594                 } else {
1595                         cto->ct_iid = cso->init_id;
1596                         if (ISP_CAP_SCCFW(isp) == 0) {
1597                                 cto->ct_lun = ccb->ccb_h.target_lun;
1598                         }
1599                 }
1600
1601
1602                 cto->ct_rxid = cso->tag_id;
1603                 if (cso->dxfer_len == 0) {
1604                         cto->ct_flags |= CT2_FLAG_MODE1 | CT2_NO_DATA | CT2_SENDSTATUS;
1605                         cto->rsp.m1.ct_scsi_status = cso->scsi_status;
1606                         cto->ct_resid = atp->orig_datalen - atp->bytes_xfered;
1607                         if (cto->ct_resid < 0) {
1608                                 cto->rsp.m1.ct_scsi_status |= CT2_DATA_OVER;
1609                         } else if (cto->ct_resid > 0) {
1610                                 cto->rsp.m1.ct_scsi_status |= CT2_DATA_UNDER;
1611                         }
1612                         if ((ccb->ccb_h.flags & CAM_SEND_SENSE) != 0) {
1613                                 int m = min(cso->sense_len, MAXRESPLEN);
1614                                 memcpy(cto->rsp.m1.ct_resp, &cso->sense_data, m);
1615                                 cto->rsp.m1.ct_senselen = m;
1616                                 cto->rsp.m1.ct_scsi_status |= CT2_SNSLEN_VALID;
1617                         } else if (cso->scsi_status == SCSI_STATUS_CHECK_COND) {
1618                                 /*
1619                                  * XXX: DEBUG
1620                                  */
1621                                 xpt_print(ccb->ccb_h.path, "CHECK CONDITION being sent without associated SENSE DATA for CDB=0x%x\n", atp->cdb0);
1622                         }
1623                 } else {
1624                         cto->ct_flags |= CT2_FLAG_MODE0;
1625                         if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1626                                 cto->ct_flags |= CT2_DATA_IN;
1627                         } else {
1628                                 cto->ct_flags |= CT2_DATA_OUT;
1629                         }
1630                         cto->ct_reloff = atp->bytes_xfered;
1631                         cto->rsp.m0.ct_xfrlen = cso->dxfer_len;
1632                         /*
1633                          * Don't overrun the limits placed on us
1634                          */
1635                         if (atp->bytes_xfered + cso->dxfer_len > atp->orig_datalen) {
1636                                 cso->dxfer_len = atp->orig_datalen - atp->bytes_xfered;
1637                         }
1638                         if ((ccb->ccb_h.flags & CAM_SEND_STATUS) != 0) {
1639                                 cto->ct_flags |= CT2_SENDSTATUS;
1640                                 cto->rsp.m0.ct_scsi_status = cso->scsi_status;
1641                                 cto->ct_resid = atp->orig_datalen - (atp->bytes_xfered + cso->dxfer_len);
1642                                 if (cto->ct_resid < 0) {
1643                                         cto->rsp.m0.ct_scsi_status |= CT2_DATA_OVER;
1644                                 } else if (cto->ct_resid > 0) {
1645                                         cto->rsp.m0.ct_scsi_status |= CT2_DATA_UNDER;
1646                                 }
1647                         } else {
1648                                 atp->last_xframt = cso->dxfer_len;
1649                         }
1650                         /*
1651                          * If we're sending data and status back together,
1652                          * we can't also send back sense data as well.
1653                          */
1654                         ccb->ccb_h.flags &= ~CAM_SEND_SENSE;
1655                 }
1656
1657                 if (cto->ct_flags & CT2_SENDSTATUS) {
1658                         int lvl = (cso->scsi_status)? ISP_LOGTINFO : ISP_LOGTDEBUG0;
1659                         cto->ct_flags |= CT2_CCINCR;
1660                         atp->state = ATPD_STATE_LAST_CTIO;
1661                         ISP_PATH_PRT(isp, lvl, cso->ccb_h.path, "%s: CTIO2[%x] CDB0=%x scsi status %x flags %x resid %d xfrlen %u offset %u\n", __func__, cto->ct_rxid,
1662                             atp->cdb0, cto->rsp.m0.ct_scsi_status, cto->ct_flags, cto->ct_resid, cso->dxfer_len, atp->bytes_xfered);
1663                 } else {
1664                         cto->ct_resid = 0;
1665                         atp->state = ATPD_STATE_CTIO;
1666                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: CTIO2[%x] flags %x xfrlen %u offset %u\n", __func__, cto->ct_rxid, cto->ct_flags,
1667                             cso->dxfer_len, atp->bytes_xfered);
1668                 }
1669                 cto->ct_timeout = 10;
1670         } else {
1671                 ct_entry_t *cto = (ct_entry_t *) local;
1672
1673                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO;
1674                 cto->ct_header.rqs_entry_count = 1;
1675                 cto->ct_header.rqs_seqno = 1;
1676                 cto->ct_iid = cso->init_id;
1677                 cto->ct_iid |= XS_CHANNEL(ccb) << 7;
1678                 cto->ct_tgt = ccb->ccb_h.target_id;
1679                 cto->ct_lun = ccb->ccb_h.target_lun;
1680                 cto->ct_fwhandle = cso->tag_id >> 16;
1681                 if (AT_HAS_TAG(cso->tag_id)) {
1682                         cto->ct_tag_val = cso->tag_id;
1683                         cto->ct_flags |= CT_TQAE;
1684                 }
1685                 if (ccb->ccb_h.flags & CAM_DIS_DISCONNECT) {
1686                         cto->ct_flags |= CT_NODISC;
1687                 }
1688                 if (cso->dxfer_len == 0) {
1689                         cto->ct_flags |= CT_NO_DATA;
1690                 } else if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1691                         cto->ct_flags |= CT_DATA_IN;
1692                 } else {
1693                         cto->ct_flags |= CT_DATA_OUT;
1694                 }
1695                 if (ccb->ccb_h.flags & CAM_SEND_STATUS) {
1696                         cto->ct_flags |= CT_SENDSTATUS|CT_CCINCR;
1697                         cto->ct_scsi_status = cso->scsi_status;
1698                         cto->ct_resid = cso->resid;
1699                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: CTIO[%x] scsi status %x resid %d tag_id %x\n", __func__,
1700                             cto->ct_fwhandle, cso->scsi_status, cso->resid, cso->tag_id);
1701                 }
1702                 ccb->ccb_h.flags &= ~CAM_SEND_SENSE;
1703                 cto->ct_timeout = 10;
1704         }
1705
1706         if (isp_allocate_xs_tgt(isp, ccb, &handle)) {
1707                 xpt_print(ccb->ccb_h.path, "No XFLIST pointers for %s\n", __func__);
1708                 ccb->ccb_h.status = CAM_REQUEUE_REQ;
1709                 goto out;
1710         }
1711
1712
1713         /*
1714          * Call the dma setup routines for this entry (and any subsequent
1715          * CTIOs) if there's data to move, and then tell the f/w it's got
1716          * new things to play with. As with isp_start's usage of DMA setup,
1717          * any swizzling is done in the machine dependent layer. Because
1718          * of this, we put the request onto the queue area first in native
1719          * format.
1720          */
1721
1722         if (IS_24XX(isp)) {
1723                 ct7_entry_t *cto = (ct7_entry_t *) local;
1724                 cto->ct_syshandle = handle;
1725         } else if (IS_FC(isp)) {
1726                 ct2_entry_t *cto = (ct2_entry_t *) local;
1727                 cto->ct_syshandle = handle;
1728         } else {
1729                 ct_entry_t *cto = (ct_entry_t *) local;
1730                 cto->ct_syshandle = handle;
1731         }
1732
1733         dmaresult = ISP_DMASETUP(isp, cso, (ispreq_t *) local);
1734         if (dmaresult == CMD_QUEUED) {
1735                 isp->isp_nactive++;
1736                 ccb->ccb_h.status |= CAM_SIM_QUEUED;
1737                 rls_lun_statep(isp, tptr);
1738                 return;
1739         }
1740         if (dmaresult == CMD_EAGAIN) {
1741                 ccb->ccb_h.status = CAM_REQUEUE_REQ;
1742         } else {
1743                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1744         }
1745         isp_destroy_tgt_handle(isp, handle);
1746 out:
1747         rls_lun_statep(isp, tptr);
1748         isp_free_pcmd(isp, ccb);
1749         xpt_done(ccb);
1750 }
1751
1752 static void
1753 isp_refire_putback_atio(void *arg)
1754 {
1755         union ccb *ccb = arg;
1756         ispsoftc_t *isp = XS_ISP(ccb);
1757         ISP_LOCK(isp);
1758         isp_target_putback_atio(ccb);
1759         ISP_UNLOCK(isp);
1760 }
1761
1762 static void
1763 isp_target_putback_atio(union ccb *ccb)
1764 {
1765         ispsoftc_t *isp;
1766         struct ccb_scsiio *cso;
1767         void *qe;
1768
1769         isp = XS_ISP(ccb);
1770
1771         qe = isp_getrqentry(isp);
1772         if (qe == NULL) {
1773                 xpt_print(ccb->ccb_h.path, rqo, __func__);
1774                 (void) timeout(isp_refire_putback_atio, ccb, 10);
1775                 return;
1776         }
1777         memset(qe, 0, QENTRY_LEN);
1778         cso = &ccb->csio;
1779         if (IS_FC(isp)) {
1780                 at2_entry_t local, *at = &local;
1781                 ISP_MEMZERO(at, sizeof (at2_entry_t));
1782                 at->at_header.rqs_entry_type = RQSTYPE_ATIO2;
1783                 at->at_header.rqs_entry_count = 1;
1784                 if (ISP_CAP_SCCFW(isp)) {
1785                         at->at_scclun = (uint16_t) ccb->ccb_h.target_lun;
1786                 } else {
1787                         at->at_lun = (uint8_t) ccb->ccb_h.target_lun;
1788                 }
1789                 at->at_status = CT_OK;
1790                 at->at_rxid = cso->tag_id;
1791                 at->at_iid = cso->ccb_h.target_id;
1792                 isp_put_atio2(isp, at, qe);
1793         } else {
1794                 at_entry_t local, *at = &local;
1795                 ISP_MEMZERO(at, sizeof (at_entry_t));
1796                 at->at_header.rqs_entry_type = RQSTYPE_ATIO;
1797                 at->at_header.rqs_entry_count = 1;
1798                 at->at_iid = cso->init_id;
1799                 at->at_iid |= XS_CHANNEL(ccb) << 7;
1800                 at->at_tgt = cso->ccb_h.target_id;
1801                 at->at_lun = cso->ccb_h.target_lun;
1802                 at->at_status = CT_OK;
1803                 at->at_tag_val = AT_GET_TAG(cso->tag_id);
1804                 at->at_handle = AT_GET_HANDLE(cso->tag_id);
1805                 isp_put_atio(isp, at, qe);
1806         }
1807         ISP_TDQE(isp, "isp_target_putback_atio", isp->isp_reqidx, qe);
1808         ISP_SYNC_REQUEST(isp);
1809         isp_complete_ctio(ccb);
1810 }
1811
1812 static void
1813 isp_complete_ctio(union ccb *ccb)
1814 {
1815         if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG) {
1816                 ccb->ccb_h.status |= CAM_REQ_CMP;
1817         }
1818         ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1819         isp_free_pcmd(XS_ISP(ccb), ccb);
1820         xpt_done(ccb);
1821 }
1822
1823 /*
1824  * Handle ATIO stuff that the generic code can't.
1825  * This means handling CDBs.
1826  */
1827
1828 static void
1829 isp_handle_platform_atio(ispsoftc_t *isp, at_entry_t *aep)
1830 {
1831         tstate_t *tptr;
1832         int status, bus;
1833         struct ccb_accept_tio *atiop;
1834         atio_private_data_t *atp;
1835
1836         /*
1837          * The firmware status (except for the QLTM_SVALID bit)
1838          * indicates why this ATIO was sent to us.
1839          *
1840          * If QLTM_SVALID is set, the firmware has recommended Sense Data.
1841          *
1842          * If the DISCONNECTS DISABLED bit is set in the flags field,
1843          * we're still connected on the SCSI bus.
1844          */
1845         status = aep->at_status;
1846         if ((status & ~QLTM_SVALID) == AT_PHASE_ERROR) {
1847                 /*
1848                  * Bus Phase Sequence error. We should have sense data
1849                  * suggested by the f/w. I'm not sure quite yet what
1850                  * to do about this for CAM.
1851                  */
1852                 isp_prt(isp, ISP_LOGWARN, "PHASE ERROR");
1853                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
1854                 return;
1855         }
1856         if ((status & ~QLTM_SVALID) != AT_CDB) {
1857                 isp_prt(isp, ISP_LOGWARN, "bad atio (0x%x) leaked to platform", status);
1858                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
1859                 return;
1860         }
1861
1862         bus = GET_BUS_VAL(aep->at_iid);
1863         tptr = get_lun_statep(isp, bus, aep->at_lun);
1864         if (tptr == NULL) {
1865                 tptr = get_lun_statep(isp, bus, CAM_LUN_WILDCARD);
1866                 if (tptr == NULL) {
1867                         /*
1868                          * Because we can't autofeed sense data back with
1869                          * a command for parallel SCSI, we can't give back
1870                          * a CHECK CONDITION. We'll give back a BUSY status
1871                          * instead. This works out okay because the only
1872                          * time we should, in fact, get this, is in the
1873                          * case that somebody configured us without the
1874                          * blackhole driver, so they get what they deserve.
1875                          */
1876                         isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
1877                         return;
1878                 }
1879         }
1880
1881         atp = isp_get_atpd(isp, tptr, 0);
1882         atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
1883         if (atiop == NULL || atp == NULL) {
1884                 /*
1885                  * Because we can't autofeed sense data back with
1886                  * a command for parallel SCSI, we can't give back
1887                  * a CHECK CONDITION. We'll give back a QUEUE FULL status
1888                  * instead. This works out okay because the only time we
1889                  * should, in fact, get this, is in the case that we've
1890                  * run out of ATIOS.
1891                  */
1892                 xpt_print(tptr->owner, "no %s for lun %d from initiator %d\n", (atp == NULL && atiop == NULL)? "ATIOs *or* ATPS" :
1893                     ((atp == NULL)? "ATPs" : "ATIOs"), aep->at_lun, aep->at_iid);
1894                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
1895                 if (atp) {
1896                         isp_put_atpd(isp, tptr, atp);
1897                 }
1898                 rls_lun_statep(isp, tptr);
1899                 return;
1900         }
1901         atp->tag = aep->at_tag_val;
1902         if (atp->tag == 0) {
1903                 atp->tag = ~0;
1904         }
1905         atp->state = ATPD_STATE_ATIO;
1906         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
1907         tptr->atio_count--;
1908         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, atiop->ccb_h.path, "Take FREE ATIO count now %d\n", tptr->atio_count);
1909         atiop->ccb_h.target_id = aep->at_tgt;
1910         atiop->ccb_h.target_lun = aep->at_lun;
1911         if (aep->at_flags & AT_NODISC) {
1912                 atiop->ccb_h.flags = CAM_DIS_DISCONNECT;
1913         } else {
1914                 atiop->ccb_h.flags = 0;
1915         }
1916
1917         if (status & QLTM_SVALID) {
1918                 size_t amt = ISP_MIN(QLTM_SENSELEN, sizeof (atiop->sense_data));
1919                 atiop->sense_len = amt;
1920                 ISP_MEMCPY(&atiop->sense_data, aep->at_sense, amt);
1921         } else {
1922                 atiop->sense_len = 0;
1923         }
1924
1925         atiop->init_id = GET_IID_VAL(aep->at_iid);
1926         atiop->cdb_len = aep->at_cdblen;
1927         ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cdb, aep->at_cdblen);
1928         atiop->ccb_h.status = CAM_CDB_RECVD;
1929         /*
1930          * Construct a tag 'id' based upon tag value (which may be 0..255)
1931          * and the handle (which we have to preserve).
1932          */
1933         atiop->tag_id = atp->tag;
1934         if (aep->at_flags & AT_TQAE) {
1935                 atiop->tag_action = aep->at_tag_type;
1936                 atiop->ccb_h.status |= CAM_TAG_ACTION_VALID;
1937         }
1938         atp->orig_datalen = 0;
1939         atp->bytes_xfered = 0;
1940         atp->last_xframt = 0;
1941         atp->lun = aep->at_lun;
1942         atp->nphdl = aep->at_iid;
1943         atp->portid = PORT_NONE;
1944         atp->oxid = 0;
1945         atp->cdb0 = atiop->cdb_io.cdb_bytes[0];
1946         atp->tattr = aep->at_tag_type;
1947         atp->state = ATPD_STATE_CAM;
1948         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, tptr->owner, "ATIO[%x] CDB=0x%x lun %d\n", aep->at_tag_val, atp->cdb0, atp->lun);
1949         rls_lun_statep(isp, tptr);
1950 }
1951
1952 static void
1953 isp_handle_platform_atio2(ispsoftc_t *isp, at2_entry_t *aep)
1954 {
1955         lun_id_t lun;
1956         fcportdb_t *lp;
1957         tstate_t *tptr;
1958         struct ccb_accept_tio *atiop;
1959         uint16_t nphdl;
1960         atio_private_data_t *atp;
1961         inot_private_data_t *ntp;
1962
1963         /*
1964          * The firmware status (except for the QLTM_SVALID bit)
1965          * indicates why this ATIO was sent to us.
1966          *
1967          * If QLTM_SVALID is set, the firmware has recommended Sense Data.
1968          */
1969         if ((aep->at_status & ~QLTM_SVALID) != AT_CDB) {
1970                 isp_prt(isp, ISP_LOGWARN, "bogus atio (0x%x) leaked to platform", aep->at_status);
1971                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
1972                 return;
1973         }
1974
1975         if (ISP_CAP_SCCFW(isp)) {
1976                 lun = aep->at_scclun;
1977         } else {
1978                 lun = aep->at_lun;
1979         }
1980         if (ISP_CAP_2KLOGIN(isp)) {
1981                 nphdl = ((at2e_entry_t *)aep)->at_iid;
1982         } else {
1983                 nphdl = aep->at_iid;
1984         }
1985         tptr = get_lun_statep(isp, 0, lun);
1986         if (tptr == NULL) {
1987                 tptr = get_lun_statep(isp, 0, CAM_LUN_WILDCARD);
1988                 if (tptr == NULL) {
1989                         isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] no state pointer for lun %d", aep->at_rxid, lun);
1990                         isp_endcmd(isp, aep, SCSI_STATUS_CHECK_COND | ECMD_SVALID | (0x5 << 12) | (0x25 << 16), 0);
1991                         return;
1992                 }
1993         }
1994
1995         /*
1996          * Start any commands pending resources first.
1997          */
1998         if (tptr->restart_queue) {
1999                 inot_private_data_t *restart_queue = tptr->restart_queue;
2000                 tptr->restart_queue = NULL;
2001                 while (restart_queue) {
2002                         ntp = restart_queue;
2003                         restart_queue = ntp->rd.nt.nt_hba;
2004                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at2_entry_t *)ntp->rd.data)->at_rxid);
2005                         isp_handle_platform_atio2(isp, (at2_entry_t *) ntp->rd.data);
2006                         isp_put_ntpd(isp, tptr, ntp);
2007                         /*
2008                          * If a recursion caused the restart queue to start to fill again,
2009                          * stop and splice the new list on top of the old list and restore
2010                          * it and go to noresrc.
2011                          */
2012                         if (tptr->restart_queue) {
2013                                 ntp = tptr->restart_queue;
2014                                 tptr->restart_queue = restart_queue;
2015                                 while (restart_queue->rd.nt.nt_hba) {
2016                                         restart_queue = restart_queue->rd.nt.nt_hba;
2017                                 }
2018                                 restart_queue->rd.nt.nt_hba = ntp;
2019                                 goto noresrc;
2020                         }
2021                 }
2022         }
2023
2024         atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
2025         if (atiop == NULL) {
2026                 goto noresrc;
2027         }
2028
2029         atp = isp_get_atpd(isp, tptr, 0);
2030         if (atp == NULL) {
2031                 goto noresrc;
2032         }
2033
2034         atp->tag = aep->at_rxid;
2035         atp->state = ATPD_STATE_ATIO;
2036         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
2037         tptr->atio_count--;
2038         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, atiop->ccb_h.path, "Take FREE ATIO count now %d\n", tptr->atio_count);
2039         atiop->ccb_h.target_id = FCPARAM(isp, 0)->isp_loopid;
2040         atiop->ccb_h.target_lun = lun;
2041
2042         /*
2043          * We don't get 'suggested' sense data as we do with SCSI cards.
2044          */
2045         atiop->sense_len = 0;
2046         if (ISP_CAP_2KLOGIN(isp)) {
2047                 /*
2048                  * NB: We could not possibly have 2K logins if we
2049                  * NB: also did not have SCC FW.
2050                  */
2051                 atiop->init_id = ((at2e_entry_t *)aep)->at_iid;
2052         } else {
2053                 atiop->init_id = aep->at_iid;
2054         }
2055
2056         /*
2057          * If we're not in the port database, add ourselves.
2058          */
2059         if (!IS_2100(isp) && isp_find_pdb_by_loopid(isp, 0, atiop->init_id, &lp) == 0) {
2060                 uint64_t iid =
2061                         (((uint64_t) aep->at_wwpn[0]) << 48) |
2062                         (((uint64_t) aep->at_wwpn[1]) << 32) |
2063                         (((uint64_t) aep->at_wwpn[2]) << 16) |
2064                         (((uint64_t) aep->at_wwpn[3]) <<  0);
2065                 /*
2066                  * However, make sure we delete ourselves if otherwise
2067                  * we were there but at a different loop id.
2068                  */
2069                 if (isp_find_pdb_by_wwn(isp, 0, iid, &lp)) {
2070                         isp_del_wwn_entry(isp, 0, iid, lp->handle, lp->portid);
2071                 }
2072                 isp_add_wwn_entry(isp, 0, iid, atiop->init_id, PORT_ANY);
2073         }
2074         atiop->cdb_len = ATIO2_CDBLEN;
2075         ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cdb, ATIO2_CDBLEN);
2076         atiop->ccb_h.status = CAM_CDB_RECVD;
2077         atiop->tag_id = atp->tag;
2078         switch (aep->at_taskflags & ATIO2_TC_ATTR_MASK) {
2079         case ATIO2_TC_ATTR_SIMPLEQ:
2080                 atiop->ccb_h.flags = CAM_TAG_ACTION_VALID;
2081                 atiop->tag_action = MSG_SIMPLE_Q_TAG;
2082                 break;
2083         case ATIO2_TC_ATTR_HEADOFQ:
2084                 atiop->ccb_h.flags = CAM_TAG_ACTION_VALID;
2085                 atiop->tag_action = MSG_HEAD_OF_Q_TAG;
2086                 break;
2087         case ATIO2_TC_ATTR_ORDERED:
2088                 atiop->ccb_h.flags = CAM_TAG_ACTION_VALID;
2089                 atiop->tag_action = MSG_ORDERED_Q_TAG;
2090                 break;
2091         case ATIO2_TC_ATTR_ACAQ:                /* ?? */
2092         case ATIO2_TC_ATTR_UNTAGGED:
2093         default:
2094                 atiop->tag_action = 0;
2095                 break;
2096         }
2097
2098         atp->orig_datalen = aep->at_datalen;
2099         atp->bytes_xfered = 0;
2100         atp->last_xframt = 0;
2101         atp->lun = lun;
2102         atp->nphdl = atiop->init_id;
2103         atp->sid = PORT_ANY;
2104         atp->oxid = aep->at_oxid;
2105         atp->cdb0 = aep->at_cdb[0];
2106         atp->tattr = aep->at_taskflags & ATIO2_TC_ATTR_MASK;
2107         atp->state = ATPD_STATE_CAM;
2108         xpt_done((union ccb *)atiop);
2109         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, tptr->owner, "ATIO2[%x] CDB=0x%x lun %d datalen %u\n", aep->at_rxid, atp->cdb0, lun, atp->orig_datalen);
2110         rls_lun_statep(isp, tptr);
2111         return;
2112 noresrc:
2113         ntp = isp_get_ntpd(isp, tptr);
2114         if (ntp == NULL) {
2115                 rls_lun_statep(isp, tptr);
2116                 isp_endcmd(isp, aep, nphdl, 0, SCSI_STATUS_BUSY, 0);
2117                 return;
2118         }
2119         memcpy(ntp->rd.data, aep, QENTRY_LEN);
2120         ntp->rd.nt.nt_hba = tptr->restart_queue;
2121         tptr->restart_queue = ntp;
2122         rls_lun_statep(isp, tptr);
2123 }
2124
2125 static void
2126 isp_handle_platform_atio7(ispsoftc_t *isp, at7_entry_t *aep)
2127 {
2128         int cdbxlen;
2129         uint16_t lun, chan, nphdl = NIL_HANDLE;
2130         uint32_t did, sid;
2131         uint64_t wwn = INI_NONE;
2132         fcportdb_t *lp;
2133         tstate_t *tptr;
2134         struct ccb_accept_tio *atiop;
2135         atio_private_data_t *atp = NULL;
2136         inot_private_data_t *ntp;
2137
2138         did = (aep->at_hdr.d_id[0] << 16) | (aep->at_hdr.d_id[1] << 8) | aep->at_hdr.d_id[2];
2139         sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2];
2140         lun = (aep->at_cmnd.fcp_cmnd_lun[0] << 8) | aep->at_cmnd.fcp_cmnd_lun[1];
2141
2142         /*
2143          * Find the N-port handle, and Virtual Port Index for this command.
2144          *
2145          * If we can't, we're somewhat in trouble because we can't actually respond w/o that information.
2146          * We also, as a matter of course, need to know the WWN of the initiator too.
2147          */
2148         if (ISP_CAP_MULTI_ID(isp)) {
2149                 /*
2150                  * Find the right channel based upon D_ID
2151                  */
2152                 isp_find_chan_by_did(isp, did, &chan);
2153
2154                 if (chan == ISP_NOCHAN) {
2155                         NANOTIME_T now;
2156
2157                         /*
2158                          * If we don't recognizer our own D_DID, terminate the exchange, unless we're within 2 seconds of startup
2159                          * It's a bit tricky here as we need to stash this command *somewhere*.
2160                          */
2161                         GET_NANOTIME(&now);
2162                         if (NANOTIME_SUB(&isp->isp_init_time, &now) > 2000000000ULL) {
2163                                 isp_prt(isp, ISP_LOGWARN, "%s: [RX_ID 0x%x] D_ID %x not found on any channel- dropping", __func__, aep->at_rxid, did);
2164                                 isp_endcmd(isp, aep, NIL_HANDLE, ISP_NOCHAN, ECMD_TERMINATE, 0);
2165                                 return;
2166                         }
2167                         tptr = get_lun_statep(isp, 0, 0);
2168                         if (tptr == NULL) {
2169                                 tptr = get_lun_statep(isp, 0, CAM_LUN_WILDCARD);
2170                                 if (tptr == NULL) {
2171                                         isp_prt(isp, ISP_LOGWARN, "%s: [RX_ID 0x%x] D_ID %x not found on any channel and no tptr- dropping", __func__, aep->at_rxid, did);
2172                                         isp_endcmd(isp, aep, NIL_HANDLE, ISP_NOCHAN, ECMD_TERMINATE, 0);
2173                                         return;
2174                                 }
2175                         }
2176                         isp_prt(isp, ISP_LOGWARN, "%s: [RX_ID 0x%x] D_ID %x not found on any channel- deferring", __func__, aep->at_rxid, did);
2177                         goto noresrc;
2178                 }
2179                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: [RX_ID 0x%x] D_ID 0x%06x found on Chan %d for S_ID 0x%06x", __func__, aep->at_rxid, did, chan, sid);
2180         } else {
2181                 chan = 0;
2182         }
2183
2184         /*
2185          * Find the PDB entry for this initiator
2186          */
2187         if (isp_find_pdb_by_sid(isp, chan, sid, &lp) == 0) {
2188                 /*
2189                  * If we're not in the port database terminate the exchange.
2190                  */
2191                 isp_prt(isp, ISP_LOGTINFO, "%s: [RX_ID 0x%x] D_ID 0x%06x found on Chan %d for S_ID 0x%06x wasn't in PDB already",
2192                     __func__, aep->at_rxid, did, chan, sid);
2193                 isp_endcmd(isp, aep, NIL_HANDLE, chan, ECMD_TERMINATE, 0);
2194                 return;
2195         }
2196         nphdl = lp->handle;
2197         wwn = lp->port_wwn;
2198
2199         /*
2200          * Get the tstate pointer
2201          */
2202         tptr = get_lun_statep(isp, chan, lun);
2203         if (tptr == NULL) {
2204                 tptr = get_lun_statep(isp, chan, CAM_LUN_WILDCARD);
2205                 if (tptr == NULL) {
2206                         isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] no state pointer for lun %d or wildcard", aep->at_rxid, lun);
2207                         isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_CHECK_COND | ECMD_SVALID | (0x5 << 12) | (0x25 << 16), 0);
2208                         return;
2209                 }
2210         }
2211
2212         /*
2213          * Start any commands pending resources first.
2214          */
2215         if (tptr->restart_queue) {
2216                 inot_private_data_t *restart_queue = tptr->restart_queue;
2217                 tptr->restart_queue = NULL;
2218                 while (restart_queue) {
2219                         ntp = restart_queue;
2220                         restart_queue = ntp->rd.nt.nt_hba;
2221                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at7_entry_t *)ntp->rd.data)->at_rxid);
2222                         isp_handle_platform_atio7(isp, (at7_entry_t *) ntp->rd.data);
2223                         isp_put_ntpd(isp, tptr, ntp);
2224                         /*
2225                          * If a recursion caused the restart queue to start to fill again,
2226                          * stop and splice the new list on top of the old list and restore
2227                          * it and go to noresrc.
2228                          */
2229                         if (tptr->restart_queue) {
2230                                 if (restart_queue) {
2231                                         ntp = tptr->restart_queue;
2232                                         tptr->restart_queue = restart_queue;
2233                                         while (restart_queue->rd.nt.nt_hba) {
2234                                                 restart_queue = restart_queue->rd.nt.nt_hba;
2235                                         }
2236                                         restart_queue->rd.nt.nt_hba = ntp;
2237                                 }
2238                                 goto noresrc;
2239                         }
2240                 }
2241         }
2242
2243         /*
2244          * If the f/w is out of resources, just send a BUSY status back.
2245          */
2246         if (aep->at_rxid == AT7_NORESRC_RXID) {
2247                 rls_lun_statep(isp, tptr);
2248                 isp_endcmd(isp, aep, nphdl, chan, SCSI_BUSY, 0);
2249                 return;
2250         }
2251
2252         /*
2253          * If we're out of resources, just send a BUSY status back.
2254          */
2255         atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
2256         if (atiop == NULL) {
2257                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atios", aep->at_rxid);
2258                 goto noresrc;
2259         }
2260
2261         atp = isp_get_atpd(isp, tptr, 0);
2262         if (atp == NULL) {
2263                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atps", aep->at_rxid);
2264                 goto noresrc;
2265         }
2266         if (isp_get_atpd(isp, tptr, aep->at_rxid)) {
2267                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] tag wraparound in isp_handle_platforms_atio7 (N-Port Handle 0x%04x S_ID 0x%04x OX_ID 0x%04x)\n",
2268                     aep->at_rxid, nphdl, sid, aep->at_hdr.ox_id);
2269                 /*
2270                  * It's not a "no resource" condition- but we can treat it like one
2271                  */
2272                 goto noresrc;
2273         }
2274
2275         atp->tag = aep->at_rxid;
2276         atp->state = ATPD_STATE_ATIO;
2277         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
2278         tptr->atio_count--;
2279         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, atiop->ccb_h.path, "Take FREE ATIO count now %d\n", tptr->atio_count);
2280         atiop->init_id = nphdl;
2281         atiop->ccb_h.target_id = FCPARAM(isp, chan)->isp_loopid;
2282         atiop->ccb_h.target_lun = lun;
2283         atiop->sense_len = 0;
2284         cdbxlen = aep->at_cmnd.fcp_cmnd_alen_datadir >> FCP_CMND_ADDTL_CDBLEN_SHIFT;
2285         if (cdbxlen) {
2286                 isp_prt(isp, ISP_LOGWARN, "additional CDBLEN ignored");
2287         }
2288         cdbxlen = sizeof (aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb);
2289         ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb, cdbxlen);
2290         atiop->cdb_len = cdbxlen;
2291         atiop->ccb_h.status = CAM_CDB_RECVD;
2292         atiop->tag_id = atp->tag;
2293         switch (aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK) {
2294         case FCP_CMND_TASK_ATTR_SIMPLE:
2295                 atiop->ccb_h.flags = CAM_TAG_ACTION_VALID;
2296                 atiop->tag_action = MSG_SIMPLE_Q_TAG;
2297                 break;
2298         case FCP_CMND_TASK_ATTR_HEAD:
2299                 atiop->ccb_h.flags = CAM_TAG_ACTION_VALID;
2300                 atiop->tag_action = MSG_HEAD_OF_Q_TAG;
2301                 break;
2302         case FCP_CMND_TASK_ATTR_ORDERED:
2303                 atiop->ccb_h.flags = CAM_TAG_ACTION_VALID;
2304                 atiop->tag_action = MSG_ORDERED_Q_TAG;
2305                 break;
2306         default:
2307                 /* FALLTHROUGH */
2308         case FCP_CMND_TASK_ATTR_ACA:
2309         case FCP_CMND_TASK_ATTR_UNTAGGED:
2310                 atiop->tag_action = 0;
2311                 break;
2312         }
2313         atp->orig_datalen = aep->at_cmnd.cdb_dl.sf.fcp_cmnd_dl;
2314         atp->bytes_xfered = 0;
2315         atp->last_xframt = 0;
2316         atp->lun = lun;
2317         atp->nphdl = nphdl;
2318         atp->portid = sid;
2319         atp->oxid = aep->at_hdr.ox_id;
2320         atp->cdb0 = atiop->cdb_io.cdb_bytes[0];
2321         atp->tattr = aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK;
2322         atp->state = ATPD_STATE_CAM;
2323         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, tptr->owner, "ATIO7[%x] CDB=0x%x lun %d datalen %u\n", aep->at_rxid, atp->cdb0, lun, atp->orig_datalen);
2324         xpt_done((union ccb *)atiop);
2325         rls_lun_statep(isp, tptr);
2326         return;
2327 noresrc:
2328         if (atp) {
2329                 isp_put_atpd(isp, tptr, atp);
2330         }
2331         ntp = isp_get_ntpd(isp, tptr);
2332         if (ntp == NULL) {
2333                 rls_lun_statep(isp, tptr);
2334                 isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_BUSY, 0);
2335                 return;
2336         }
2337         memcpy(ntp->rd.data, aep, QENTRY_LEN);
2338         ntp->rd.nt.nt_hba = tptr->restart_queue;
2339         tptr->restart_queue = ntp;
2340         rls_lun_statep(isp, tptr);
2341 }
2342
2343 static void
2344 isp_handle_platform_ctio(ispsoftc_t *isp, void *arg)
2345 {
2346         union ccb *ccb;
2347         int sentstatus, ok, notify_cam, resid = 0;
2348         tstate_t *tptr = NULL;
2349         atio_private_data_t *atp = NULL;
2350         int bus;
2351         uint32_t tval, handle;
2352
2353         /*
2354          * CTIO handles are 16 bits.
2355          * CTIO2 and CTIO7 are 32 bits.
2356          */
2357
2358         if (IS_SCSI(isp)) {
2359                 handle = ((ct_entry_t *)arg)->ct_syshandle;
2360         } else {
2361                 handle = ((ct2_entry_t *)arg)->ct_syshandle;
2362         }
2363         ccb = isp_find_xs_tgt(isp, handle);
2364         if (ccb == NULL) {
2365                 isp_print_bytes(isp, "null ccb in isp_handle_platform_ctio", QENTRY_LEN, arg);
2366                 return;
2367         }
2368         isp_destroy_tgt_handle(isp, handle);
2369         bus = XS_CHANNEL(ccb);
2370         tptr = get_lun_statep(isp, bus, XS_LUN(ccb));
2371         if (tptr == NULL) {
2372                 tptr = get_lun_statep(isp, bus, CAM_LUN_WILDCARD);
2373         }
2374         KASSERT((tptr != NULL), ("cannot get state pointer"));
2375         if (isp->isp_nactive) {
2376                 isp->isp_nactive++;
2377         }
2378         if (IS_24XX(isp)) {
2379                 ct7_entry_t *ct = arg;
2380
2381                 atp = isp_get_atpd(isp, tptr, ct->ct_rxid);
2382                 if (atp == NULL) {
2383                         rls_lun_statep(isp, tptr);
2384                         isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x after I/O", __func__, ct->ct_rxid);
2385                         return;
2386                 }
2387
2388                 sentstatus = ct->ct_flags & CT7_SENDSTATUS;
2389                 ok = (ct->ct_nphdl == CT7_OK);
2390                 if (ok && sentstatus && (ccb->ccb_h.flags & CAM_SEND_SENSE)) {
2391                         ccb->ccb_h.status |= CAM_SENT_SENSE;
2392                 }
2393                 notify_cam = ct->ct_header.rqs_seqno & 0x1;
2394                 if ((ct->ct_flags & CT7_DATAMASK) != CT7_NO_DATA) {
2395                         resid = ct->ct_resid;
2396                         atp->bytes_xfered += (atp->last_xframt - resid);
2397                         atp->last_xframt = 0;
2398                 }
2399                 if (ct->ct_nphdl == CT_HBA_RESET) {
2400                         ok = 0;
2401                         notify_cam = 1;
2402                         sentstatus = 1;
2403                         ccb->ccb_h.status |= CAM_UNREC_HBA_ERROR;
2404                 } else if (!ok) {
2405                         ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
2406                 }
2407                 tval = atp->tag;
2408                 isp_prt(isp, ok? ISP_LOGTDEBUG0 : ISP_LOGWARN, "%s: CTIO7[%x] sts 0x%x flg 0x%x sns %d resid %d %s", __func__,
2409                     ct->ct_rxid, ct->ct_nphdl, ct->ct_flags, (ccb->ccb_h.status & CAM_SENT_SENSE) != 0, resid, sentstatus? "FIN" : "MID");
2410                 atp->state = ATPD_STATE_PDON; /* XXX: should really come after isp_complete_ctio */
2411         } else if (IS_FC(isp)) {
2412                 ct2_entry_t *ct = arg;
2413
2414                 atp = isp_get_atpd(isp, tptr, ct->ct_rxid);
2415                 if (atp == NULL) {
2416                         rls_lun_statep(isp, tptr);
2417                         isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x after I/O", __func__, ct->ct_rxid);
2418                         return;
2419                 }
2420                 sentstatus = ct->ct_flags & CT2_SENDSTATUS;
2421                 ok = (ct->ct_status & ~QLTM_SVALID) == CT_OK;
2422                 if (ok && sentstatus && (ccb->ccb_h.flags & CAM_SEND_SENSE)) {
2423                         ccb->ccb_h.status |= CAM_SENT_SENSE;
2424                 }
2425                 notify_cam = ct->ct_header.rqs_seqno & 0x1;
2426                 if ((ct->ct_flags & CT2_DATAMASK) != CT2_NO_DATA) {
2427                         resid = ct->ct_resid;
2428                         atp->bytes_xfered += (atp->last_xframt - resid);
2429                         atp->last_xframt = 0;
2430                 }
2431                 if (ct->ct_status == CT_HBA_RESET) {
2432                         ok = 0;
2433                         notify_cam = 1;
2434                         sentstatus = 1;
2435                         ccb->ccb_h.status |= CAM_UNREC_HBA_ERROR;
2436                 } else if (!ok) {
2437                         ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
2438                 }
2439                 isp_prt(isp, ok? ISP_LOGTDEBUG0 : ISP_LOGWARN, "%s: CTIO2[%x] sts 0x%x flg 0x%x sns %d resid %d %s", __func__,
2440                     ct->ct_rxid, ct->ct_status, ct->ct_flags, (ccb->ccb_h.status & CAM_SENT_SENSE) != 0, resid, sentstatus? "FIN" : "MID");
2441                 tval = atp->tag;
2442                 atp->state = ATPD_STATE_PDON; /* XXX: should really come after isp_complete_ctio */
2443         } else {
2444                 ct_entry_t *ct = arg;
2445                 sentstatus = ct->ct_flags & CT_SENDSTATUS;
2446                 ok = (ct->ct_status  & ~QLTM_SVALID) == CT_OK;
2447                 /*
2448                  * We *ought* to be able to get back to the original ATIO
2449                  * here, but for some reason this gets lost. It's just as
2450                  * well because it's squirrelled away as part of periph
2451                  * private data.
2452                  *
2453                  * We can live without it as long as we continue to use
2454                  * the auto-replenish feature for CTIOs.
2455                  */
2456                 notify_cam = ct->ct_header.rqs_seqno & 0x1;
2457                 if (ct->ct_status == (CT_HBA_RESET & 0xff)) {
2458                         ok = 0;
2459                         notify_cam = 1;
2460                         sentstatus = 1;
2461                         ccb->ccb_h.status |= CAM_UNREC_HBA_ERROR;
2462                 } else if (!ok) {
2463                         ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
2464                 } else if (ct->ct_status & QLTM_SVALID) {
2465                         char *sp = (char *)ct;
2466                         sp += CTIO_SENSE_OFFSET;
2467                         ccb->csio.sense_len = min(sizeof (ccb->csio.sense_data), QLTM_SENSELEN);
2468                         ISP_MEMCPY(&ccb->csio.sense_data, sp, ccb->csio.sense_len);
2469                         ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2470                 }
2471                 if ((ct->ct_flags & CT_DATAMASK) != CT_NO_DATA) {
2472                         resid = ct->ct_resid;
2473                 }
2474                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO[%x] tag %x S_ID 0x%x lun %d sts %x flg %x resid %d %s", __func__,
2475                     ct->ct_fwhandle, ct->ct_tag_val, ct->ct_iid, ct->ct_lun, ct->ct_status, ct->ct_flags, resid, sentstatus? "FIN" : "MID");
2476                 tval = ct->ct_fwhandle;
2477         }
2478         ccb->csio.resid += resid;
2479
2480         /*
2481          * We're here either because intermediate data transfers are done
2482          * and/or the final status CTIO (which may have joined with a
2483          * Data Transfer) is done.
2484          *
2485          * In any case, for this platform, the upper layers figure out
2486          * what to do next, so all we do here is collect status and
2487          * pass information along. Any DMA handles have already been
2488          * freed.
2489          */
2490         if (notify_cam == 0) {
2491                 isp_prt(isp, ISP_LOGTDEBUG0, "  INTER CTIO[0x%x] done", tval);
2492                 return;
2493         }
2494         if (tptr) {
2495                 rls_lun_statep(isp, tptr);
2496         }
2497         isp_prt(isp, ISP_LOGTDEBUG0, "%s CTIO[0x%x] done", (sentstatus)? "  FINAL " : "MIDTERM ", tval);
2498
2499         if (!ok && !IS_24XX(isp)) {
2500                 isp_target_putback_atio(ccb);
2501         } else {
2502                 isp_complete_ctio(ccb);
2503         }
2504 }
2505
2506 static void
2507 isp_handle_platform_notify_scsi(ispsoftc_t *isp, in_entry_t *inot)
2508 {
2509         (void) isp_notify_ack(isp, inot);
2510 }
2511
2512 static void
2513 isp_handle_platform_notify_fc(ispsoftc_t *isp, in_fcentry_t *inp)
2514 {
2515         int needack = 1;
2516         switch (inp->in_status) {
2517         case IN_PORT_LOGOUT:
2518                 /*
2519                  * XXX: Need to delete this initiator's WWN from the database
2520                  * XXX: Need to send this LOGOUT upstream
2521                  */
2522                 isp_prt(isp, ISP_LOGWARN, "port logout of S_ID 0x%x", inp->in_iid);
2523                 break;
2524         case IN_PORT_CHANGED:
2525                 isp_prt(isp, ISP_LOGWARN, "port changed for S_ID 0x%x", inp->in_iid);
2526                 break;
2527         case IN_GLOBAL_LOGO:
2528                 isp_del_all_wwn_entries(isp, 0);
2529                 isp_prt(isp, ISP_LOGINFO, "all ports logged out");
2530                 break;
2531         case IN_ABORT_TASK:
2532         {
2533                 tstate_t *tptr;
2534                 uint16_t lun;
2535                 uint32_t loopid;
2536                 uint64_t wwn;
2537                 atio_private_data_t *atp;
2538                 fcportdb_t *lp;
2539                 struct ccb_immediate_notify *inot = NULL;
2540
2541                 if (ISP_CAP_SCCFW(isp)) {
2542                         lun = inp->in_scclun;
2543                 } else {
2544                         lun = inp->in_lun;
2545                 }
2546                 if (ISP_CAP_2KLOGIN(isp)) {
2547                         loopid = ((in_fcentry_e_t *)inp)->in_iid;
2548                 } else {
2549                         loopid = inp->in_iid;
2550                 }
2551                 if (isp_find_pdb_by_loopid(isp, 0, loopid, &lp)) {
2552                         wwn = lp->port_wwn;
2553                 } else {
2554                         wwn = INI_ANY;
2555                 }
2556                 tptr = get_lun_statep(isp, 0, lun);
2557                 if (tptr == NULL) {
2558                         tptr = get_lun_statep(isp, 0, CAM_LUN_WILDCARD);
2559                         if (tptr == NULL) {
2560                                 isp_prt(isp, ISP_LOGWARN, "ABORT TASK for lun %u- but no tstate", lun);
2561                                 return;
2562                         }
2563                 }
2564                 atp = isp_get_atpd(isp, tptr, inp->in_seqid);
2565
2566                 if (atp) {
2567                         inot = (struct ccb_immediate_notify *) SLIST_FIRST(&tptr->inots);
2568                         isp_prt(isp, ISP_LOGTDEBUG0, "ABORT TASK RX_ID %x WWN 0x%016llx state %d", inp->in_seqid, (unsigned long long) wwn, atp->state);
2569                         if (inot) {
2570                                 tptr->inot_count--;
2571                                 SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle);
2572                                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, inot->ccb_h.path, "%s: Take FREE INOT count now %d\n", __func__, tptr->inot_count);
2573                         } else {
2574                                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, tptr->owner, "out of INOT structures\n");
2575                         }
2576                 } else {
2577                         ISP_PATH_PRT(isp, ISP_LOGWARN, tptr->owner, "abort task RX_ID %x from wwn 0x%016llx, state unknown\n", inp->in_seqid, wwn);
2578                 }
2579                 if (inot) {
2580                         isp_notify_t tmp, *nt = &tmp;
2581                         ISP_MEMZERO(nt, sizeof (isp_notify_t));
2582                         nt->nt_hba = isp;
2583                         nt->nt_tgt = FCPARAM(isp, 0)->isp_wwpn;
2584                         nt->nt_wwn = wwn;
2585                         nt->nt_nphdl = loopid;
2586                         nt->nt_sid = PORT_ANY;
2587                         nt->nt_did = PORT_ANY;
2588                         nt->nt_lun = lun;
2589                         nt->nt_need_ack = 1;
2590                         nt->nt_channel = 0;
2591                         nt->nt_ncode = NT_ABORT_TASK;
2592                         nt->nt_lreserved = inot;
2593                         isp_handle_platform_target_tmf(isp, nt);
2594                         needack = 0;
2595                 }
2596                 rls_lun_statep(isp, tptr);
2597                 break;
2598         }
2599         default:
2600                 break;
2601         }
2602         if (needack) {
2603                 (void) isp_notify_ack(isp, inp);
2604         }
2605 }
2606
2607 static void
2608 isp_handle_platform_notify_24xx(ispsoftc_t *isp, in_fcentry_24xx_t *inot)
2609 {
2610         uint16_t nphdl;
2611         uint32_t portid;
2612         fcportdb_t *lp;
2613         uint8_t *ptr = NULL;
2614         uint64_t wwn;
2615
2616         nphdl = inot->in_nphdl;
2617         if (nphdl != NIL_HANDLE) {
2618                 portid = inot->in_portid_hi << 16 | inot->in_portid_lo;
2619         } else {
2620                 portid = PORT_ANY;
2621         }
2622
2623         switch (inot->in_status) {
2624         case IN24XX_ELS_RCVD:
2625         {
2626                 char buf[16], *msg;
2627                 int chan = ISP_GET_VPIDX(isp, inot->in_vpidx);
2628
2629                 /*
2630                  * Note that we're just getting notification that an ELS was received
2631                  * (possibly with some associated information sent upstream). This is
2632                  * *not* the same as being given the ELS frame to accept or reject.
2633                  */
2634                 switch (inot->in_status_subcode) {
2635                 case LOGO:
2636                         msg = "LOGO";
2637                         if (ISP_FW_NEWER_THAN(isp, 4, 0, 25)) {
2638                                 ptr = (uint8_t *)inot;  /* point to unswizzled entry! */
2639                                 wwn =   (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF])   << 56) |
2640                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+1]) << 48) |
2641                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+2]) << 40) |
2642                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+3]) << 32) |
2643                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+4]) << 24) |
2644                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+5]) << 16) |
2645                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+6]) <<  8) |
2646                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+7]));
2647                         } else {
2648                                 wwn = INI_ANY;
2649                         }
2650                         isp_del_wwn_entry(isp, chan, wwn, nphdl, portid);
2651                         break;
2652                 case PRLO:
2653                         msg = "PRLO";
2654                         break;
2655                 case PLOGI:
2656                 case PRLI:
2657                         /*
2658                          * Treat PRLI the same as PLOGI and make a database entry for it.
2659                          */
2660                         if (inot->in_status_subcode == PLOGI)
2661                                 msg = "PLOGI";
2662                         else
2663                                 msg = "PRLI";
2664                         if (ISP_FW_NEWER_THAN(isp, 4, 0, 25)) {
2665                                 ptr = (uint8_t *)inot;  /* point to unswizzled entry! */
2666                                 wwn =   (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF])   << 56) |
2667                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+1]) << 48) |
2668                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+2]) << 40) |
2669                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+3]) << 32) |
2670                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+4]) << 24) |
2671                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+5]) << 16) |
2672                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+6]) <<  8) |
2673                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+7]));
2674                         } else {
2675                                 wwn = INI_NONE;
2676                         }
2677                         isp_add_wwn_entry(isp, chan, wwn, nphdl, portid);
2678                         break;
2679                 case PDISC:
2680                         msg = "PDISC";
2681                         break;
2682                 case ADISC:
2683                         msg = "ADISC";
2684                         break;
2685                 default:
2686                         ISP_SNPRINTF(buf, sizeof (buf), "ELS 0x%x", inot->in_status_subcode);
2687                         msg = buf;
2688                         break;
2689                 }
2690                 if (inot->in_flags & IN24XX_FLAG_PUREX_IOCB) {
2691                         isp_prt(isp, ISP_LOGERR, "%s Chan %d ELS N-port handle %x PortID 0x%06x marked as needing a PUREX response", msg, chan, nphdl, portid);
2692                         break;
2693                 }
2694                 isp_prt(isp, ISP_LOGTDEBUG0, "%s Chan %d ELS N-port handle %x PortID 0x%06x RX_ID 0x%x OX_ID 0x%x", msg, chan, nphdl, portid,
2695                     inot->in_rxid, inot->in_oxid);
2696                 (void) isp_notify_ack(isp, inot);
2697                 break;
2698         }
2699
2700         case IN24XX_PORT_LOGOUT:
2701                 ptr = "PORT LOGOUT";
2702                 if (isp_find_pdb_by_loopid(isp, ISP_GET_VPIDX(isp, inot->in_vpidx), nphdl, &lp)) {
2703                         isp_del_wwn_entry(isp, ISP_GET_VPIDX(isp, inot->in_vpidx), lp->port_wwn, nphdl, lp->portid);
2704                 }
2705                 /* FALLTHROUGH */
2706         case IN24XX_PORT_CHANGED:
2707                 if (ptr == NULL) {
2708                         ptr = "PORT CHANGED";
2709                 }
2710                 /* FALLTHROUGH */
2711         case IN24XX_LIP_RESET:
2712                 if (ptr == NULL) {
2713                         ptr = "LIP RESET";
2714                 }
2715                 isp_prt(isp, ISP_LOGINFO, "Chan %d %s (sub-status 0x%x) for N-port handle 0x%x", ISP_GET_VPIDX(isp, inot->in_vpidx), ptr, inot->in_status_subcode, nphdl);
2716
2717                 /*
2718                  * All subcodes here are irrelevant. What is relevant
2719                  * is that we need to terminate all active commands from
2720                  * this initiator (known by N-port handle).
2721                  */
2722                 /* XXX IMPLEMENT XXX */
2723                 (void) isp_notify_ack(isp, inot);
2724                 break;
2725
2726         case IN24XX_LINK_RESET:
2727         case IN24XX_LINK_FAILED:
2728         case IN24XX_SRR_RCVD:
2729         default:
2730                 (void) isp_notify_ack(isp, inot);
2731                 break;
2732         }
2733 }
2734
2735 static int
2736 isp_handle_platform_target_notify_ack(ispsoftc_t *isp, isp_notify_t *mp)
2737 {
2738
2739         if (isp->isp_state != ISP_RUNSTATE) {
2740                 isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) acked- h/w not ready (dropping)", mp->nt_ncode, mp->nt_lreserved != NULL);
2741                 return (0);
2742         }
2743
2744         /*
2745          * This case is for a Task Management Function, which shows up as an ATIO7 entry.
2746          */
2747         if (IS_24XX(isp) && mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ATIO) {
2748                 ct7_entry_t local, *cto = &local;
2749                 at7_entry_t *aep = (at7_entry_t *)mp->nt_lreserved;
2750                 fcportdb_t *lp;
2751                 uint32_t sid;
2752                 uint16_t nphdl;
2753
2754                 sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2];
2755                 if (isp_find_pdb_by_sid(isp, mp->nt_channel, sid, &lp)) {
2756                         nphdl = lp->handle;
2757                 } else {
2758                         nphdl = NIL_HANDLE;
2759                 }
2760                 ISP_MEMZERO(&local, sizeof (local));
2761                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
2762                 cto->ct_header.rqs_entry_count = 1;
2763                 cto->ct_nphdl = nphdl;
2764                 cto->ct_rxid = aep->at_rxid;
2765                 cto->ct_vpidx = mp->nt_channel;
2766                 cto->ct_iid_lo = sid;
2767                 cto->ct_iid_hi = sid >> 16;
2768                 cto->ct_oxid = aep->at_hdr.ox_id;
2769                 cto->ct_flags = CT7_SENDSTATUS|CT7_NOACK|CT7_NO_DATA|CT7_FLAG_MODE1;
2770                 cto->ct_flags |= (aep->at_ta_len >> 12) << CT7_TASK_ATTR_SHIFT;
2771                 return (isp_target_put_entry(isp, &local));
2772         }
2773
2774         /*
2775          * This case is for a responding to an ABTS frame
2776          */
2777         if (IS_24XX(isp) && mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) {
2778
2779                 /*
2780                  * Overload nt_need_ack here to mark whether we've terminated the associated command.
2781                  */
2782                 if (mp->nt_need_ack) {
2783                         uint8_t storage[QENTRY_LEN];
2784                         ct7_entry_t *cto = (ct7_entry_t *) storage;
2785                         abts_t *abts = (abts_t *)mp->nt_lreserved;
2786
2787                         ISP_MEMZERO(cto, sizeof (ct7_entry_t));
2788                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: [%x] terminating after ABTS received", __func__, abts->abts_rxid_task);
2789                         cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
2790                         cto->ct_header.rqs_entry_count = 1;
2791                         cto->ct_nphdl = mp->nt_nphdl;
2792                         cto->ct_rxid = abts->abts_rxid_task;
2793                         cto->ct_iid_lo = mp->nt_sid;
2794                         cto->ct_iid_hi = mp->nt_sid >> 16;
2795                         cto->ct_oxid = abts->abts_ox_id;
2796                         cto->ct_vpidx = mp->nt_channel;
2797                         cto->ct_flags = CT7_NOACK|CT7_TERMINATE;
2798                         if (isp_target_put_entry(isp, cto)) {
2799                                 return (ENOMEM);
2800                         }
2801                         mp->nt_need_ack = 0;
2802                 }
2803                 if (isp_acknak_abts(isp, mp->nt_lreserved, 0) == ENOMEM) {
2804                         return (ENOMEM);
2805                 } else {
2806                         return (0);
2807                 }
2808         }
2809
2810         /*
2811          * Handle logout cases here
2812          */
2813         if (mp->nt_ncode == NT_GLOBAL_LOGOUT) {
2814                 isp_del_all_wwn_entries(isp, mp->nt_channel);
2815         }
2816
2817         if (mp->nt_ncode == NT_LOGOUT) {
2818                 if (!IS_2100(isp) && IS_FC(isp)) {
2819                         isp_del_wwn_entries(isp, mp);
2820                 }
2821         }
2822
2823         /*
2824          * General purpose acknowledgement
2825          */
2826         if (mp->nt_need_ack) {
2827                 isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) being acked", mp->nt_ncode, mp->nt_lreserved != NULL);
2828                 return (isp_notify_ack(isp, mp->nt_lreserved));
2829         }
2830         return (0);
2831 }
2832
2833 /*
2834  * Handle task management functions.
2835  *
2836  * We show up here with a notify structure filled out.
2837  *
2838  * The nt_lreserved tag points to the original queue entry
2839  */
2840 static void
2841 isp_handle_platform_target_tmf(ispsoftc_t *isp, isp_notify_t *notify)
2842 {
2843         tstate_t *tptr;
2844         fcportdb_t *lp;
2845         struct ccb_immediate_notify *inot;
2846         inot_private_data_t *ntp = NULL;
2847         lun_id_t lun;
2848
2849         isp_prt(isp, ISP_LOGTDEBUG0, "%s: code 0x%x sid  0x%x tagval 0x%016llx chan %d lun 0x%x", __func__, notify->nt_ncode,
2850             notify->nt_sid, (unsigned long long) notify->nt_tagval, notify->nt_channel, notify->nt_lun);
2851         /*
2852          * NB: This assignment is necessary because of tricky type conversion.
2853          * XXX: This is tricky and I need to check this. If the lun isn't known
2854          * XXX: for the task management function, it does not of necessity follow
2855          * XXX: that it should go up stream to the wildcard listener.
2856          */
2857         if (notify->nt_lun == LUN_ANY) {
2858                 lun = CAM_LUN_WILDCARD;
2859         } else {
2860                 lun = notify->nt_lun;
2861         }
2862         tptr = get_lun_statep(isp, notify->nt_channel, lun);
2863         if (tptr == NULL) {
2864                 tptr = get_lun_statep(isp, notify->nt_channel, CAM_LUN_WILDCARD);
2865                 if (tptr == NULL) {
2866                         isp_prt(isp, ISP_LOGWARN, "%s: no state pointer found for chan %d lun 0x%x", __func__, notify->nt_channel, lun);
2867                         goto bad;
2868                 }
2869         }
2870         inot = (struct ccb_immediate_notify *) SLIST_FIRST(&tptr->inots);
2871         if (inot == NULL) {
2872                 isp_prt(isp, ISP_LOGWARN, "%s: out of immediate notify structures for chan %d lun 0x%x", __func__, notify->nt_channel, lun);
2873                 goto bad;
2874         }
2875
2876         if (isp_find_pdb_by_sid(isp, notify->nt_channel, notify->nt_sid, &lp) == 0) {
2877                 inot->initiator_id = CAM_TARGET_WILDCARD;
2878         } else {
2879                 inot->initiator_id = lp->handle;
2880         }
2881         inot->seq_id = notify->nt_tagval;
2882         inot->tag_id = notify->nt_tagval >> 32;
2883
2884         switch (notify->nt_ncode) {
2885         case NT_ABORT_TASK:
2886                 isp_target_mark_aborted_early(isp, tptr, inot->tag_id);
2887                 inot->arg = MSG_ABORT_TASK;
2888                 break;
2889         case NT_ABORT_TASK_SET:
2890                 isp_target_mark_aborted_early(isp, tptr, TAG_ANY);
2891                 inot->arg = MSG_ABORT_TASK_SET;
2892                 break;
2893         case NT_CLEAR_ACA:
2894                 inot->arg = MSG_CLEAR_ACA;
2895                 break;
2896         case NT_CLEAR_TASK_SET:
2897                 inot->arg = MSG_CLEAR_TASK_SET;
2898                 break;
2899         case NT_LUN_RESET:
2900                 inot->arg = MSG_LOGICAL_UNIT_RESET;
2901                 break;
2902         case NT_TARGET_RESET:
2903                 inot->arg = MSG_TARGET_RESET;
2904                 break;
2905         default:
2906                 isp_prt(isp, ISP_LOGWARN, "%s: unknown TMF code 0x%x for chan %d lun 0x%x", __func__, notify->nt_ncode, notify->nt_channel, lun);
2907                 goto bad;
2908         }
2909
2910         ntp = isp_get_ntpd(isp, tptr);
2911         if (ntp == NULL) {
2912                 isp_prt(isp, ISP_LOGWARN, "%s: out of inotify private structures", __func__);
2913                 goto bad;
2914         }
2915         ISP_MEMCPY(&ntp->rd.nt, notify, sizeof (isp_notify_t));
2916         if (notify->nt_lreserved) {
2917                 ISP_MEMCPY(&ntp->rd.data, notify->nt_lreserved, QENTRY_LEN);
2918                 ntp->rd.nt.nt_lreserved = &ntp->rd.data;
2919         }
2920         ntp->rd.seq_id = notify->nt_tagval;
2921         ntp->rd.tag_id = notify->nt_tagval >> 32;
2922
2923         tptr->inot_count--;
2924         SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle);
2925         rls_lun_statep(isp, tptr);
2926         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, inot->ccb_h.path, "%s: Take FREE INOT count now %d\n", __func__, tptr->inot_count);
2927         inot->ccb_h.status = CAM_MESSAGE_RECV;
2928         xpt_done((union ccb *)inot);
2929         return;
2930 bad:
2931         if (tptr) {
2932                 rls_lun_statep(isp, tptr);
2933         }
2934         if (notify->nt_need_ack && notify->nt_lreserved) {
2935                 if (((isphdr_t *)notify->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) {
2936                         (void) isp_acknak_abts(isp, notify->nt_lreserved, ENOMEM);
2937                 } else {
2938                         (void) isp_notify_ack(isp, notify->nt_lreserved);
2939                 }
2940         }
2941 }
2942
2943 /*
2944  * Find the associated private data and mark it as dead so
2945  * we don't try to work on it any further.
2946  */
2947 static void
2948 isp_target_mark_aborted(ispsoftc_t *isp, union ccb *ccb)
2949 {
2950         tstate_t *tptr;
2951         atio_private_data_t *atp;
2952
2953         tptr = get_lun_statep(isp, XS_CHANNEL(ccb), XS_LUN(ccb));
2954         if (tptr == NULL) {
2955                 tptr = get_lun_statep(isp, XS_CHANNEL(ccb), CAM_LUN_WILDCARD);
2956                 if (tptr == NULL) {
2957                         ccb->ccb_h.status = CAM_REQ_INVALID;
2958                         return;
2959                 }
2960         }
2961
2962         atp = isp_get_atpd(isp, tptr, ccb->atio.tag_id);
2963         if (atp == NULL) {
2964                 ccb->ccb_h.status = CAM_REQ_INVALID;
2965                 return;
2966         }
2967         atp->dead = 1;
2968         ccb->ccb_h.status = CAM_REQ_CMP;
2969 }
2970
2971 static void
2972 isp_target_mark_aborted_early(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag_id)
2973 {
2974         atio_private_data_t *atp;
2975         inot_private_data_t *restart_queue = tptr->restart_queue;
2976
2977         /*
2978          * First, clean any commands pending restart
2979          */
2980         tptr->restart_queue = NULL;
2981         while (restart_queue) {
2982                 uint32_t this_tag_id;
2983                 inot_private_data_t *ntp = restart_queue;
2984
2985                 restart_queue = ntp->rd.nt.nt_hba;
2986
2987                 if (IS_24XX(isp)) {
2988                         this_tag_id = ((at7_entry_t *)ntp->rd.data)->at_rxid;
2989                 } else {
2990                         this_tag_id = ((at2_entry_t *)ntp->rd.data)->at_rxid;
2991                 }
2992                 if ((uint64_t)tag_id == TAG_ANY || tag_id == this_tag_id) {
2993                         isp_put_ntpd(isp, tptr, ntp);
2994                 } else {
2995                         ntp->rd.nt.nt_hba = tptr->restart_queue;
2996                         tptr->restart_queue = ntp;
2997                 }
2998         }
2999
3000         /*
3001          * Now mark other ones dead as well.
3002          */
3003         for (atp = tptr->atpool; atp < &tptr->atpool[ATPDPSIZE]; atp++) {
3004                 if ((uint64_t)tag_id == TAG_ANY || atp->tag == tag_id) {
3005                         atp->dead = 1;
3006                 }
3007         }
3008 }
3009
3010
3011 #ifdef  ISP_INTERNAL_TARGET
3012 // #define      ISP_FORCE_TIMEOUT               1
3013 // #define      ISP_TEST_WWNS                   1
3014 // #define      ISP_TEST_SEPARATE_STATUS        1
3015
3016 #define ccb_data_offset         ppriv_field0
3017 #define ccb_atio                ppriv_ptr1
3018 #define ccb_inot                ppriv_ptr1
3019
3020 #define MAX_ISP_TARG_TRANSFER   (2 << 20)
3021 #define NISP_TARG_CMDS          1024
3022 #define NISP_TARG_NOTIFIES      1024
3023 #define DISK_SHIFT              9
3024 #define JUNK_SIZE               256
3025
3026 #ifndef VERIFY_10
3027 #define VERIFY_10       0x2f
3028 #endif
3029
3030 TAILQ_HEAD(ccb_queue, ccb_hdr);
3031 extern u_int vm_kmem_size;
3032 static int ca;
3033 static uint32_t disk_size;
3034 static uint8_t *disk_data = NULL;
3035 static uint8_t *junk_data;
3036 static MALLOC_DEFINE(M_ISPTARG, "ISPTARG", "ISP TARGET data");
3037 struct isptarg_softc {
3038         /* CCBs (CTIOs, ATIOs, INOTs) pending on the controller */
3039         struct ccb_queue        work_queue;
3040         struct ccb_queue        rework_queue;
3041         struct ccb_queue        running_queue;
3042         struct ccb_queue        inot_queue;
3043         struct cam_periph       *periph;
3044         struct cam_path         *path;
3045         ispsoftc_t              *isp;
3046 };
3047 static periph_ctor_t    isptargctor;
3048 static periph_dtor_t    isptargdtor;
3049 static periph_start_t   isptargstart;
3050 static periph_init_t    isptarginit;
3051 static void             isptarg_done(struct cam_periph *, union ccb *);
3052 static void             isptargasync(void *, u_int32_t, struct cam_path *, void *);
3053
3054
3055 static int isptarg_rwparm(uint8_t *, uint8_t *, uint64_t, uint32_t, uint8_t **, uint32_t *, int *);
3056
3057 static struct periph_driver isptargdriver =
3058 {
3059         isptarginit, "isptarg", TAILQ_HEAD_INITIALIZER(isptargdriver.units), /* generation */ 0
3060 };
3061
3062 static void
3063 isptarginit(void)
3064 {
3065 }
3066
3067 static void
3068 isptargnotify(ispsoftc_t *isp, union ccb *iccb, struct ccb_immediate_notify *inot)
3069 {
3070         struct ccb_notify_acknowledge *ack = &iccb->cna2;
3071
3072         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, inot->ccb_h.path, "%s: [0x%x] immediate notify for 0x%x from 0x%x status 0x%x arg 0x%x\n", __func__,
3073             inot->tag_id, inot->initiator_id, inot->seq_id, inot->ccb_h.status, inot->arg);
3074         ack->ccb_h.func_code = XPT_NOTIFY_ACKNOWLEDGE;
3075         ack->ccb_h.flags = 0;
3076         ack->ccb_h.retry_count = 0;
3077         ack->ccb_h.cbfcnp = isptarg_done;
3078         ack->ccb_h.timeout = 0;
3079         ack->ccb_h.ccb_inot = inot;
3080         ack->tag_id = inot->tag_id;
3081         ack->seq_id = inot->seq_id;
3082         ack->initiator_id = inot->initiator_id;
3083         xpt_action(iccb);
3084 }
3085
3086 static void
3087 isptargstart(struct cam_periph *periph, union ccb *iccb)
3088 {
3089         const uint8_t niliqd[SHORT_INQUIRY_LENGTH] = {
3090                 0x7f, 0x0, 0x5, 0x2, 32, 0, 0, 0x32,
3091                 'F', 'R', 'E', 'E', 'B', 'S', 'D', ' ',
3092                 'S', 'C', 'S', 'I', ' ', 'N', 'U', 'L',
3093                 'L', ' ', 'D', 'E', 'V', 'I', 'C', 'E',
3094                 '0', '0', '0', '1'
3095         };
3096         const uint8_t iqd[SHORT_INQUIRY_LENGTH] = {
3097                 0, 0x0, 0x5, 0x2, 32, 0, 0, 0x32,
3098                 'F', 'R', 'E', 'E', 'B', 'S', 'D', ' ',
3099                 'S', 'C', 'S', 'I', ' ', 'M', 'E', 'M',
3100                 'O', 'R', 'Y', ' ', 'D', 'I', 'S', 'K',
3101                 '0', '0', '0', '1'
3102         };
3103         int i, more = 0, last;
3104         struct isptarg_softc *softc = periph->softc;
3105         struct ccb_scsiio *csio;
3106         lun_id_t return_lun;
3107         struct ccb_accept_tio *atio;
3108         uint8_t *cdb, *ptr, status;
3109         uint8_t *data_ptr;
3110         uint32_t data_len, flags;
3111         struct ccb_hdr *ccbh;
3112
3113         KKASSERT(lockstatus(periph->sim->mtx, curthread) != 0);
3114         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, iccb->ccb_h.path, "%s: function code 0x%x INOTQ=%c WORKQ=%c REWORKQ=%c\n", __func__, iccb->ccb_h.func_code,
3115             TAILQ_FIRST(&softc->inot_queue)? 'y' : 'n', TAILQ_FIRST(&softc->work_queue)? 'y' : 'n', TAILQ_FIRST(&softc->rework_queue)? 'y' : 'n');
3116         /*
3117          * Check for immediate notifies first
3118          */
3119         ccbh = TAILQ_FIRST(&softc->inot_queue);
3120         if (ccbh) {
3121                 TAILQ_REMOVE(&softc->inot_queue, ccbh, periph_links.tqe);
3122                 if (TAILQ_FIRST(&softc->inot_queue) || TAILQ_FIRST(&softc->work_queue) || TAILQ_FIRST(&softc->rework_queue)) {
3123                         xpt_schedule(periph, 1);
3124                 }
3125                 isptargnotify(softc->isp, iccb, (struct ccb_immediate_notify *)ccbh);
3126                 return;
3127         }
3128
3129         /*
3130          * Check the rework (continuation) work queue first.
3131          */
3132         ccbh = TAILQ_FIRST(&softc->rework_queue);
3133         if (ccbh) {
3134                 atio = (struct ccb_accept_tio *)ccbh;
3135                 TAILQ_REMOVE(&softc->rework_queue, ccbh, periph_links.tqe);
3136                 more = TAILQ_FIRST(&softc->work_queue) || TAILQ_FIRST(&softc->rework_queue);
3137         } else {
3138                 ccbh = TAILQ_FIRST(&softc->work_queue);
3139                 if (ccbh == NULL) {
3140                         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, iccb->ccb_h.path, "%s: woken up but no work?\n", __func__);
3141                         xpt_release_ccb(iccb);
3142                         return;
3143                 }
3144                 atio = (struct ccb_accept_tio *)ccbh;
3145                 TAILQ_REMOVE(&softc->work_queue, ccbh, periph_links.tqe);
3146                 more = TAILQ_FIRST(&softc->work_queue) != NULL;
3147                 atio->ccb_h.ccb_data_offset = 0;
3148         }
3149
3150         if (atio->tag_id == 0xffffffff || atio->ccb_h.func_code != XPT_ACCEPT_TARGET_IO) {
3151                 panic("BAD ATIO");
3152         }
3153
3154         data_ptr = NULL;
3155         data_len = 0;
3156         csio = &iccb->csio;
3157         status = SCSI_STATUS_OK;
3158         flags = CAM_SEND_STATUS;
3159         memset(&atio->sense_data, 0, sizeof (atio->sense_data));
3160         cdb = atio->cdb_io.cdb_bytes;
3161         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, ccbh->path, "%s: [0x%x] processing ATIO from 0x%x CDB=0x%x data_offset=%u\n", __func__, atio->tag_id, atio->init_id,
3162             cdb[0], atio->ccb_h.ccb_data_offset);
3163
3164         return_lun = XS_LUN(atio);
3165         if (return_lun != 0) {
3166                 xpt_print(atio->ccb_h.path, "[0x%x] Non-Zero Lun %d: cdb0=0x%x\n", atio->tag_id, return_lun, cdb[0]);
3167                 if (cdb[0] != INQUIRY && cdb[0] != REPORT_LUNS && cdb[0] != REQUEST_SENSE) {
3168                         status = SCSI_STATUS_CHECK_COND;
3169                         atio->sense_data.error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_ILLEGAL_REQUEST;
3170                         atio->sense_data.add_sense_code = 0x25;
3171                         atio->sense_data.add_sense_code_qual = 0x0;
3172                         atio->sense_len = sizeof (atio->sense_data);
3173                 }
3174                 return_lun = CAM_LUN_WILDCARD;
3175         }
3176
3177         switch (cdb[0]) {
3178         case REQUEST_SENSE:
3179                 flags |= CAM_DIR_IN;
3180                 data_len = sizeof (atio->sense_data);
3181                 junk_data[0] = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_NO_SENSE;
3182                 memset(junk_data+1, 0, data_len-1);
3183                 if (data_len > cdb[4]) {
3184                         data_len = cdb[4];
3185                 }
3186                 if (data_len) {
3187                         data_ptr = junk_data;
3188                 }
3189                 break;
3190         case READ_6:
3191         case READ_10:
3192         case READ_12:
3193         case READ_16:
3194                 if (isptarg_rwparm(cdb, disk_data, disk_size, atio->ccb_h.ccb_data_offset, &data_ptr, &data_len, &last)) {
3195                         status = SCSI_STATUS_CHECK_COND;
3196                         atio->sense_data.error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_UNIT_ATTENTION;
3197                         atio->sense_data.add_sense_code = 0x5;
3198                         atio->sense_data.add_sense_code_qual = 0x24;
3199                         atio->sense_len = sizeof (atio->sense_data);
3200                 } else {
3201 #ifdef  ISP_FORCE_TIMEOUT
3202                         {
3203                                 static int foo;
3204                                 if (foo++ == 500) {
3205                                         if (more) {
3206                                                 xpt_schedule(periph, 1);
3207                                         }
3208                                         foo = 0;
3209                                         return;
3210                                 }
3211                         }
3212 #endif
3213 #ifdef  ISP_TEST_SEPARATE_STATUS
3214                         if (last && data_len) {
3215                                 last = 0;
3216                         }
3217 #endif
3218                         if (last == 0) {
3219                                 flags &= ~CAM_SEND_STATUS;
3220                         }
3221                         if (data_len) {
3222                                 atio->ccb_h.ccb_data_offset += data_len;
3223                                 flags |= CAM_DIR_IN;
3224                         } else {
3225                                 flags |= CAM_DIR_NONE;
3226                         }
3227                 }
3228                 break;
3229         case WRITE_6:
3230         case WRITE_10:
3231         case WRITE_12:
3232         case WRITE_16:
3233                 if (isptarg_rwparm(cdb, disk_data, disk_size, atio->ccb_h.ccb_data_offset, &data_ptr, &data_len, &last)) {
3234                         status = SCSI_STATUS_CHECK_COND;
3235                         atio->sense_data.error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_UNIT_ATTENTION;
3236                         atio->sense_data.add_sense_code = 0x5;
3237                         atio->sense_data.add_sense_code_qual = 0x24;
3238                         atio->sense_len = sizeof (atio->sense_data);
3239                 } else {
3240 #ifdef  ISP_FORCE_TIMEOUT
3241                         {
3242                                 static int foo;
3243                                 if (foo++ == 500) {
3244                                         if (more) {
3245                                                 xpt_schedule(periph, 1);
3246                                         }
3247                                         foo = 0;
3248                                         return;
3249                                 }
3250                         }
3251 #endif
3252 #ifdef  ISP_TEST_SEPARATE_STATUS
3253                         if (last && data_len) {
3254                                 last = 0;
3255                         }
3256 #endif
3257                         if (last == 0) {
3258                                 flags &= ~CAM_SEND_STATUS;
3259                         }
3260                         if (data_len) {
3261                                 atio->ccb_h.ccb_data_offset += data_len;
3262                                 flags |= CAM_DIR_OUT;
3263                         } else {
3264                                 flags |= CAM_DIR_NONE;
3265                         }
3266                 }
3267                 break;
3268         case INQUIRY:
3269                 flags |= CAM_DIR_IN;
3270                 if (cdb[1] || cdb[2] || cdb[3]) {
3271                         status = SCSI_STATUS_CHECK_COND;
3272                         atio->sense_data.error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_UNIT_ATTENTION;
3273                         atio->sense_data.add_sense_code = 0x5;
3274                         atio->sense_data.add_sense_code_qual = 0x20;
3275                         atio->sense_len = sizeof (atio->sense_data);
3276                         break;
3277                 }
3278                 data_len = sizeof (iqd);
3279                 if (data_len > cdb[4]) {
3280                         data_len = cdb[4];
3281                 }
3282                 if (data_len) {
3283                         if (XS_LUN(iccb) != 0) {
3284                                 memcpy(junk_data, niliqd, sizeof (iqd));
3285                         } else {
3286                                 memcpy(junk_data, iqd, sizeof (iqd));
3287                         }
3288                         data_ptr = junk_data;
3289                 }
3290                 break;
3291         case TEST_UNIT_READY:
3292                 flags |= CAM_DIR_NONE;
3293                 if (ca) {
3294                         ca = 0;
3295                         status = SCSI_STATUS_CHECK_COND;
3296                         atio->sense_data.error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_UNIT_ATTENTION;
3297                         atio->sense_data.add_sense_code = 0x28;
3298                         atio->sense_data.add_sense_code_qual = 0x0;
3299                         atio->sense_len = sizeof (atio->sense_data);
3300                 }
3301                 break;
3302         case SYNCHRONIZE_CACHE:
3303         case START_STOP:
3304         case RESERVE:
3305         case RELEASE:
3306         case VERIFY_10:
3307                 flags |= CAM_DIR_NONE;
3308                 break;
3309
3310         case READ_CAPACITY:
3311                 flags |= CAM_DIR_IN;
3312                 if (cdb[2] || cdb[3] || cdb[4] || cdb[5]) {
3313                         status = SCSI_STATUS_CHECK_COND;
3314                         atio->sense_data.error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_UNIT_ATTENTION;
3315                         atio->sense_data.add_sense_code = 0x5;
3316                         atio->sense_data.add_sense_code_qual = 0x24;
3317                         atio->sense_len = sizeof (atio->sense_data);
3318                         break;
3319                 }
3320                 if (cdb[8] & 0x1) { /* PMI */
3321                         junk_data[0] = 0xff;
3322                         junk_data[1] = 0xff;
3323                         junk_data[2] = 0xff;
3324                         junk_data[3] = 0xff;
3325                 } else {
3326                         uint64_t last_blk = (disk_size >> DISK_SHIFT) - 1;
3327                         if (last_blk < 0xffffffffULL) {
3328                             junk_data[0] = (last_blk >> 24) & 0xff;
3329                             junk_data[1] = (last_blk >> 16) & 0xff;
3330                             junk_data[2] = (last_blk >>  8) & 0xff;
3331                             junk_data[3] = (last_blk) & 0xff;
3332                         } else {
3333                             junk_data[0] = 0xff;
3334                             junk_data[1] = 0xff;
3335                             junk_data[2] = 0xff;
3336                             junk_data[3] = 0xff;
3337                         }
3338                 }
3339                 junk_data[4] = ((1 << DISK_SHIFT) >> 24) & 0xff;
3340                 junk_data[5] = ((1 << DISK_SHIFT) >> 16) & 0xff;
3341                 junk_data[6] = ((1 << DISK_SHIFT) >>  8) & 0xff;
3342                 junk_data[7] = ((1 << DISK_SHIFT)) & 0xff;
3343                 data_ptr = junk_data;
3344                 data_len = 8;
3345                 break;
3346         case REPORT_LUNS:
3347                 flags |= CAM_DIR_IN;
3348                 memset(junk_data, 0, JUNK_SIZE);
3349                 junk_data[0] = (1 << 3) >> 24;
3350                 junk_data[1] = (1 << 3) >> 16;
3351                 junk_data[2] = (1 << 3) >> 8;
3352                 junk_data[3] = (1 << 3);
3353                 ptr = NULL;
3354                 for (i = 0; i < 1; i++) {
3355                         ptr = &junk_data[8 + (1 << 3)];
3356                         if (i >= 256) {
3357                                 ptr[0] = 0x40 | ((i >> 8) & 0x3f);
3358                         }
3359                         ptr[1] = i;
3360                 }
3361                 data_ptr = junk_data;
3362                 data_len = (ptr + 8) - junk_data;
3363                 break;
3364
3365         default:
3366                 flags |= CAM_DIR_NONE;
3367                 status = SCSI_STATUS_CHECK_COND;
3368                 atio->sense_data.error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_UNIT_ATTENTION;
3369                 atio->sense_data.add_sense_code = 0x5;
3370                 atio->sense_data.add_sense_code_qual = 0x20;
3371                 atio->sense_len = sizeof (atio->sense_data);
3372                 break;
3373         }
3374
3375         /*
3376          * If we are done with the transaction, tell the
3377          * controller to send status and perform a CMD_CMPLT.
3378          * If we have associated sense data, see if we can
3379          * send that too.
3380          */
3381         if (status == SCSI_STATUS_CHECK_COND) {
3382                 flags |= CAM_SEND_SENSE;
3383                 csio->sense_len = atio->sense_len;
3384                 csio->sense_data = atio->sense_data;
3385                 flags &= ~CAM_DIR_MASK;
3386                 data_len = 0;
3387                 data_ptr = NULL;
3388         }
3389         cam_fill_ctio(csio, 0, isptarg_done, flags, MSG_SIMPLE_Q_TAG, atio->tag_id, atio->init_id, status, data_ptr, data_len, 0);
3390         iccb->ccb_h.target_id = atio->ccb_h.target_id;
3391         iccb->ccb_h.target_lun = return_lun;
3392         iccb->ccb_h.ccb_atio = atio;
3393         xpt_action(iccb);
3394
3395         if ((atio->ccb_h.status & CAM_DEV_QFRZN) != 0) {
3396                 cam_release_devq(periph->path, 0, 0, 0, 0);
3397                 atio->ccb_h.status &= ~CAM_DEV_QFRZN;
3398         }
3399         if (more) {
3400                 xpt_schedule(periph, 1);
3401         }
3402 }
3403
3404 static cam_status
3405 isptargctor(struct cam_periph *periph, void *arg)
3406 {
3407         struct isptarg_softc *softc;
3408
3409         softc = (struct isptarg_softc *)arg;
3410         periph->softc = softc;
3411         softc->periph = periph;
3412         softc->path = periph->path;
3413         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, periph->path, "%s called\n", __func__);
3414         return (CAM_REQ_CMP);
3415 }
3416
3417 static void
3418 isptargdtor(struct cam_periph *periph)
3419 {
3420         struct isptarg_softc *softc;
3421         softc = (struct isptarg_softc *)periph->softc;
3422         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, periph->path, "%s called\n", __func__);
3423         softc->periph = NULL;
3424         softc->path = NULL;
3425         periph->softc = NULL;
3426 }
3427
3428 static void
3429 isptarg_done(struct cam_periph *periph, union ccb *ccb)
3430 {
3431         struct isptarg_softc *softc;
3432         ispsoftc_t *isp;
3433         struct ccb_accept_tio *atio;
3434         struct ccb_immediate_notify *inot;
3435         cam_status status;
3436
3437         softc = (struct isptarg_softc *)periph->softc;
3438         isp = softc->isp;
3439         status = ccb->ccb_h.status & CAM_STATUS_MASK;
3440
3441         switch (ccb->ccb_h.func_code) {
3442         case XPT_ACCEPT_TARGET_IO:
3443                 atio = (struct ccb_accept_tio *) ccb;
3444                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] ATIO seen in %s\n", atio->tag_id, __func__);
3445                 TAILQ_INSERT_TAIL(&softc->work_queue, &ccb->ccb_h, periph_links.tqe);
3446                 xpt_schedule(periph, 1);
3447                 break;
3448         case XPT_IMMEDIATE_NOTIFY:
3449                 inot = (struct ccb_immediate_notify *) ccb;
3450                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] INOT for 0x%x seen in %s\n", inot->tag_id, inot->seq_id, __func__);
3451                 TAILQ_INSERT_TAIL(&softc->inot_queue, &ccb->ccb_h, periph_links.tqe);
3452                 xpt_schedule(periph, 1);
3453                 break;
3454         case XPT_CONT_TARGET_IO:
3455                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
3456                         cam_release_devq(ccb->ccb_h.path, 0, 0, 0, 0);
3457                         ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3458                 }
3459                 atio = ccb->ccb_h.ccb_atio;
3460                 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
3461                         cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
3462                         xpt_action((union ccb *)atio);
3463                 } else if ((ccb->ccb_h.flags & CAM_SEND_STATUS) == 0) {
3464                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] MID CTIO seen in %s\n", atio->tag_id, __func__);
3465                         TAILQ_INSERT_TAIL(&softc->rework_queue, &atio->ccb_h, periph_links.tqe);
3466                         xpt_schedule(periph, 1);
3467                 } else {
3468                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] FINAL CTIO seen in %s\n", atio->tag_id, __func__);
3469                         xpt_action((union ccb *)atio);
3470                 }
3471                 xpt_release_ccb(ccb);
3472                 break;
3473         case XPT_NOTIFY_ACKNOWLEDGE:
3474                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
3475                         cam_release_devq(ccb->ccb_h.path, 0, 0, 0, 0);
3476                         ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3477                 }
3478                 inot = ccb->ccb_h.ccb_inot;
3479                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, inot->ccb_h.path, "[0x%x] recycle notify for tag 0x%x\n", inot->tag_id, inot->seq_id);
3480                 xpt_release_ccb(ccb);
3481                 xpt_action((union ccb *)inot);
3482                 break;
3483         default:
3484                 xpt_print(ccb->ccb_h.path, "unexpected code 0x%x\n", ccb->ccb_h.func_code);
3485                 break;
3486         }
3487 }
3488
3489 static void
3490 isptargasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg)
3491 {
3492         struct ac_contract *acp = arg;
3493         struct ac_device_changed *fc = (struct ac_device_changed *) acp->contract_data;
3494
3495         if (code != AC_CONTRACT) {
3496                 return;
3497         }
3498         xpt_print(path, "0x%016llx Port ID 0x%06x %s\n", (unsigned long long) fc->wwpn, fc->port, fc->arrived? "arrived" : "departed");
3499 }
3500
3501 static void
3502 isp_target_thread(ispsoftc_t *isp, int chan)
3503 {
3504         union ccb *ccb = NULL;
3505         int i;
3506         void *wchan;
3507         cam_status status;
3508         struct isptarg_softc *softc = NULL;
3509         struct cam_periph *periph = NULL, *wperiph = NULL;
3510         struct cam_path *path, *wpath;
3511         struct cam_sim *sim;
3512
3513         if (disk_data == NULL) {
3514                 disk_size = roundup2(vm_kmem_size >> 1, (1ULL << 20));
3515                 if (disk_size < (50 << 20)) {
3516                         disk_size = 50 << 20;
3517                 }
3518                 disk_data = kmalloc(disk_size, M_ISPTARG, M_WAITOK | M_ZERO);
3519                 isp_prt(isp, ISP_LOGINFO, "allocated a %ju MiB disk", (uintmax_t) (disk_size >> 20));
3520         }
3521         junk_data = kmalloc(JUNK_SIZE, M_ISPTARG, M_WAITOK | M_ZERO);
3522
3523
3524         softc = kmalloc(sizeof (*softc), M_ISPTARG, M_WAITOK | M_ZERO);
3525         TAILQ_INIT(&softc->work_queue);
3526         TAILQ_INIT(&softc->rework_queue);
3527         TAILQ_INIT(&softc->running_queue);
3528         TAILQ_INIT(&softc->inot_queue);
3529         softc->isp = isp;
3530
3531         periphdriver_register(&isptargdriver);
3532         ISP_GET_PC(isp, chan, sim, sim);
3533         ISP_GET_PC(isp, chan, path,  path);
3534         status = xpt_create_path_unlocked(&wpath, NULL, cam_sim_path(sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3535         if (status != CAM_REQ_CMP) {
3536                 isp_prt(isp, ISP_LOGERR, "%s: could not allocate wildcard path", __func__);
3537                 return;
3538         }
3539         status = xpt_create_path_unlocked(&path, NULL, cam_sim_path(sim), 0, 0);
3540         if (status != CAM_REQ_CMP) {
3541                 xpt_free_path(wpath);
3542                 isp_prt(isp, ISP_LOGERR, "%s: could not allocate path", __func__);
3543                 return;
3544         }
3545
3546         ISP_LOCK(isp);
3547         status = cam_periph_alloc(isptargctor, NULL, isptargdtor, isptargstart, "isptarg", CAM_PERIPH_BIO, wpath, NULL, 0, softc);
3548         if (status != CAM_REQ_CMP) {
3549                 ISP_UNLOCK(isp);
3550                 isp_prt(isp, ISP_LOGERR, "%s: cam_periph_alloc for wildcard failed", __func__);
3551                 goto out;
3552         }
3553         wperiph = cam_periph_find(wpath, "isptarg");
3554         if (wperiph == NULL) {
3555                 ISP_UNLOCK(isp);
3556                 isp_prt(isp, ISP_LOGERR, "%s: wildcard periph already allocated but doesn't exist", __func__);
3557                 goto out;
3558         }
3559
3560         status = cam_periph_alloc(isptargctor, NULL, isptargdtor, isptargstart, "isptarg", CAM_PERIPH_BIO, path, NULL, 0, softc);
3561         if (status != CAM_REQ_CMP) {
3562                 ISP_UNLOCK(isp);
3563                 isp_prt(isp, ISP_LOGERR, "%s: cam_periph_alloc failed", __func__);
3564                 goto out;
3565         }
3566
3567         periph = cam_periph_find(path, "isptarg");
3568         if (periph == NULL) {
3569                 ISP_UNLOCK(isp);
3570                 isp_prt(isp, ISP_LOGERR, "%s: periph already allocated but doesn't exist", __func__);
3571                 goto out;
3572         }
3573
3574         status = xpt_register_async(AC_CONTRACT, isptargasync, isp, wpath);
3575         if (status != CAM_REQ_CMP) {
3576                 ISP_UNLOCK(isp);
3577                 isp_prt(isp, ISP_LOGERR, "%s: xpt_register_async failed", __func__);
3578                 goto out;
3579         }
3580
3581         ISP_UNLOCK(isp);
3582
3583         ccb = xpt_alloc_ccb();
3584
3585         /*
3586          * Make sure role is none.
3587          */
3588         xpt_setup_ccb(&ccb->ccb_h, periph->path, 10);
3589         ccb->ccb_h.func_code = XPT_SET_SIM_KNOB;
3590         ccb->knob.xport_specific.fc.role = KNOB_ROLE_NONE;
3591 #ifdef  ISP_TEST_WWNS
3592         ccb->knob.xport_specific.fc.valid = KNOB_VALID_ROLE | KNOB_VALID_ADDRESS;
3593         ccb->knob.xport_specific.fc.wwnn = 0x508004d000000000ULL | (device_get_unit(isp->isp_osinfo.dev) << 8) | (chan << 16);
3594         ccb->knob.xport_specific.fc.wwpn = 0x508004d000000001ULL | (device_get_unit(isp->isp_osinfo.dev) << 8) | (chan << 16);
3595 #else
3596         ccb->knob.xport_specific.fc.valid = KNOB_VALID_ROLE;
3597 #endif
3598
3599         ISP_LOCK(isp);
3600         xpt_action(ccb);
3601         ISP_UNLOCK(isp);
3602
3603         /*
3604          * Now enable luns
3605          */
3606         xpt_setup_ccb(&ccb->ccb_h, periph->path, 10);
3607         ccb->ccb_h.func_code = XPT_EN_LUN;
3608         ccb->cel.enable = 1;
3609         ISP_LOCK(isp);
3610         xpt_action(ccb);
3611         ISP_UNLOCK(isp);
3612         if (ccb->ccb_h.status != CAM_REQ_CMP) {
3613                 xpt_free_ccb(&ccb->ccb_h);
3614                 xpt_print(periph->path, "failed to enable lun (0x%x)\n", ccb->ccb_h.status);
3615                 goto out;
3616         }
3617
3618         xpt_setup_ccb(&ccb->ccb_h, wperiph->path, 10);
3619         ccb->ccb_h.func_code = XPT_EN_LUN;
3620         ccb->cel.enable = 1;
3621         ISP_LOCK(isp);
3622         xpt_action(ccb);
3623         ISP_UNLOCK(isp);
3624         if (ccb->ccb_h.status != CAM_REQ_CMP) {