sys/kern/subr_diskiocom.c

   1 /*
   2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/proc.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/buf.h>
  40 #include <sys/conf.h>
  41 #include <sys/disklabel.h>
  42 #include <sys/disklabel32.h>
  43 #include <sys/disklabel64.h>
  44 #include <sys/diskslice.h>
  45 #include <sys/diskmbr.h>
  46 #include <sys/disk.h>
  47 #include <sys/malloc.h>
  48 #include <sys/device.h>
  49 #include <sys/devfs.h>
  50 #include <sys/thread.h>
  51 #include <sys/queue.h>
  52 #include <sys/lock.h>
  53 #include <sys/stat.h>
  54 #include <sys/uuid.h>
  55 #include <sys/dmsg.h>
  56
  57 #include <sys/buf2.h>
  58 #include <sys/msgport2.h>
  59
  60 struct dios_open {
  61         int     openrd;
  62         int     openwr;
  63 };
  64
  65 struct dios_io {
  66         int     count;
  67         int     eof;
  68         kdmsg_data_t data;
  69 };
  70
  71 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg");
  72
  73 static int blk_active;
  74 SYSCTL_INT(_debug, OID_AUTO, blk_active, CTLFLAG_RW, &blk_active, 0,
  75            "Number of active iocom IOs");
  76
  77 static int disk_iocom_reconnect(struct disk *dp, struct file *fp);
  78 static int disk_rcvdmsg(kdmsg_msg_t *msg);
  79
  80 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg);
  81 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg);
  82 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg);
  83 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg);
  84 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg);
  85 static void diskiodone(struct bio *bio);
  86
  87 void
  88 disk_iocom_init(struct disk *dp)
  89 {
  90         kdmsg_iocom_init(&dp->d_iocom, dp,
  91                          KDMSG_IOCOMF_AUTOCONN |
  92                          KDMSG_IOCOMF_AUTORXSPAN |
  93                          KDMSG_IOCOMF_AUTOTXSPAN,
  94                          M_DMSG_DISK, disk_rcvdmsg);
  95 }
  96
  97 void
  98 disk_iocom_update(struct disk *dp)
  99 {
 100 }
 101
 102 void
 103 disk_iocom_uninit(struct disk *dp)
 104 {
 105         kdmsg_iocom_uninit(&dp->d_iocom);
 106 }
 107
 108 int
 109 disk_iocom_ioctl(struct disk *dp, u_long cmd, void *data)
 110 {
 111         struct file *fp;
 112         struct disk_ioc_recluster *recl;
 113         int error;
 114
 115         switch(cmd) {
 116         case DIOCRECLUSTER:
 117                 recl = data;
 118                 fp = holdfp(curthread, recl->fd, -1);
 119                 if (fp) {
 120                         error = disk_iocom_reconnect(dp, fp);
 121                 } else {
 122                         error = EINVAL;
 123                 }
 124                 break;
 125         default:
 126                 error = EOPNOTSUPP;
 127                 break;
 128         }
 129         return error;
 130 }
 131
 132 static
 133 int
 134 disk_iocom_reconnect(struct disk *dp, struct file *fp)
 135 {
 136         char devname[64];
 137
 138         ksnprintf(devname, sizeof(devname), "%s%d",
 139                   dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev));
 140
 141         kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname);
 142
 143         dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1;
 144         dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK;
 145         dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK;
 146         dp->d_iocom.auto_lnk_conn.peer_mask = (uint64_t)-1;
 147 #if 0
 148         if (dp->d_info.d_serialno) {
 149                 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
 150                           sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
 151                           "%s/%s", hostname, dp->d_info.d_serialno);
 152         } else {
 153                 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
 154                           sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
 155                           "%s/%s", hostname, devname);
 156         }
 157 #endif
 158         ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
 159                   sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
 160                   "%s/%s", hostname, devname);
 161
 162         dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1;
 163         dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK;
 164         dp->d_iocom.auto_lnk_span.media.block.bytes =
 165                                                 dp->d_info.d_media_size;
 166         dp->d_iocom.auto_lnk_span.media.block.blksize =
 167                                                 dp->d_info.d_media_blksize;
 168         ksnprintf(dp->d_iocom.auto_lnk_span.peer_label,
 169                   sizeof(dp->d_iocom.auto_lnk_span.peer_label),
 170                   "%s", dp->d_iocom.auto_lnk_conn.peer_label);
 171         if (dp->d_info.d_serialno) {
 172                 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
 173                           sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
 174                           "%s", dp->d_info.d_serialno);
 175         } else {
 176                 /*
 177                  * If no serial number is available generate a dummy serial
 178                  * number from the host and device name and pray.  This will
 179                  * allow e.g. /dev/vn* to look meaningful on a remote machine.
 180                  */
 181                 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
 182                           sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
 183                           "%s.%s", hostname, devname);
 184         }
 185
 186         kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL);
 187
 188         return (0);
 189 }
 190
 191 static int
 192 disk_rcvdmsg(kdmsg_msg_t *msg)
 193 {
 194         struct disk *dp = msg->state->iocom->handle;
 195
 196         /*
 197          * Handle debug messages (these might not be in transactions)
 198          */
 199         switch(msg->any.head.cmd & DMSGF_CMDSWMASK) {
 200         case DMSG_DBG_SHELL:
 201                 /*
 202                  * Execute shell command (not supported atm)
 203                  */
 204                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 205                 return(0);
 206         case DMSG_DBG_SHELL | DMSGF_REPLY:
 207                 if (msg->aux_data) {
 208                         msg->aux_data[msg->aux_size - 1] = 0;
 209                         kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data);
 210                 }
 211                 return(0);
 212         }
 213
 214         /*
 215          * All remaining messages must be in a transaction.
 216          *
 217          * NOTE!  We currently don't care if the transaction is just
 218          *        the span transaction (for disk probes) or if it is the
 219          *        BLK_OPEN transaction.
 220          *
 221          * NOTE!  We are switching on the first message's command.  The
 222          *        actual message command within the transaction may be
 223          *        different (if streaming within a transaction).
 224          */
 225         if (msg->state == &msg->state->iocom->state0) {
 226                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 227                 return(0);
 228         }
 229
 230         switch(msg->state->rxcmd & DMSGF_CMDSWMASK) {
 231         case DMSG_BLK_OPEN:
 232                 disk_blk_open(dp, msg);
 233                 break;
 234         case DMSG_BLK_READ:
 235                 /*
 236                  * not reached normally but leave in for completeness
 237                  */
 238                 disk_blk_read(dp, msg);
 239                 break;
 240         case DMSG_BLK_WRITE:
 241                 disk_blk_write(dp, msg);
 242                 break;
 243         case DMSG_BLK_FLUSH:
 244                 disk_blk_flush(dp, msg);
 245                 break;
 246         case DMSG_BLK_FREEBLKS:
 247                 disk_blk_freeblks(dp, msg);
 248                 break;
 249         default:
 250                 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) {
 251                         if (msg->any.head.cmd & DMSGF_DELETE)
 252                                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 253                         else
 254                                 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP);
 255                 }
 256                 break;
 257         }
 258         return (0);
 259 }
 260
 261 static
 262 void
 263 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg)
 264 {
 265         struct dios_open *openst;
 266         int error = DMSG_ERR_NOSUPP;
 267         int fflags;
 268
 269         openst = msg->state->any.any;
 270         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) {
 271                 if (openst == NULL) {
 272                         openst = kmalloc(sizeof(*openst), M_DEVBUF,
 273                                                 M_WAITOK | M_ZERO);
 274                         msg->state->any.any = openst;
 275                 }
 276                 fflags = 0;
 277                 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 278                         fflags = FREAD;
 279                 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 280                         fflags |= FWRITE;
 281                 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR,
 282                                   proc0.p_ucred, NULL, NULL);
 283                 if (error) {
 284                         error = DMSG_ERR_IO;
 285                 } else {
 286                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 287                                 ++openst->openrd;
 288                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 289                                 ++openst->openwr;
 290                 }
 291         }
 292 #if 0
 293         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE &&
 294             openst) {
 295                 fflags = 0;
 296                 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) &&
 297                     openst->openrd) {
 298                         fflags = FREAD;
 299                 }
 300                 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) &&
 301                     openst->openwr) {
 302                         fflags |= FWRITE;
 303                 }
 304                 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL);
 305                 if (error) {
 306                         error = DMSG_ERR_IO;
 307                 } else {
 308                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 309                                 --openst->openrd;
 310                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 311                                 --openst->openwr;
 312                 }
 313         }
 314 #endif
 315         if (msg->any.head.cmd & DMSGF_DELETE) {
 316                 if (openst) {
 317                         while (openst->openrd && openst->openwr) {
 318                                 --openst->openrd;
 319                                 --openst->openwr;
 320                                 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL);
 321                         }
 322                         while (openst->openrd) {
 323                                 --openst->openrd;
 324                                 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL);
 325                         }
 326                         while (openst->openwr) {
 327                                 --openst->openwr;
 328                                 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL);
 329                         }
 330                         kfree(openst, M_DEVBUF);
 331                         msg->state->any.any = NULL;
 332                 }
 333                 kdmsg_msg_reply(msg, error);
 334         } else {
 335                 kdmsg_msg_result(msg, error);
 336         }
 337 }
 338
 339 static
 340 void
 341 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg)
 342 {
 343         struct dios_io *iost;
 344         struct buf *bp;
 345         struct bio *bio;
 346         int error = DMSG_ERR_NOSUPP;
 347         int reterr = 1;
 348
 349         /*
 350          * Only DMSG_BLK_READ commands imply read ops.
 351          */
 352         iost = msg->state->any.any;
 353         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) {
 354                 if (msg->any.blk_read.bytes < DEV_BSIZE ||
 355                     msg->any.blk_read.bytes > MAXPHYS) {
 356                         error = DMSG_ERR_PARAM;
 357                         goto done;
 358                 }
 359                 if (iost == NULL) {
 360                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 361                                        M_WAITOK | M_ZERO);
 362                         msg->state->any.any = iost;
 363                 }
 364                 reterr = 0;
 365                 bp = getpbuf_mem(NULL);
 366                 KKASSERT(msg->any.blk_read.bytes <= bp->b_bufsize);
 367                 bio = &bp->b_bio1;
 368                 bp->b_cmd = BUF_CMD_READ;
 369                 bp->b_bcount = msg->any.blk_read.bytes;
 370                 bp->b_resid = bp->b_bcount;
 371                 bio->bio_offset = msg->any.blk_read.offset;
 372                 bio->bio_caller_info1.ptr = msg->state;
 373                 bio->bio_done = diskiodone;
 374
 375                 /* kdmsg_state_hold(msg->state); */
 376                 atomic_add_int(&blk_active, 1);
 377                 atomic_add_int(&iost->count, 1);
 378                 if (msg->any.head.cmd & DMSGF_DELETE)
 379                         iost->eof = 1;
 380                 BUF_KERNPROC(bp);
 381                 dev_dstrategy(dp->d_rawdev, bio);
 382         }
 383 done:
 384         if (reterr) {
 385                 if (msg->any.head.cmd & DMSGF_DELETE) {
 386                         if (iost && iost->count == 0) {
 387                                 kfree(iost, M_DEVBUF);
 388                                 msg->state->any.any = NULL;
 389                         }
 390                         kdmsg_msg_reply(msg, error);
 391                 } else {
 392                         kdmsg_msg_result(msg, error);
 393                 }
 394         }
 395 }
 396
 397 static
 398 void
 399 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg)
 400 {
 401         struct dios_io *iost;
 402         struct buf *bp;
 403         struct bio *bio;
 404         int error = DMSG_ERR_NOSUPP;
 405         int reterr = 1;
 406
 407         /*
 408          * Only DMSG_BLK_WRITE commands imply read ops.
 409          */
 410         iost = msg->state->any.any;
 411         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) {
 412                 if (msg->any.blk_write.bytes < DEV_BSIZE ||
 413                     msg->any.blk_write.bytes > MAXPHYS) {
 414                         error = DMSG_ERR_PARAM;
 415                         goto done;
 416                 }
 417                 if (iost == NULL) {
 418                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 419                                        M_WAITOK | M_ZERO);
 420                         msg->state->any.any = iost;
 421                 }
 422
 423                 /*
 424                  * Issue WRITE.  Short data implies zeros.  Try to optimize
 425                  * the buffer cache buffer for the case where we can just
 426                  * use the message's data pointer.
 427                  */
 428                 reterr = 0;
 429                 if (msg->aux_size >= msg->any.blk_write.bytes)
 430                         bp = getpbuf(NULL);
 431                 else
 432                         bp = getpbuf_mem(NULL);
 433                 KKASSERT(msg->any.blk_write.bytes <= bp->b_bufsize);
 434                 bio = &bp->b_bio1;
 435                 bp->b_cmd = BUF_CMD_WRITE;
 436                 bp->b_bcount = msg->any.blk_write.bytes;
 437                 bp->b_resid = bp->b_bcount;
 438                 if (msg->aux_size >= msg->any.blk_write.bytes) {
 439                         bp->b_data = msg->aux_data;
 440                         kdmsg_detach_aux_data(msg, &iost->data);
 441                 } else {
 442                         bcopy(msg->aux_data, bp->b_data, msg->aux_size);
 443                         bzero(bp->b_data + msg->aux_size,
 444                               msg->any.blk_write.bytes - msg->aux_size);
 445                         bzero(&iost->data, sizeof(iost->data));
 446                 }
 447                 bio->bio_offset = msg->any.blk_write.offset;
 448                 bio->bio_caller_info1.ptr = msg->state;
 449                 bio->bio_done = diskiodone;
 450
 451                 /* kdmsg_state_hold(msg->state); */
 452                 atomic_add_int(&blk_active, 1);
 453                 atomic_add_int(&iost->count, 1);
 454                 if (msg->any.head.cmd & DMSGF_DELETE)
 455                         iost->eof = 1;
 456                 BUF_KERNPROC(bp);
 457                 dev_dstrategy(dp->d_rawdev, bio);
 458         }
 459 done:
 460         if (reterr) {
 461                 if (msg->any.head.cmd & DMSGF_DELETE) {
 462                         if (iost && iost->count == 0) {
 463                                 kfree(iost, M_DEVBUF);
 464                                 msg->state->any.any = NULL;
 465                         }
 466                         kdmsg_msg_reply(msg, error);
 467                 } else {
 468                         kdmsg_msg_result(msg, error);
 469                 }
 470         }
 471 }
 472
 473 static
 474 void
 475 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg)
 476 {
 477         struct dios_io *iost;
 478         struct buf *bp;
 479         struct bio *bio;
 480         int error = DMSG_ERR_NOSUPP;
 481         int reterr = 1;
 482
 483         /*
 484          * Only DMSG_BLK_FLUSH commands imply read ops.
 485          */
 486         iost = msg->state->any.any;
 487         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) {
 488                 if (iost == NULL) {
 489                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 490                                        M_WAITOK | M_ZERO);
 491                         msg->state->any.any = iost;
 492                 }
 493                 reterr = 0;
 494                 bp = getpbuf(NULL);
 495                 bio = &bp->b_bio1;
 496                 bp->b_cmd = BUF_CMD_FLUSH;
 497                 bp->b_bcount = msg->any.blk_flush.bytes;
 498                 bp->b_resid = 0;
 499                 bio->bio_offset = msg->any.blk_flush.offset;
 500                 bio->bio_caller_info1.ptr = msg->state;
 501                 bio->bio_done = diskiodone;
 502
 503                 /* kdmsg_state_hold(msg->state); */
 504                 atomic_add_int(&blk_active, 1);
 505                 atomic_add_int(&iost->count, 1);
 506                 if (msg->any.head.cmd & DMSGF_DELETE)
 507                         iost->eof = 1;
 508                 BUF_KERNPROC(bp);
 509                 dev_dstrategy(dp->d_rawdev, bio);
 510         }
 511         if (reterr) {
 512                 if (msg->any.head.cmd & DMSGF_DELETE) {
 513                         if (iost && iost->count == 0) {
 514                                 kfree(iost, M_DEVBUF);
 515                                 msg->state->any.any = NULL;
 516                         }
 517                         kdmsg_msg_reply(msg, error);
 518                 } else {
 519                         kdmsg_msg_result(msg, error);
 520                 }
 521         }
 522 }
 523
 524 static
 525 void
 526 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg)
 527 {
 528         struct dios_io *iost;
 529         struct buf *bp;
 530         struct bio *bio;
 531         int error = DMSG_ERR_NOSUPP;
 532         int reterr = 1;
 533
 534         /*
 535          * Only DMSG_BLK_FREEBLKS commands imply read ops.
 536          */
 537         iost = msg->state->any.any;
 538         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) {
 539                 if (iost == NULL) {
 540                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 541                                        M_WAITOK | M_ZERO);
 542                         msg->state->any.any = iost;
 543                 }
 544                 reterr = 0;
 545                 bp = getpbuf(NULL);
 546                 bio = &bp->b_bio1;
 547                 bp->b_cmd = BUF_CMD_FREEBLKS;
 548                 bp->b_bcount = msg->any.blk_freeblks.bytes;
 549                 bp->b_resid = 0;
 550                 bio->bio_offset = msg->any.blk_freeblks.offset;
 551                 bio->bio_caller_info1.ptr = msg->state;
 552                 bio->bio_done = diskiodone;
 553
 554                 /* kdmsg_state_hold(msg->state); */
 555                 atomic_add_int(&blk_active, 1);
 556                 atomic_add_int(&iost->count, 1);
 557                 if (msg->any.head.cmd & DMSGF_DELETE)
 558                         iost->eof = 1;
 559                 BUF_KERNPROC(bp);
 560                 dev_dstrategy(dp->d_rawdev, bio);
 561         }
 562         if (reterr) {
 563                 if (msg->any.head.cmd & DMSGF_DELETE) {
 564                         if (iost && iost->count == 0) {
 565                                 kfree(iost, M_DEVBUF);
 566                                 msg->state->any.any = NULL;
 567                         }
 568                         kdmsg_msg_reply(msg, error);
 569                 } else {
 570                         kdmsg_msg_result(msg, error);
 571                 }
 572         }
 573 }
 574
 575 static
 576 void
 577 diskiodone(struct bio *bio)
 578 {
 579         struct buf *bp = bio->bio_buf;
 580         kdmsg_state_t *state = bio->bio_caller_info1.ptr;
 581         kdmsg_msg_t *rmsg;
 582         struct dios_io *iost = state->any.any;
 583         int error;
 584         int resid = 0;
 585         int bytes;
 586         uint32_t cmd;
 587         void *data;
 588
 589         cmd = DMSG_LNK_ERROR;
 590         data = NULL;
 591         bytes = 0;
 592
 593         switch(bp->b_cmd) {
 594         case BUF_CMD_READ:
 595                 cmd = DMSG_LNK_ERROR;
 596                 data = bp->b_data;
 597                 bytes = bp->b_bcount;
 598                 /* fall through */
 599         case BUF_CMD_WRITE:
 600                 if (bp->b_flags & B_ERROR) {
 601                         error = bp->b_error;
 602                 } else {
 603                         error = 0;
 604                         resid = bp->b_resid;
 605                 }
 606                 kdmsg_free_aux_data(&iost->data);
 607                 break;
 608         case BUF_CMD_FLUSH:
 609         case BUF_CMD_FREEBLKS:
 610                 if (bp->b_flags & B_ERROR)
 611                         error = bp->b_error;
 612                 else
 613                         error = 0;
 614                 break;
 615         default:
 616                 panic("diskiodone: Unknown bio cmd = %d\n",
 617                       bio->bio_buf->b_cmd);
 618                 error = 0;      /* avoid compiler warning */
 619                 break;          /* NOT REACHED */
 620         }
 621
 622         /*
 623          * Convert error to DMSG_ERR_* code.
 624          */
 625         if (error)
 626                 error = DMSG_ERR_IO;
 627
 628         /*
 629          * Convert LNK_ERROR or BLK_ERROR if non-zero resid.  READS will
 630          * have already converted cmd to BLK_ERROR and set up data to return.
 631          */
 632         if (resid && cmd == DMSG_LNK_ERROR)
 633                 cmd = DMSG_BLK_ERROR;
 634         /* XXX txcmd is delayed so this won't work for streaming */
 635         if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */
 636                 cmd |= DMSGF_CREATE;
 637         if (iost->eof) {
 638                 if (atomic_fetchadd_int(&iost->count, -1) == 1)
 639                         cmd |= DMSGF_DELETE;
 640         } else {
 641                 atomic_add_int(&iost->count, -1);
 642         }
 643         atomic_add_int(&blk_active, -1);
 644         cmd |= DMSGF_REPLY;
 645
 646         /*
 647          * Allocate a basic or extended reply.  Be careful not to populate
 648          * extended header fields unless we allocated an extended reply.
 649          */
 650         rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0);
 651         if (data) {
 652                 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT);
 653                 rmsg->aux_size = bytes;
 654                 rmsg->flags |= KDMSG_FLAG_AUXALLOC;
 655                 bcopy(data, rmsg->aux_data, bytes);
 656         }
 657         rmsg->any.blk_error.head.error = error;
 658         if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR)
 659                 rmsg->any.blk_error.resid = resid;
 660         bio->bio_caller_info1.ptr = NULL;
 661         /* kdmsg_state_drop(state); */
 662         kdmsg_msg_write(rmsg);
 663         if (bp->b_flags & B_PAGING) {
 664                 relpbuf(bp, NULL);
 665         } else {
 666                 bp->b_flags |= B_INVAL | B_AGE;
 667                 relpbuf(bp, NULL);
 668         }
 669 }