sys/kern/subr_diskiocom.c

   1 /*
   2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/proc.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/buf.h>
  40 #include <sys/conf.h>
  41 #include <sys/disklabel.h>
  42 #include <sys/disklabel32.h>
  43 #include <sys/disklabel64.h>
  44 #include <sys/diskslice.h>
  45 #include <sys/diskmbr.h>
  46 #include <sys/disk.h>
  47 #include <sys/malloc.h>
  48 #include <sys/device.h>
  49 #include <sys/devfs.h>
  50 #include <sys/thread.h>
  51 #include <sys/queue.h>
  52 #include <sys/lock.h>
  53 #include <sys/stat.h>
  54 #include <sys/uuid.h>
  55 #include <sys/dmsg.h>
  56
  57 #include <sys/buf2.h>
  58 #include <sys/mplock2.h>
  59 #include <sys/msgport2.h>
  60 #include <sys/thread2.h>
  61
  62 struct dios_open {
  63         int     openrd;
  64         int     openwr;
  65 };
  66
  67 struct dios_io {
  68         int     count;
  69         int     eof;
  70         kdmsg_data_t data;
  71 };
  72
  73 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg");
  74
  75 static int disk_iocom_reconnect(struct disk *dp, struct file *fp);
  76 static int disk_rcvdmsg(kdmsg_msg_t *msg);
  77
  78 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg);
  79 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg);
  80 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg);
  81 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg);
  82 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg);
  83 static void diskiodone(struct bio *bio);
  84
  85 void
  86 disk_iocom_init(struct disk *dp)
  87 {
  88         kdmsg_iocom_init(&dp->d_iocom, dp,
  89                          KDMSG_IOCOMF_AUTOCONN |
  90                          KDMSG_IOCOMF_AUTORXSPAN |
  91                          KDMSG_IOCOMF_AUTOTXSPAN,
  92                          M_DMSG_DISK, disk_rcvdmsg);
  93 }
  94
  95 void
  96 disk_iocom_update(struct disk *dp)
  97 {
  98 }
  99
 100 void
 101 disk_iocom_uninit(struct disk *dp)
 102 {
 103         kdmsg_iocom_uninit(&dp->d_iocom);
 104 }
 105
 106 int
 107 disk_iocom_ioctl(struct disk *dp, int cmd, void *data)
 108 {
 109         struct file *fp;
 110         struct disk_ioc_recluster *recl;
 111         int error;
 112
 113         switch(cmd) {
 114         case DIOCRECLUSTER:
 115                 recl = data;
 116                 fp = holdfp(curproc->p_fd, recl->fd, -1);
 117                 if (fp) {
 118                         error = disk_iocom_reconnect(dp, fp);
 119                 } else {
 120                         error = EINVAL;
 121                 }
 122                 break;
 123         default:
 124                 error = EOPNOTSUPP;
 125                 break;
 126         }
 127         return error;
 128 }
 129
 130 static
 131 int
 132 disk_iocom_reconnect(struct disk *dp, struct file *fp)
 133 {
 134         char devname[64];
 135
 136         ksnprintf(devname, sizeof(devname), "%s%d",
 137                   dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev));
 138
 139         kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname);
 140
 141         dp->d_iocom.auto_lnk_conn.pfs_type = DMSG_PFSTYPE_SERVER;
 142         dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1;
 143         dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK;
 144         dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK;
 145         dp->d_iocom.auto_lnk_conn.pfs_mask = (uint64_t)-1;
 146         ksnprintf(dp->d_iocom.auto_lnk_conn.cl_label,
 147                   sizeof(dp->d_iocom.auto_lnk_conn.cl_label),
 148                   "%s/%s", hostname, devname);
 149         if (dp->d_info.d_serialno) {
 150                 ksnprintf(dp->d_iocom.auto_lnk_conn.fs_label,
 151                           sizeof(dp->d_iocom.auto_lnk_conn.fs_label),
 152                           "%s", dp->d_info.d_serialno);
 153         }
 154
 155         dp->d_iocom.auto_lnk_span.pfs_type = DMSG_PFSTYPE_SERVER;
 156         dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1;
 157         dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK;
 158         dp->d_iocom.auto_lnk_span.media.block.bytes =
 159                                                 dp->d_info.d_media_size;
 160         dp->d_iocom.auto_lnk_span.media.block.blksize =
 161                                                 dp->d_info.d_media_blksize;
 162         ksnprintf(dp->d_iocom.auto_lnk_span.cl_label,
 163                   sizeof(dp->d_iocom.auto_lnk_span.cl_label),
 164                   "%s/%s", hostname, devname);
 165         if (dp->d_info.d_serialno) {
 166                 ksnprintf(dp->d_iocom.auto_lnk_span.fs_label,
 167                           sizeof(dp->d_iocom.auto_lnk_span.fs_label),
 168                           "%s", dp->d_info.d_serialno);
 169         }
 170
 171         kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL);
 172
 173         return (0);
 174 }
 175
 176 static int
 177 disk_rcvdmsg(kdmsg_msg_t *msg)
 178 {
 179         struct disk *dp = msg->state->iocom->handle;
 180
 181         /*
 182          * Handle debug messages (these might not be in transactions)
 183          */
 184         switch(msg->any.head.cmd & DMSGF_CMDSWMASK) {
 185         case DMSG_DBG_SHELL:
 186                 /*
 187                  * Execute shell command (not supported atm)
 188                  */
 189                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 190                 return(0);
 191         case DMSG_DBG_SHELL | DMSGF_REPLY:
 192                 if (msg->aux_data) {
 193                         msg->aux_data[msg->aux_size - 1] = 0;
 194                         kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data);
 195                 }
 196                 return(0);
 197         }
 198
 199         /*
 200          * All remaining messages must be in a transaction.
 201          *
 202          * NOTE!  We currently don't care if the transaction is just
 203          *        the span transaction (for disk probes) or if it is the
 204          *        BLK_OPEN transaction.
 205          *
 206          * NOTE!  We are switching on the first message's command.  The
 207          *        actual message command within the transaction may be
 208          *        different (if streaming within a transaction).
 209          */
 210         if (msg->state == &msg->state->iocom->state0) {
 211                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 212                 return(0);
 213         }
 214
 215         switch(msg->state->rxcmd & DMSGF_CMDSWMASK) {
 216         case DMSG_BLK_OPEN:
 217                 disk_blk_open(dp, msg);
 218                 break;
 219         case DMSG_BLK_READ:
 220                 /*
 221                  * not reached normally but leave in for completeness
 222                  */
 223                 disk_blk_read(dp, msg);
 224                 break;
 225         case DMSG_BLK_WRITE:
 226                 disk_blk_write(dp, msg);
 227                 break;
 228         case DMSG_BLK_FLUSH:
 229                 disk_blk_flush(dp, msg);
 230                 break;
 231         case DMSG_BLK_FREEBLKS:
 232                 disk_blk_freeblks(dp, msg);
 233                 break;
 234         default:
 235                 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) {
 236                         if (msg->any.head.cmd & DMSGF_DELETE)
 237                                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 238                         else
 239                                 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP);
 240                 }
 241                 break;
 242         }
 243         return (0);
 244 }
 245
 246 static
 247 void
 248 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg)
 249 {
 250         struct dios_open *openst;
 251         int error = DMSG_ERR_NOSUPP;
 252         int fflags;
 253
 254         openst = msg->state->any.any;
 255         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) {
 256                 if (openst == NULL) {
 257                         openst = kmalloc(sizeof(*openst), M_DEVBUF,
 258                                                 M_WAITOK | M_ZERO);
 259                         msg->state->any.any = openst;
 260                 }
 261                 fflags = 0;
 262                 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 263                         fflags = FREAD;
 264                 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 265                         fflags |= FWRITE;
 266                 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR, proc0.p_ucred, NULL);
 267                 if (error) {
 268                         error = DMSG_ERR_IO;
 269                 } else {
 270                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 271                                 ++openst->openrd;
 272                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 273                                 ++openst->openwr;
 274                 }
 275         }
 276 #if 0
 277         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE &&
 278             openst) {
 279                 fflags = 0;
 280                 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) &&
 281                     openst->openrd) {
 282                         fflags = FREAD;
 283                 }
 284                 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) &&
 285                     openst->openwr) {
 286                         fflags |= FWRITE;
 287                 }
 288                 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL);
 289                 if (error) {
 290                         error = DMSG_ERR_IO;
 291                 } else {
 292                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 293                                 --openst->openrd;
 294                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 295                                 --openst->openwr;
 296                 }
 297         }
 298 #endif
 299         if (msg->any.head.cmd & DMSGF_DELETE) {
 300                 if (openst) {
 301                         while (openst->openrd && openst->openwr) {
 302                                 --openst->openrd;
 303                                 --openst->openwr;
 304                                 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL);
 305                         }
 306                         while (openst->openrd) {
 307                                 --openst->openrd;
 308                                 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL);
 309                         }
 310                         while (openst->openwr) {
 311                                 --openst->openwr;
 312                                 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL);
 313                         }
 314                         kfree(openst, M_DEVBUF);
 315                         msg->state->any.any = NULL;
 316                 }
 317                 kdmsg_msg_reply(msg, error);
 318         } else {
 319                 kdmsg_msg_result(msg, error);
 320         }
 321 }
 322
 323 static
 324 void
 325 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg)
 326 {
 327         struct dios_io *iost;
 328         struct buf *bp;
 329         struct bio *bio;
 330         int error = DMSG_ERR_NOSUPP;
 331         int reterr = 1;
 332
 333         /*
 334          * Only DMSG_BLK_READ commands imply read ops.
 335          */
 336         iost = msg->state->any.any;
 337         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) {
 338                 if (msg->any.blk_read.bytes < DEV_BSIZE ||
 339                     msg->any.blk_read.bytes > MAXPHYS) {
 340                         error = DMSG_ERR_PARAM;
 341                         goto done;
 342                 }
 343                 if (iost == NULL) {
 344                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 345                                        M_WAITOK | M_ZERO);
 346                         msg->state->any.any = iost;
 347                 }
 348                 reterr = 0;
 349                 bp = geteblk(msg->any.blk_read.bytes);
 350                 bio = &bp->b_bio1;
 351                 bp->b_cmd = BUF_CMD_READ;
 352                 bp->b_bcount = msg->any.blk_read.bytes;
 353                 bp->b_resid = bp->b_bcount;
 354                 bio->bio_offset = msg->any.blk_read.offset;
 355                 bio->bio_caller_info1.ptr = msg->state;
 356                 bio->bio_done = diskiodone;
 357                 /* kdmsg_state_hold(msg->state); */
 358
 359                 atomic_add_int(&iost->count, 1);
 360                 if (msg->any.head.cmd & DMSGF_DELETE)
 361                         iost->eof = 1;
 362                 BUF_KERNPROC(bp);
 363                 dev_dstrategy(dp->d_rawdev, bio);
 364         }
 365 done:
 366         if (reterr) {
 367                 if (msg->any.head.cmd & DMSGF_DELETE) {
 368                         if (iost && iost->count == 0) {
 369                                 kfree(iost, M_DEVBUF);
 370                                 msg->state->any.any = NULL;
 371                         }
 372                         kdmsg_msg_reply(msg, error);
 373                 } else {
 374                         kdmsg_msg_result(msg, error);
 375                 }
 376         }
 377 }
 378
 379 static
 380 void
 381 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg)
 382 {
 383         struct dios_io *iost;
 384         struct buf *bp;
 385         struct bio *bio;
 386         int error = DMSG_ERR_NOSUPP;
 387         int reterr = 1;
 388
 389         /*
 390          * Only DMSG_BLK_WRITE commands imply read ops.
 391          */
 392         iost = msg->state->any.any;
 393         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) {
 394                 if (msg->any.blk_write.bytes < DEV_BSIZE ||
 395                     msg->any.blk_write.bytes > MAXPHYS) {
 396                         error = DMSG_ERR_PARAM;
 397                         goto done;
 398                 }
 399                 if (iost == NULL) {
 400                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 401                                        M_WAITOK | M_ZERO);
 402                         msg->state->any.any = iost;
 403                 }
 404
 405                 /*
 406                  * Issue WRITE.  Short data implies zeros.  Try to optimize
 407                  * the buffer cache buffer for the case where we can just
 408                  * use the message's data pointer.
 409                  */
 410                 reterr = 0;
 411                 if (msg->aux_size >= msg->any.blk_write.bytes)
 412                         bp = getpbuf(NULL);
 413                 else
 414                         bp = geteblk(msg->any.blk_write.bytes);
 415                 bio = &bp->b_bio1;
 416                 bp->b_cmd = BUF_CMD_WRITE;
 417                 bp->b_bcount = msg->any.blk_write.bytes;
 418                 bp->b_resid = bp->b_bcount;
 419                 if (msg->aux_size >= msg->any.blk_write.bytes) {
 420                         bp->b_data = msg->aux_data;
 421                         kdmsg_detach_aux_data(msg, &iost->data);
 422                 } else {
 423                         bcopy(msg->aux_data, bp->b_data, msg->aux_size);
 424                         bzero(bp->b_data + msg->aux_size,
 425                               msg->any.blk_write.bytes - msg->aux_size);
 426                         bzero(&iost->data, sizeof(iost->data));
 427                 }
 428                 bio->bio_offset = msg->any.blk_write.offset;
 429                 bio->bio_caller_info1.ptr = msg->state;
 430                 bio->bio_done = diskiodone;
 431                 /* kdmsg_state_hold(msg->state); */
 432
 433                 atomic_add_int(&iost->count, 1);
 434                 if (msg->any.head.cmd & DMSGF_DELETE)
 435                         iost->eof = 1;
 436                 BUF_KERNPROC(bp);
 437                 dev_dstrategy(dp->d_rawdev, bio);
 438         }
 439 done:
 440         if (reterr) {
 441                 if (msg->any.head.cmd & DMSGF_DELETE) {
 442                         if (iost && iost->count == 0) {
 443                                 kfree(iost, M_DEVBUF);
 444                                 msg->state->any.any = NULL;
 445                         }
 446                         kdmsg_msg_reply(msg, error);
 447                 } else {
 448                         kdmsg_msg_result(msg, error);
 449                 }
 450         }
 451 }
 452
 453 static
 454 void
 455 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg)
 456 {
 457         struct dios_io *iost;
 458         struct buf *bp;
 459         struct bio *bio;
 460         int error = DMSG_ERR_NOSUPP;
 461         int reterr = 1;
 462
 463         /*
 464          * Only DMSG_BLK_FLUSH commands imply read ops.
 465          */
 466         iost = msg->state->any.any;
 467         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) {
 468                 if (iost == NULL) {
 469                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 470                                        M_WAITOK | M_ZERO);
 471                         msg->state->any.any = iost;
 472                 }
 473                 reterr = 0;
 474                 bp = getpbuf(NULL);
 475                 bio = &bp->b_bio1;
 476                 bp->b_cmd = BUF_CMD_FLUSH;
 477                 bp->b_bcount = msg->any.blk_flush.bytes;
 478                 bp->b_resid = 0;
 479                 bio->bio_offset = msg->any.blk_flush.offset;
 480                 bio->bio_caller_info1.ptr = msg->state;
 481                 bio->bio_done = diskiodone;
 482                 /* kdmsg_state_hold(msg->state); */
 483
 484                 atomic_add_int(&iost->count, 1);
 485                 if (msg->any.head.cmd & DMSGF_DELETE)
 486                         iost->eof = 1;
 487                 BUF_KERNPROC(bp);
 488                 dev_dstrategy(dp->d_rawdev, bio);
 489         }
 490         if (reterr) {
 491                 if (msg->any.head.cmd & DMSGF_DELETE) {
 492                         if (iost && iost->count == 0) {
 493                                 kfree(iost, M_DEVBUF);
 494                                 msg->state->any.any = NULL;
 495                         }
 496                         kdmsg_msg_reply(msg, error);
 497                 } else {
 498                         kdmsg_msg_result(msg, error);
 499                 }
 500         }
 501 }
 502
 503 static
 504 void
 505 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg)
 506 {
 507         struct dios_io *iost;
 508         struct buf *bp;
 509         struct bio *bio;
 510         int error = DMSG_ERR_NOSUPP;
 511         int reterr = 1;
 512
 513         /*
 514          * Only DMSG_BLK_FREEBLKS commands imply read ops.
 515          */
 516         iost = msg->state->any.any;
 517         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) {
 518                 if (iost == NULL) {
 519                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 520                                        M_WAITOK | M_ZERO);
 521                         msg->state->any.any = iost;
 522                 }
 523                 reterr = 0;
 524                 bp = getpbuf(NULL);
 525                 bio = &bp->b_bio1;
 526                 bp->b_cmd = BUF_CMD_FREEBLKS;
 527                 bp->b_bcount = msg->any.blk_freeblks.bytes;
 528                 bp->b_resid = 0;
 529                 bio->bio_offset = msg->any.blk_freeblks.offset;
 530                 bio->bio_caller_info1.ptr = msg->state;
 531                 bio->bio_done = diskiodone;
 532                 /* kdmsg_state_hold(msg->state); */
 533
 534                 atomic_add_int(&iost->count, 1);
 535                 if (msg->any.head.cmd & DMSGF_DELETE)
 536                         iost->eof = 1;
 537                 BUF_KERNPROC(bp);
 538                 dev_dstrategy(dp->d_rawdev, bio);
 539         }
 540         if (reterr) {
 541                 if (msg->any.head.cmd & DMSGF_DELETE) {
 542                         if (iost && iost->count == 0) {
 543                                 kfree(iost, M_DEVBUF);
 544                                 msg->state->any.any = NULL;
 545                         }
 546                         kdmsg_msg_reply(msg, error);
 547                 } else {
 548                         kdmsg_msg_result(msg, error);
 549                 }
 550         }
 551 }
 552
 553 static
 554 void
 555 diskiodone(struct bio *bio)
 556 {
 557         struct buf *bp = bio->bio_buf;
 558         kdmsg_state_t *state = bio->bio_caller_info1.ptr;
 559         kdmsg_msg_t *rmsg;
 560         struct dios_io *iost = state->any.any;
 561         int error;
 562         int resid = 0;
 563         int bytes;
 564         uint32_t cmd;
 565         void *data;
 566
 567         cmd = DMSG_LNK_ERROR;
 568         data = NULL;
 569         bytes = 0;
 570
 571         switch(bp->b_cmd) {
 572         case BUF_CMD_READ:
 573                 cmd = DMSG_LNK_ERROR;
 574                 data = bp->b_data;
 575                 bytes = bp->b_bcount;
 576                 /* fall through */
 577         case BUF_CMD_WRITE:
 578                 if (bp->b_flags & B_ERROR) {
 579                         error = bp->b_error;
 580                 } else {
 581                         error = 0;
 582                         resid = bp->b_resid;
 583                 }
 584                 kdmsg_free_aux_data(&iost->data);
 585                 break;
 586         case BUF_CMD_FLUSH:
 587         case BUF_CMD_FREEBLKS:
 588                 if (bp->b_flags & B_ERROR)
 589                         error = bp->b_error;
 590                 else
 591                         error = 0;
 592                 break;
 593         default:
 594                 panic("diskiodone: Unknown bio cmd = %d\n",
 595                       bio->bio_buf->b_cmd);
 596                 error = 0;      /* avoid compiler warning */
 597                 break;          /* NOT REACHED */
 598         }
 599
 600         /*
 601          * Convert error to DMSG_ERR_* code.
 602          */
 603         if (error)
 604                 error = DMSG_ERR_IO;
 605
 606         /*
 607          * Convert LNK_ERROR or BLK_ERROR if non-zero resid.  READS will
 608          * have already converted cmd to BLK_ERROR and set up data to return.
 609          */
 610         if (resid && cmd == DMSG_LNK_ERROR)
 611                 cmd = DMSG_BLK_ERROR;
 612         /* XXX txcmd is delayed so this won't work for streaming */
 613         if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */
 614                 cmd |= DMSGF_CREATE;
 615         if (iost->eof) {
 616                 if (atomic_fetchadd_int(&iost->count, -1) == 1)
 617                         cmd |= DMSGF_DELETE;
 618         } else {
 619                 atomic_add_int(&iost->count, -1);
 620         }
 621         cmd |= DMSGF_REPLY;
 622
 623         /*
 624          * Allocate a basic or extended reply.  Be careful not to populate
 625          * extended header fields unless we allocated an extended reply.
 626          */
 627         rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0);
 628         if (data) {
 629                 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT);
 630                 rmsg->aux_size = bytes;
 631                 rmsg->flags |= KDMSG_FLAG_AUXALLOC;
 632                 bcopy(data, rmsg->aux_data, bytes);
 633         }
 634         rmsg->any.blk_error.head.error = error;
 635         if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR)
 636                 rmsg->any.blk_error.resid = resid;
 637         bio->bio_caller_info1.ptr = NULL;
 638         /* kdmsg_state_drop(state); */
 639         kdmsg_msg_write(rmsg);
 640         if (bp->b_flags & B_PAGING) {
 641                 relpbuf(bio->bio_buf, NULL);
 642         } else {
 643                 bp->b_flags |= B_INVAL | B_AGE;
 644                 brelse(bp);
 645         }
 646 }