sys/kern/subr_diskiocom.c

   1 /*
   2  * Copyright (c) 2012 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/proc.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/buf.h>
  40 #include <sys/conf.h>
  41 #include <sys/disklabel.h>
  42 #include <sys/disklabel32.h>
  43 #include <sys/disklabel64.h>
  44 #include <sys/diskslice.h>
  45 #include <sys/diskmbr.h>
  46 #include <sys/disk.h>
  47 #include <sys/malloc.h>
  48 #include <sys/device.h>
  49 #include <sys/devfs.h>
  50 #include <sys/thread.h>
  51 #include <sys/queue.h>
  52 #include <sys/lock.h>
  53 #include <sys/stat.h>
  54 #include <sys/uuid.h>
  55
  56 #include <vfs/hammer2/hammer2_disk.h>
  57 #include <sys/dmsg.h>
  58
  59 #include <sys/buf2.h>
  60 #include <sys/mplock2.h>
  61 #include <sys/msgport2.h>
  62 #include <sys/thread2.h>
  63
  64 struct dios_open {
  65         int     openrd;
  66         int     openwr;
  67 };
  68
  69 struct dios_io {
  70         int     count;
  71         int     eof;
  72 };
  73
  74 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg");
  75
  76 static int disk_iocom_reconnect(struct disk *dp, struct file *fp);
  77 static int disk_rcvdmsg(kdmsg_msg_t *msg);
  78
  79 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg);
  80 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg);
  81 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg);
  82 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg);
  83 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg);
  84 static void diskiodone(struct bio *bio);
  85
  86 void
  87 disk_iocom_init(struct disk *dp)
  88 {
  89         kdmsg_iocom_init(&dp->d_iocom, dp,
  90                          KDMSG_IOCOMF_AUTOCONN |
  91                          KDMSG_IOCOMF_AUTOSPAN |
  92                          KDMSG_IOCOMF_AUTOCIRC,
  93                          M_DMSG_DISK, disk_rcvdmsg);
  94 }
  95
  96 void
  97 disk_iocom_update(struct disk *dp)
  98 {
  99 }
 100
 101 void
 102 disk_iocom_uninit(struct disk *dp)
 103 {
 104         kdmsg_iocom_uninit(&dp->d_iocom);
 105 }
 106
 107 int
 108 disk_iocom_ioctl(struct disk *dp, int cmd, void *data)
 109 {
 110         struct file *fp;
 111         struct disk_ioc_recluster *recl;
 112         int error;
 113
 114         switch(cmd) {
 115         case DIOCRECLUSTER:
 116                 recl = data;
 117                 fp = holdfp(curproc->p_fd, recl->fd, -1);
 118                 if (fp) {
 119                         error = disk_iocom_reconnect(dp, fp);
 120                 } else {
 121                         error = EINVAL;
 122                 }
 123                 break;
 124         default:
 125                 error = EOPNOTSUPP;
 126                 break;
 127         }
 128         return error;
 129 }
 130
 131 static
 132 int
 133 disk_iocom_reconnect(struct disk *dp, struct file *fp)
 134 {
 135         char devname[64];
 136
 137         ksnprintf(devname, sizeof(devname), "%s%d",
 138                   dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev));
 139
 140         kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname);
 141
 142         dp->d_iocom.auto_lnk_conn.pfs_type = DMSG_PFSTYPE_SERVER;
 143         dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1;
 144         dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK;
 145         dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK;
 146         dp->d_iocom.auto_lnk_conn.pfs_mask = (uint64_t)-1;
 147         ksnprintf(dp->d_iocom.auto_lnk_conn.cl_label,
 148                   sizeof(dp->d_iocom.auto_lnk_conn.cl_label),
 149                   "%s/%s", hostname, devname);
 150         if (dp->d_info.d_serialno) {
 151                 ksnprintf(dp->d_iocom.auto_lnk_conn.fs_label,
 152                           sizeof(dp->d_iocom.auto_lnk_conn.fs_label),
 153                           "%s", dp->d_info.d_serialno);
 154         }
 155
 156         dp->d_iocom.auto_lnk_span.pfs_type = DMSG_PFSTYPE_SERVER;
 157         dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1;
 158         dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK;
 159         dp->d_iocom.auto_lnk_span.media.block.bytes =
 160                                                 dp->d_info.d_media_size;
 161         dp->d_iocom.auto_lnk_span.media.block.blksize =
 162                                                 dp->d_info.d_media_blksize;
 163         ksnprintf(dp->d_iocom.auto_lnk_span.cl_label,
 164                   sizeof(dp->d_iocom.auto_lnk_span.cl_label),
 165                   "%s/%s", hostname, devname);
 166         if (dp->d_info.d_serialno) {
 167                 ksnprintf(dp->d_iocom.auto_lnk_span.fs_label,
 168                           sizeof(dp->d_iocom.auto_lnk_span.fs_label),
 169                           "%s", dp->d_info.d_serialno);
 170         }
 171
 172         kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL);
 173
 174         return (0);
 175 }
 176
 177 int
 178 disk_rcvdmsg(kdmsg_msg_t *msg)
 179 {
 180         struct disk *dp = msg->iocom->handle;
 181
 182         /*
 183          * Handle debug messages (these might not be in transactions)
 184          */
 185         switch(msg->any.head.cmd & DMSGF_CMDSWMASK) {
 186         case DMSG_DBG_SHELL:
 187                 /*
 188                  * Execute shell command (not supported atm)
 189                  */
 190                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 191                 return(0);
 192         case DMSG_DBG_SHELL | DMSGF_REPLY:
 193                 if (msg->aux_data) {
 194                         msg->aux_data[msg->aux_size - 1] = 0;
 195                         kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data);
 196                 }
 197                 return(0);
 198         }
 199
 200         /*
 201          * All remaining messages must be in a transaction
 202          *
 203          * NOTE!  We are switching on the first message's command.  The
 204          *        actual message command within the transaction may be
 205          *        different (if streaming within a transaction).
 206          */
 207         if (msg->state == NULL) {
 208                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 209                 return(0);
 210         }
 211
 212         switch(msg->state->rxcmd & DMSGF_CMDSWMASK) {
 213         case DMSG_BLK_OPEN:
 214         case DMSG_BLK_CLOSE:
 215                 disk_blk_open(dp, msg);
 216                 break;
 217         case DMSG_BLK_READ:
 218                 disk_blk_read(dp, msg);
 219                 break;
 220         case DMSG_BLK_WRITE:
 221                 disk_blk_write(dp, msg);
 222                 break;
 223         case DMSG_BLK_FLUSH:
 224                 disk_blk_flush(dp, msg);
 225                 break;
 226         case DMSG_BLK_FREEBLKS:
 227                 disk_blk_freeblks(dp, msg);
 228                 break;
 229         default:
 230                 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) {
 231                         if (msg->any.head.cmd & DMSGF_DELETE)
 232                                 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
 233                         else
 234                                 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP);
 235                 }
 236                 break;
 237         }
 238         return (0);
 239 }
 240
 241 static
 242 void
 243 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg)
 244 {
 245         struct dios_open *openst;
 246         int error = DMSG_ERR_NOSUPP;
 247         int fflags;
 248
 249         openst = msg->state->any.any;
 250         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) {
 251                 if (openst == NULL) {
 252                         openst = kmalloc(sizeof(*openst), M_DEVBUF,
 253                                                 M_WAITOK | M_ZERO);
 254                         msg->state->any.any = openst;
 255                 }
 256                 fflags = 0;
 257                 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 258                         fflags = FREAD;
 259                 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 260                         fflags |= FWRITE;
 261                 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR, proc0.p_ucred, NULL);
 262                 if (error) {
 263                         error = DMSG_ERR_IO;
 264                 } else {
 265                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 266                                 ++openst->openrd;
 267                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 268                                 ++openst->openwr;
 269                 }
 270         }
 271         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE &&
 272             openst) {
 273                 fflags = 0;
 274                 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) &&
 275                     openst->openrd) {
 276                         fflags = FREAD;
 277                 }
 278                 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) &&
 279                     openst->openwr) {
 280                         fflags |= FWRITE;
 281                 }
 282                 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL);
 283                 if (error) {
 284                         error = DMSG_ERR_IO;
 285                 } else {
 286                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
 287                                 --openst->openrd;
 288                         if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
 289                                 --openst->openwr;
 290                 }
 291         }
 292         if (msg->any.head.cmd & DMSGF_DELETE) {
 293                 if (openst) {
 294                         while (openst->openrd && openst->openwr) {
 295                                 --openst->openrd;
 296                                 --openst->openwr;
 297                                 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL);
 298                         }
 299                         while (openst->openrd) {
 300                                 --openst->openrd;
 301                                 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL);
 302                         }
 303                         while (openst->openwr) {
 304                                 --openst->openwr;
 305                                 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL);
 306                         }
 307                         kfree(openst, M_DEVBUF);
 308                         msg->state->any.any = NULL;
 309                 }
 310                 kdmsg_msg_reply(msg, error);
 311         } else {
 312                 kdmsg_msg_result(msg, error);
 313         }
 314 }
 315
 316 static
 317 void
 318 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg)
 319 {
 320         struct dios_io *iost;
 321         struct buf *bp;
 322         struct bio *bio;
 323         int error = DMSG_ERR_NOSUPP;
 324         int reterr = 1;
 325
 326         /*
 327          * Only DMSG_BLK_READ commands imply read ops.
 328          */
 329         iost = msg->state->any.any;
 330         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) {
 331                 if (msg->any.blk_read.bytes < DEV_BSIZE ||
 332                     msg->any.blk_read.bytes > MAXPHYS) {
 333                         error = DMSG_ERR_PARAM;
 334                         goto done;
 335                 }
 336                 if (iost == NULL) {
 337                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 338                                        M_WAITOK | M_ZERO);
 339                         msg->state->any.any = iost;
 340                 }
 341                 reterr = 0;
 342                 bp = geteblk(msg->any.blk_read.bytes);
 343                 bio = &bp->b_bio1;
 344                 bp->b_cmd = BUF_CMD_READ;
 345                 bp->b_bcount = msg->any.blk_read.bytes;
 346                 bp->b_resid = bp->b_bcount;
 347                 bio->bio_offset = msg->any.blk_read.offset;
 348                 bio->bio_caller_info1.ptr = msg->state;
 349                 bio->bio_done = diskiodone;
 350                 /* kdmsg_state_hold(msg->state); */
 351
 352                 atomic_add_int(&iost->count, 1);
 353                 if (msg->any.head.cmd & DMSGF_DELETE)
 354                         iost->eof = 1;
 355                 BUF_KERNPROC(bp);
 356                 dev_dstrategy(dp->d_rawdev, bio);
 357         }
 358 done:
 359         if (reterr) {
 360                 if (msg->any.head.cmd & DMSGF_DELETE) {
 361                         if (iost && iost->count == 0) {
 362                                 kfree(iost, M_DEVBUF);
 363                                 msg->state->any.any = NULL;
 364                         }
 365                         kdmsg_msg_reply(msg, error);
 366                 } else {
 367                         kdmsg_msg_result(msg, error);
 368                 }
 369         }
 370 }
 371
 372 static
 373 void
 374 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg)
 375 {
 376         struct dios_io *iost;
 377         struct buf *bp;
 378         struct bio *bio;
 379         int error = DMSG_ERR_NOSUPP;
 380         int reterr = 1;
 381
 382         /*
 383          * Only DMSG_BLK_WRITE commands imply read ops.
 384          */
 385         iost = msg->state->any.any;
 386         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) {
 387                 if (msg->any.blk_write.bytes < DEV_BSIZE ||
 388                     msg->any.blk_write.bytes > MAXPHYS) {
 389                         error = DMSG_ERR_PARAM;
 390                         goto done;
 391                 }
 392                 if (iost == NULL) {
 393                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 394                                        M_WAITOK | M_ZERO);
 395                         msg->state->any.any = iost;
 396                 }
 397
 398                 /*
 399                  * Issue WRITE.  Short data implies zeros.  Try to optimize
 400                  * the buffer cache buffer for the case where we can just
 401                  * use the message's data pointer.
 402                  */
 403                 reterr = 0;
 404                 if (msg->aux_size >= msg->any.blk_write.bytes)
 405                         bp = getpbuf(NULL);
 406                 else
 407                         bp = geteblk(msg->any.blk_write.bytes);
 408                 bio = &bp->b_bio1;
 409                 bp->b_cmd = BUF_CMD_WRITE;
 410                 bp->b_bcount = msg->any.blk_write.bytes;
 411                 bp->b_resid = bp->b_bcount;
 412                 if (msg->aux_size >= msg->any.blk_write.bytes) {
 413                         bp->b_data = msg->aux_data;
 414                 } else {
 415                         bcopy(msg->aux_data, bp->b_data, msg->aux_size);
 416                         bzero(bp->b_data + msg->aux_size,
 417                               msg->any.blk_write.bytes - msg->aux_size);
 418                 }
 419                 bio->bio_offset = msg->any.blk_write.offset;
 420                 bio->bio_caller_info1.ptr = msg->state;
 421                 bio->bio_done = diskiodone;
 422                 /* kdmsg_state_hold(msg->state); */
 423
 424                 atomic_add_int(&iost->count, 1);
 425                 if (msg->any.head.cmd & DMSGF_DELETE)
 426                         iost->eof = 1;
 427                 BUF_KERNPROC(bp);
 428                 dev_dstrategy(dp->d_rawdev, bio);
 429         }
 430 done:
 431         if (reterr) {
 432                 if (msg->any.head.cmd & DMSGF_DELETE) {
 433                         if (iost && iost->count == 0) {
 434                                 kfree(iost, M_DEVBUF);
 435                                 msg->state->any.any = NULL;
 436                         }
 437                         kdmsg_msg_reply(msg, error);
 438                 } else {
 439                         kdmsg_msg_result(msg, error);
 440                 }
 441         }
 442 }
 443
 444 static
 445 void
 446 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg)
 447 {
 448         struct dios_io *iost;
 449         struct buf *bp;
 450         struct bio *bio;
 451         int error = DMSG_ERR_NOSUPP;
 452         int reterr = 1;
 453
 454         /*
 455          * Only DMSG_BLK_FLUSH commands imply read ops.
 456          */
 457         iost = msg->state->any.any;
 458         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) {
 459                 if (iost == NULL) {
 460                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 461                                        M_WAITOK | M_ZERO);
 462                         msg->state->any.any = iost;
 463                 }
 464                 reterr = 0;
 465                 bp = getpbuf(NULL);
 466                 bio = &bp->b_bio1;
 467                 bp->b_cmd = BUF_CMD_FLUSH;
 468                 bp->b_bcount = msg->any.blk_flush.bytes;
 469                 bp->b_resid = 0;
 470                 bio->bio_offset = msg->any.blk_flush.offset;
 471                 bio->bio_caller_info1.ptr = msg->state;
 472                 bio->bio_done = diskiodone;
 473                 /* kdmsg_state_hold(msg->state); */
 474
 475                 atomic_add_int(&iost->count, 1);
 476                 if (msg->any.head.cmd & DMSGF_DELETE)
 477                         iost->eof = 1;
 478                 BUF_KERNPROC(bp);
 479                 dev_dstrategy(dp->d_rawdev, bio);
 480         }
 481         if (reterr) {
 482                 if (msg->any.head.cmd & DMSGF_DELETE) {
 483                         if (iost && iost->count == 0) {
 484                                 kfree(iost, M_DEVBUF);
 485                                 msg->state->any.any = NULL;
 486                         }
 487                         kdmsg_msg_reply(msg, error);
 488                 } else {
 489                         kdmsg_msg_result(msg, error);
 490                 }
 491         }
 492 }
 493
 494 static
 495 void
 496 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg)
 497 {
 498         struct dios_io *iost;
 499         struct buf *bp;
 500         struct bio *bio;
 501         int error = DMSG_ERR_NOSUPP;
 502         int reterr = 1;
 503
 504         /*
 505          * Only DMSG_BLK_FREEBLKS commands imply read ops.
 506          */
 507         iost = msg->state->any.any;
 508         if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) {
 509                 if (iost == NULL) {
 510                         iost = kmalloc(sizeof(*iost), M_DEVBUF,
 511                                        M_WAITOK | M_ZERO);
 512                         msg->state->any.any = iost;
 513                 }
 514                 reterr = 0;
 515                 bp = getpbuf(NULL);
 516                 bio = &bp->b_bio1;
 517                 bp->b_cmd = BUF_CMD_FREEBLKS;
 518                 bp->b_bcount = msg->any.blk_freeblks.bytes;
 519                 bp->b_resid = 0;
 520                 bio->bio_offset = msg->any.blk_freeblks.offset;
 521                 bio->bio_caller_info1.ptr = msg->state;
 522                 bio->bio_done = diskiodone;
 523                 /* kdmsg_state_hold(msg->state); */
 524
 525                 atomic_add_int(&iost->count, 1);
 526                 if (msg->any.head.cmd & DMSGF_DELETE)
 527                         iost->eof = 1;
 528                 BUF_KERNPROC(bp);
 529                 dev_dstrategy(dp->d_rawdev, bio);
 530         }
 531         if (reterr) {
 532                 if (msg->any.head.cmd & DMSGF_DELETE) {
 533                         if (iost && iost->count == 0) {
 534                                 kfree(iost, M_DEVBUF);
 535                                 msg->state->any.any = NULL;
 536                         }
 537                         kdmsg_msg_reply(msg, error);
 538                 } else {
 539                         kdmsg_msg_result(msg, error);
 540                 }
 541         }
 542 }
 543
 544 static
 545 void
 546 diskiodone(struct bio *bio)
 547 {
 548         struct buf *bp = bio->bio_buf;
 549         kdmsg_state_t *state = bio->bio_caller_info1.ptr;
 550         kdmsg_msg_t *rmsg;
 551         struct dios_io *iost = state->any.any;
 552         int error;
 553         int resid = 0;
 554         int bytes;
 555         uint32_t cmd;
 556         void *data;
 557
 558         cmd = DMSG_LNK_ERROR;
 559         data = NULL;
 560         bytes = 0;
 561
 562         switch(bp->b_cmd) {
 563         case BUF_CMD_READ:
 564                 cmd = DMSG_LNK_ERROR;
 565                 data = bp->b_data;
 566                 bytes = bp->b_bcount;
 567                 /* fall through */
 568         case BUF_CMD_WRITE:
 569                 if (bp->b_flags & B_ERROR) {
 570                         error = bp->b_error;
 571                 } else {
 572                         error = 0;
 573                         resid = bp->b_resid;
 574                 }
 575                 break;
 576         case BUF_CMD_FLUSH:
 577         case BUF_CMD_FREEBLKS:
 578                 if (bp->b_flags & B_ERROR)
 579                         error = bp->b_error;
 580                 else
 581                         error = 0;
 582                 break;
 583         default:
 584                 panic("diskiodone: Unknown bio cmd = %d\n",
 585                       bio->bio_buf->b_cmd);
 586                 error = 0;      /* avoid compiler warning */
 587                 break;          /* NOT REACHED */
 588         }
 589
 590         /*
 591          * Convert error to DMSG_ERR_* code.
 592          */
 593         if (error)
 594                 error = DMSG_ERR_IO;
 595
 596         /*
 597          * Convert LNK_ERROR or BLK_ERROR if non-zero resid.  READS will
 598          * have already converted cmd to BLK_ERROR and set up data to return.
 599          */
 600         if (resid && cmd == DMSG_LNK_ERROR)
 601                 cmd = DMSG_BLK_ERROR;
 602         /* XXX txcmd is delayed so this won't work for streaming */
 603         if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */
 604                 cmd |= DMSGF_CREATE;
 605         if (iost->eof) {
 606                 if (atomic_fetchadd_int(&iost->count, -1) == 1)
 607                         cmd |= DMSGF_DELETE;
 608         } else {
 609                 atomic_add_int(&iost->count, -1);
 610         }
 611         cmd |= DMSGF_REPLY;
 612
 613         /*
 614          * Allocate a basic or extended reply.  Be careful not to populate
 615          * extended header fields unless we allocated an extended reply.
 616          */
 617         rmsg = kdmsg_msg_alloc_state(state, cmd, NULL, 0);
 618         if (data) {
 619                 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT);
 620                 rmsg->aux_size = bytes;
 621                 rmsg->flags |= KDMSG_FLAG_AUXALLOC;
 622                 bcopy(data, rmsg->aux_data, bytes);
 623         }
 624         rmsg->any.blk_error.head.error = error;
 625         if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR)
 626                 rmsg->any.blk_error.resid = resid;
 627         bio->bio_caller_info1.ptr = NULL;
 628         /* kdmsg_state_drop(state); */
 629         kdmsg_msg_write(rmsg);
 630         if (bp->b_flags & B_PAGING) {
 631                 relpbuf(bio->bio_buf, NULL);
 632         } else {
 633                 bp->b_flags |= B_INVAL | B_AGE;
 634                 brelse(bp);
 635         }
 636 }