sys/vfs/hammer2/hammer2_strategy.c

   1 /*
   2  * Copyright (c) 2011-2015 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@dragonflybsd.org>
   6  * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
   7  * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  *
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  * 3. Neither the name of The DragonFly Project nor the names of its
  20  *    contributors may be used to endorse or promote products derived
  21  *    from this software without specific, prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  26  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  27  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  28  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  29  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  30  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  31  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  32  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  33  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  */
  36 /*
  37  * This module handles low level logical file I/O (strategy) which backs
  38  * the logical buffer cache.
  39  */
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #include <sys/kernel.h>
  44 #include <sys/fcntl.h>
  45 #include <sys/buf.h>
  46 #include <sys/proc.h>
  47 #include <sys/namei.h>
  48 #include <sys/mount.h>
  49 #include <sys/vnode.h>
  50 #include <sys/mountctl.h>
  51 #include <sys/dirent.h>
  52 #include <sys/uio.h>
  53 #include <sys/objcache.h>
  54 #include <sys/event.h>
  55 #include <sys/file.h>
  56 #include <vfs/fifofs/fifo.h>
  57
  58 #include "hammer2.h"
  59 #include "hammer2_lz4.h"
  60
  61 #include "zlib/hammer2_zlib.h"
  62
  63 struct objcache *cache_buffer_read;
  64 struct objcache *cache_buffer_write;
  65
  66 /*
  67  * Strategy code (async logical file buffer I/O from system)
  68  *
  69  * WARNING: The strategy code cannot safely use hammer2 transactions
  70  *          as this can deadlock against vfs_sync's vfsync() call
  71  *          if multiple flushes are queued.  All H2 structures must
  72  *          already be present and ready for the DIO.
  73  *
  74  *          Reads can be initiated asynchronously, writes have to be
  75  *          spooled to a separate thread for action to avoid deadlocks.
  76  */
  77 static int hammer2_strategy_read(struct vop_strategy_args *ap);
  78 static int hammer2_strategy_write(struct vop_strategy_args *ap);
  79 static void hammer2_strategy_read_callback(hammer2_iocb_t *iocb);
  80
  81 int
  82 hammer2_vop_strategy(struct vop_strategy_args *ap)
  83 {
  84         struct bio *biop;
  85         struct buf *bp;
  86         int error;
  87
  88         biop = ap->a_bio;
  89         bp = biop->bio_buf;
  90
  91         switch(bp->b_cmd) {
  92         case BUF_CMD_READ:
  93                 error = hammer2_strategy_read(ap);
  94                 ++hammer2_iod_file_read;
  95                 break;
  96         case BUF_CMD_WRITE:
  97                 error = hammer2_strategy_write(ap);
  98                 ++hammer2_iod_file_write;
  99                 break;
 100         default:
 101                 bp->b_error = error = EINVAL;
 102                 bp->b_flags |= B_ERROR;
 103                 biodone(biop);
 104                 break;
 105         }
 106         return (error);
 107 }
 108
 109 /*
 110  * Return the largest contiguous physical disk range for the logical
 111  * request, in bytes.
 112  *
 113  * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
 114  *
 115  * Basically disabled, the logical buffer write thread has to deal with
 116  * buffers one-at-a-time.
 117  */
 118 int
 119 hammer2_vop_bmap(struct vop_bmap_args *ap)
 120 {
 121         *ap->a_doffsetp = NOOFFSET;
 122         if (ap->a_runp)
 123                 *ap->a_runp = 0;
 124         if (ap->a_runb)
 125                 *ap->a_runb = 0;
 126         return (EOPNOTSUPP);
 127 }
 128
 129 /****************************************************************************
 130  *                              READ SUPPORT                                *
 131  ****************************************************************************/
 132 /*
 133  * Callback used in read path in case that a block is compressed with LZ4.
 134  */
 135 static
 136 void
 137 hammer2_decompress_LZ4_callback(const char *data, u_int bytes, struct bio *bio)
 138 {
 139         struct buf *bp;
 140         char *compressed_buffer;
 141         int compressed_size;
 142         int result;
 143
 144         bp = bio->bio_buf;
 145
 146 #if 0
 147         if bio->bio_caller_info2.index &&
 148               bio->bio_caller_info1.uvalue32 !=
 149               crc32(bp->b_data, bp->b_bufsize) --- return error
 150 #endif
 151
 152         KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
 153         compressed_size = *(const int *)data;
 154         KKASSERT(compressed_size <= bytes - sizeof(int));
 155
 156         compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
 157         result = LZ4_decompress_safe(__DECONST(char *, &data[sizeof(int)]),
 158                                      compressed_buffer,
 159                                      compressed_size,
 160                                      bp->b_bufsize);
 161         if (result < 0) {
 162                 kprintf("READ PATH: Error during decompression."
 163                         "bio %016jx/%d\n",
 164                         (intmax_t)bio->bio_offset, bytes);
 165                 /* make sure it isn't random garbage */
 166                 bzero(compressed_buffer, bp->b_bufsize);
 167         }
 168         KKASSERT(result <= bp->b_bufsize);
 169         bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
 170         if (result < bp->b_bufsize)
 171                 bzero(bp->b_data + result, bp->b_bufsize - result);
 172         objcache_put(cache_buffer_read, compressed_buffer);
 173         bp->b_resid = 0;
 174         bp->b_flags |= B_AGE;
 175 }
 176
 177 /*
 178  * Callback used in read path in case that a block is compressed with ZLIB.
 179  * It is almost identical to LZ4 callback, so in theory they can be unified,
 180  * but we didn't want to make changes in bio structure for that.
 181  */
 182 static
 183 void
 184 hammer2_decompress_ZLIB_callback(const char *data, u_int bytes, struct bio *bio)
 185 {
 186         struct buf *bp;
 187         char *compressed_buffer;
 188         z_stream strm_decompress;
 189         int result;
 190         int ret;
 191
 192         bp = bio->bio_buf;
 193
 194         KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
 195         strm_decompress.avail_in = 0;
 196         strm_decompress.next_in = Z_NULL;
 197
 198         ret = inflateInit(&strm_decompress);
 199
 200         if (ret != Z_OK)
 201                 kprintf("HAMMER2 ZLIB: Fatal error in inflateInit.\n");
 202
 203         compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
 204         strm_decompress.next_in = __DECONST(char *, data);
 205
 206         /* XXX supply proper size, subset of device bp */
 207         strm_decompress.avail_in = bytes;
 208         strm_decompress.next_out = compressed_buffer;
 209         strm_decompress.avail_out = bp->b_bufsize;
 210
 211         ret = inflate(&strm_decompress, Z_FINISH);
 212         if (ret != Z_STREAM_END) {
 213                 kprintf("HAMMER2 ZLIB: Fatar error during decompression.\n");
 214                 bzero(compressed_buffer, bp->b_bufsize);
 215         }
 216         bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
 217         result = bp->b_bufsize - strm_decompress.avail_out;
 218         if (result < bp->b_bufsize)
 219                 bzero(bp->b_data + result, strm_decompress.avail_out);
 220         objcache_put(cache_buffer_read, compressed_buffer);
 221         ret = inflateEnd(&strm_decompress);
 222
 223         bp->b_resid = 0;
 224         bp->b_flags |= B_AGE;
 225 }
 226
 227 /*
 228  * Logical buffer I/O, async read.
 229  */
 230 static
 231 int
 232 hammer2_strategy_read(struct vop_strategy_args *ap)
 233 {
 234         struct buf *bp;
 235         struct bio *bio;
 236         struct bio *nbio;
 237         hammer2_inode_t *ip;
 238         hammer2_cluster_t *cparent;
 239         hammer2_cluster_t *cluster;
 240         hammer2_key_t key_dummy;
 241         hammer2_key_t lbase;
 242         uint8_t btype;
 243
 244         bio = ap->a_bio;
 245         bp = bio->bio_buf;
 246         ip = VTOI(ap->a_vp);
 247         nbio = push_bio(bio);
 248
 249         lbase = bio->bio_offset;
 250         KKASSERT(((int)lbase & HAMMER2_PBUFMASK) == 0);
 251
 252         /*
 253          * Lookup the file offset.
 254          */
 255         hammer2_inode_lock(ip, HAMMER2_RESOLVE_ALWAYS |
 256                                HAMMER2_RESOLVE_SHARED);
 257         cparent = hammer2_inode_cluster(ip, HAMMER2_RESOLVE_ALWAYS |
 258                                             HAMMER2_RESOLVE_SHARED);
 259         cluster = hammer2_cluster_lookup(cparent, &key_dummy,
 260                                        lbase, lbase,
 261                                        HAMMER2_LOOKUP_NODATA |
 262                                        HAMMER2_LOOKUP_SHARED);
 263         hammer2_inode_unlock(ip, cparent);
 264
 265         /*
 266          * Data is zero-fill if no cluster could be found
 267          * (XXX or EIO on a cluster failure).
 268          */
 269         if (cluster == NULL) {
 270                 bp->b_resid = 0;
 271                 bp->b_error = 0;
 272                 bzero(bp->b_data, bp->b_bcount);
 273                 biodone(nbio);
 274                 return(0);
 275         }
 276
 277         /*
 278          * Cluster elements must be type INODE or type DATA, but the
 279          * compression mode (or not) for DATA chains can be different for
 280          * each chain.  This will be handled by the callback.
 281          *
 282          * If the cluster already has valid data the callback will be made
 283          * immediately/synchronously.
 284          */
 285         btype = hammer2_cluster_type(cluster);
 286         if (btype != HAMMER2_BREF_TYPE_INODE &&
 287             btype != HAMMER2_BREF_TYPE_DATA) {
 288                 panic("READ PATH: hammer2_strategy_read: unknown bref type");
 289         }
 290         hammer2_cluster_load_async(cluster, hammer2_strategy_read_callback,
 291                                    nbio);
 292         return(0);
 293 }
 294
 295 /*
 296  * Read callback for hammer2_cluster_load_async().  The load function may
 297  * start several actual I/Os but will only make one callback, typically with
 298  * the first valid I/O XXX
 299  */
 300 static
 301 void
 302 hammer2_strategy_read_callback(hammer2_iocb_t *iocb)
 303 {
 304         struct bio *bio = iocb->ptr;    /* original logical buffer */
 305         struct buf *bp = bio->bio_buf;  /* original logical buffer */
 306         hammer2_chain_t *chain;
 307         hammer2_cluster_t *cluster;
 308         hammer2_io_t *dio;
 309         char *data;
 310         int i;
 311
 312         /*
 313          * Extract data and handle iteration on I/O failure.  iocb->off
 314          * is the cluster index for iteration.
 315          */
 316         cluster = iocb->cluster;
 317         dio = iocb->dio;        /* can be NULL if iocb not in progress */
 318
 319         /*
 320          * Work to do if INPROG set, else dio is already good or dio is
 321          * NULL (which is the shortcut case if chain->data is already good).
 322          */
 323         if (iocb->flags & HAMMER2_IOCB_INPROG) {
 324                 /*
 325                  * Read attempt not yet made.  Issue an asynchronous read
 326                  * if necessary and return, operation will chain back to
 327                  * this function.
 328                  */
 329                 if ((iocb->flags & HAMMER2_IOCB_READ) == 0) {
 330                         if (dio->bp == NULL ||
 331                             (dio->bp->b_flags & B_CACHE) == 0) {
 332                                 if (dio->bp) {
 333                                         bqrelse(dio->bp);
 334                                         dio->bp = NULL;
 335                                 }
 336                                 iocb->flags |= HAMMER2_IOCB_READ;
 337                                 breadcb(dio->hmp->devvp,
 338                                         dio->pbase, dio->psize,
 339                                         hammer2_io_callback, iocb);
 340                                 return;
 341                         }
 342                 }
 343         }
 344
 345         /*
 346          * If we have a DIO it is now done, check for an error and
 347          * calculate the data.
 348          *
 349          * If there is no DIO it is an optimization by
 350          * hammer2_cluster_load_async(), the data is available in
 351          * chain->data.
 352          */
 353         if (dio) {
 354                 if (dio->bp->b_flags & B_ERROR) {
 355                         i = (int)iocb->lbase + 1;
 356                         if (i >= cluster->nchains) {
 357                                 bp->b_flags |= B_ERROR;
 358                                 bp->b_error = dio->bp->b_error;
 359                                 hammer2_io_complete(iocb);
 360                                 biodone(bio);
 361                                 hammer2_cluster_unlock(cluster);
 362                                 hammer2_cluster_drop(cluster);
 363                         } else {
 364                                 hammer2_io_complete(iocb); /* XXX */
 365                                 chain = cluster->array[i].chain;
 366                                 kprintf("hammer2: IO CHAIN-%d %p\n", i, chain);
 367                                 hammer2_adjreadcounter(&chain->bref,
 368                                                        chain->bytes);
 369                                 iocb->chain = chain;
 370                                 iocb->lbase = (off_t)i;
 371                                 iocb->flags = 0;
 372                                 iocb->error = 0;
 373                                 hammer2_io_getblk(chain->hmp,
 374                                                   chain->bref.data_off,
 375                                                   chain->bytes,
 376                                                   iocb);
 377                         }
 378                         return;
 379                 }
 380                 chain = iocb->chain;
 381                 data = hammer2_io_data(dio, chain->bref.data_off);
 382         } else {
 383                 /*
 384                  * Special synchronous case, data present in chain->data.
 385                  */
 386                 chain = iocb->chain;
 387                 data = (void *)chain->data;
 388         }
 389
 390         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
 391                 /*
 392                  * Data is embedded in the inode (copy from inode).
 393                  */
 394                 bcopy(((hammer2_inode_data_t *)data)->u.data,
 395                       bp->b_data, HAMMER2_EMBEDDED_BYTES);
 396                 bzero(bp->b_data + HAMMER2_EMBEDDED_BYTES,
 397                       bp->b_bcount - HAMMER2_EMBEDDED_BYTES);
 398                 bp->b_resid = 0;
 399                 bp->b_error = 0;
 400         } else if (chain->bref.type == HAMMER2_BREF_TYPE_DATA) {
 401                 /*
 402                  * Data is on-media, issue device I/O and copy.
 403                  *
 404                  * XXX direct-IO shortcut could go here XXX.
 405                  */
 406                 switch (HAMMER2_DEC_COMP(chain->bref.methods)) {
 407                 case HAMMER2_COMP_LZ4:
 408                         hammer2_decompress_LZ4_callback(data, chain->bytes,
 409                                                         bio);
 410                         break;
 411                 case HAMMER2_COMP_ZLIB:
 412                         hammer2_decompress_ZLIB_callback(data, chain->bytes,
 413                                                          bio);
 414                         break;
 415                 case HAMMER2_COMP_NONE:
 416                         KKASSERT(chain->bytes <= bp->b_bcount);
 417                         bcopy(data, bp->b_data, chain->bytes);
 418                         if (chain->bytes < bp->b_bcount) {
 419                                 bzero(bp->b_data + chain->bytes,
 420                                       bp->b_bcount - chain->bytes);
 421                         }
 422                         bp->b_flags |= B_NOTMETA;
 423                         bp->b_resid = 0;
 424                         bp->b_error = 0;
 425                         break;
 426                 default:
 427                         panic("hammer2_strategy_read: "
 428                               "unknown compression type");
 429                 }
 430         } else {
 431                 /* bqrelse the dio to help stabilize the call to panic() */
 432                 if (dio)
 433                         hammer2_io_bqrelse(&dio);
 434                 panic("hammer2_strategy_read: unknown bref type");
 435         }
 436
 437         /*
 438          * Once the iocb is cleaned up the DIO (if any) will no longer be
 439          * in-progress but will still have a ref.  Be sure to release
 440          * the ref.
 441          */
 442         hammer2_io_complete(iocb);              /* physical management */
 443         if (dio)                                /* physical dio & buffer */
 444                 hammer2_io_bqrelse(&dio);
 445         hammer2_cluster_unlock(cluster);        /* cluster management */
 446         hammer2_cluster_drop(cluster);          /* cluster management */
 447         biodone(bio);                           /* logical buffer */
 448 }
 449
 450 /****************************************************************************
 451  *                              WRITE SUPPORT                               *
 452  ****************************************************************************/
 453
 454 /*
 455  * Functions for compression in threads,
 456  * from hammer2_vnops.c
 457  */
 458 static void hammer2_write_file_core(struct buf *bp, hammer2_trans_t *trans,
 459                                 hammer2_inode_t *ip,
 460                                 hammer2_cluster_t *cparent,
 461                                 hammer2_key_t lbase, int ioflag, int pblksize,
 462                                 int *errorp);
 463 static void hammer2_compress_and_write(struct buf *bp, hammer2_trans_t *trans,
 464                                 hammer2_inode_t *ip,
 465                                 hammer2_cluster_t *cparent,
 466                                 hammer2_key_t lbase, int ioflag,
 467                                 int pblksize, int *errorp,
 468                                 int comp_algo, int check_algo);
 469 static void hammer2_zero_check_and_write(struct buf *bp,
 470                                 hammer2_trans_t *trans, hammer2_inode_t *ip,
 471                                 hammer2_cluster_t *cparent,
 472                                 hammer2_key_t lbase,
 473                                 int ioflag, int pblksize, int *errorp,
 474                                 int check_algo);
 475 static int test_block_zeros(const char *buf, size_t bytes);
 476 static void zero_write(struct buf *bp, hammer2_trans_t *trans,
 477                                 hammer2_inode_t *ip,
 478                                 hammer2_cluster_t *cparent,
 479                                 hammer2_key_t lbase,
 480                                 int *errorp);
 481 static void hammer2_write_bp(hammer2_cluster_t *cluster, struct buf *bp,
 482                                 int ioflag, int pblksize, int *errorp,
 483                                 int check_algo);
 484
 485
 486 static
 487 int
 488 hammer2_strategy_write(struct vop_strategy_args *ap)
 489 {
 490         hammer2_pfs_t *pmp;
 491         struct bio *bio;
 492         struct buf *bp;
 493         hammer2_inode_t *ip;
 494
 495         bio = ap->a_bio;
 496         bp = bio->bio_buf;
 497         ip = VTOI(ap->a_vp);
 498         pmp = ip->pmp;
 499
 500         hammer2_lwinprog_ref(pmp);
 501         hammer2_trans_assert_strategy(pmp);
 502         hammer2_mtx_ex(&pmp->wthread_mtx);
 503         if (TAILQ_EMPTY(&pmp->wthread_bioq.queue)) {
 504                 bioq_insert_tail(&pmp->wthread_bioq, ap->a_bio);
 505                 hammer2_mtx_unlock(&pmp->wthread_mtx);
 506                 wakeup(&pmp->wthread_bioq);
 507         } else {
 508                 bioq_insert_tail(&pmp->wthread_bioq, ap->a_bio);
 509                 hammer2_mtx_unlock(&pmp->wthread_mtx);
 510         }
 511         hammer2_lwinprog_wait(pmp);
 512
 513         return(0);
 514 }
 515
 516 /*
 517  * Thread to handle bioq for strategy write (started from hammer2_vfsops.c)
 518  */
 519 void
 520 hammer2_write_thread(void *arg)
 521 {
 522         hammer2_pfs_t *pmp;
 523         struct bio *bio;
 524         struct buf *bp;
 525         hammer2_trans_t trans;
 526         struct vnode *vp;
 527         hammer2_inode_t *ip;
 528         hammer2_cluster_t *cparent;
 529         hammer2_key_t lbase;
 530         int lblksize;
 531         int pblksize;
 532         int error;
 533
 534         pmp = arg;
 535
 536         hammer2_mtx_ex(&pmp->wthread_mtx);
 537         for (;;) {
 538                 /*
 539                  * Wait for work.  Break out and destroy the thread only if
 540                  * requested and no work remains.
 541                  */
 542                 if (bioq_first(&pmp->wthread_bioq) == NULL) {
 543                         if (pmp->wthread_destroy)
 544                                 break;
 545                         mtxsleep(&pmp->wthread_bioq, &pmp->wthread_mtx,
 546                                  0, "h2bioqw", 0);
 547                         continue;
 548                 }
 549
 550                 /*
 551                  * Special transaction for logical buffer cache writes.
 552                  */
 553                 hammer2_trans_init(&trans, pmp, HAMMER2_TRANS_BUFCACHE);
 554
 555                 while ((bio = bioq_takefirst(&pmp->wthread_bioq)) != NULL) {
 556                         /*
 557                          * dummy bio for synchronization.  The transaction
 558                          * must be terminated.
 559                          */
 560                         if (bio->bio_buf == NULL) {
 561                                 bio->bio_flags |= BIO_DONE;
 562                                 /* bio will become invalid after DONE set */
 563                                 wakeup(bio);
 564                                 break;
 565                         }
 566
 567                         /*
 568                          * else normal bio processing
 569                          */
 570                         hammer2_mtx_unlock(&pmp->wthread_mtx);
 571
 572                         hammer2_lwinprog_drop(pmp);
 573
 574                         error = 0;
 575                         bp = bio->bio_buf;
 576                         vp = bp->b_vp;
 577                         ip = VTOI(vp);
 578
 579                         /*
 580                          * Inode is modified, flush size and mtime changes
 581                          * to ensure that the file size remains consistent
 582                          * with the buffers being flushed.
 583                          *
 584                          * NOTE: The inode_fsync() call only flushes the
 585                          *       inode's meta-data state, it doesn't try
 586                          *       to flush underlying buffers or chains.
 587                          *
 588                          * NOTE: hammer2_write_file_core() may indirectly
 589                          *       modify and modsync the inode.
 590                          */
 591                         hammer2_inode_lock(ip, HAMMER2_RESOLVE_ALWAYS);
 592                         cparent = hammer2_inode_cluster(ip,
 593                                                         HAMMER2_RESOLVE_ALWAYS);
 594                         if (ip->flags & HAMMER2_INODE_RESIZED)
 595                                 hammer2_inode_fsync(&trans, ip, cparent);
 596                         lblksize = hammer2_calc_logical(ip, bio->bio_offset,
 597                                                         &lbase, NULL);
 598                         pblksize = hammer2_calc_physical(ip, lbase);
 599                         hammer2_write_file_core(bp, &trans, ip,
 600                                                 cparent,
 601                                                 lbase, IO_ASYNC,
 602                                                 pblksize, &error);
 603                         hammer2_inode_unlock(ip, cparent);
 604                         if (error) {
 605                                 kprintf("hammer2: error in buffer write\n");
 606                                 bp->b_flags |= B_ERROR;
 607                                 bp->b_error = EIO;
 608                         }
 609                         biodone(bio);
 610                         hammer2_mtx_ex(&pmp->wthread_mtx);
 611                 }
 612                 hammer2_trans_done(&trans);
 613         }
 614         pmp->wthread_destroy = -1;
 615         wakeup(&pmp->wthread_destroy);
 616
 617         hammer2_mtx_unlock(&pmp->wthread_mtx);
 618 }
 619
 620 /*
 621  * Wait for pending I/O to complete
 622  */
 623 void
 624 hammer2_bioq_sync(hammer2_pfs_t *pmp)
 625 {
 626         struct bio sync_bio;
 627
 628         bzero(&sync_bio, sizeof(sync_bio));     /* dummy with no bio_buf */
 629         hammer2_mtx_ex(&pmp->wthread_mtx);
 630         if (pmp->wthread_destroy == 0 &&
 631             TAILQ_FIRST(&pmp->wthread_bioq.queue)) {
 632                 bioq_insert_tail(&pmp->wthread_bioq, &sync_bio);
 633                 while ((sync_bio.bio_flags & BIO_DONE) == 0)
 634                         mtxsleep(&sync_bio, &pmp->wthread_mtx, 0, "h2bioq", 0);
 635         }
 636         hammer2_mtx_unlock(&pmp->wthread_mtx);
 637 }
 638
 639 /*
 640  * Create a new cluster at (cparent, lbase) and assign physical storage,
 641  * returning a cluster suitable for I/O.  The cluster will be in a modified
 642  * state.
 643  *
 644  * cparent can wind up being anything.
 645  *
 646  * NOTE: Special case for data embedded in inode.
 647  */
 648 static
 649 hammer2_cluster_t *
 650 hammer2_assign_physical(hammer2_trans_t *trans,
 651                         hammer2_inode_t *ip, hammer2_cluster_t *cparent,
 652                         hammer2_key_t lbase, int pblksize, int *errorp)
 653 {
 654         hammer2_cluster_t *cluster;
 655         hammer2_cluster_t *dparent;
 656         hammer2_key_t key_dummy;
 657         int pradix = hammer2_getradix(pblksize);
 658
 659         /*
 660          * Locate the chain associated with lbase, return a locked chain.
 661          * However, do not instantiate any data reference (which utilizes a
 662          * device buffer) because we will be using direct IO via the
 663          * logical buffer cache buffer.
 664          */
 665         *errorp = 0;
 666         KKASSERT(pblksize >= HAMMER2_ALLOC_MIN);
 667 retry:
 668         dparent = hammer2_cluster_lookup_init(cparent, 0);
 669         cluster = hammer2_cluster_lookup(dparent, &key_dummy,
 670                                      lbase, lbase,
 671                                      HAMMER2_LOOKUP_NODATA);
 672
 673         if (cluster == NULL) {
 674                 /*
 675                  * We found a hole, create a new chain entry.
 676                  *
 677                  * NOTE: DATA chains are created without device backing
 678                  *       store (nor do we want any).
 679                  */
 680                 *errorp = hammer2_cluster_create(trans, dparent, &cluster,
 681                                                lbase, HAMMER2_PBUFRADIX,
 682                                                HAMMER2_BREF_TYPE_DATA,
 683                                                pblksize, 0);
 684                 if (cluster == NULL) {
 685                         hammer2_cluster_lookup_done(dparent);
 686                         panic("hammer2_cluster_create: par=%p error=%d\n",
 687                                 dparent->focus, *errorp);
 688                         goto retry;
 689                 }
 690                 /*ip->delta_dcount += pblksize;*/
 691         } else {
 692                 switch (hammer2_cluster_type(cluster)) {
 693                 case HAMMER2_BREF_TYPE_INODE:
 694                         /*
 695                          * The data is embedded in the inode, which requires
 696                          * a bit more finess.
 697                          */
 698                         hammer2_cluster_modify_ip(trans, ip, cluster, 0);
 699                         break;
 700                 case HAMMER2_BREF_TYPE_DATA:
 701                         if (hammer2_cluster_need_resize(cluster, pblksize)) {
 702                                 hammer2_cluster_resize(trans, ip,
 703                                                      dparent, cluster,
 704                                                      pradix,
 705                                                      HAMMER2_MODIFY_OPTDATA);
 706                         }
 707
 708                         /*
 709                          * DATA buffers must be marked modified whether the
 710                          * data is in a logical buffer or not.  We also have
 711                          * to make this call to fixup the chain data pointers
 712                          * after resizing in case this is an encrypted or
 713                          * compressed buffer.
 714                          */
 715                         hammer2_cluster_modify(trans, cluster,
 716                                                HAMMER2_MODIFY_OPTDATA);
 717                         break;
 718                 default:
 719                         panic("hammer2_assign_physical: bad type");
 720                         /* NOT REACHED */
 721                         break;
 722                 }
 723         }
 724
 725         /*
 726          * Cleanup.  If cluster wound up being the inode itself, i.e.
 727          * the DIRECTDATA case for offset 0, then we need to update cparent.
 728          * The caller expects cparent to not become stale.
 729          */
 730         hammer2_cluster_lookup_done(dparent);
 731         /* dparent = NULL; safety */
 732         return (cluster);
 733 }
 734
 735 /*
 736  * hammer2_write_file_core() - hammer2_write_thread() helper
 737  *
 738  * The core write function which determines which path to take
 739  * depending on compression settings.  We also have to locate the
 740  * related clusters so we can calculate and set the check data for
 741  * the blockref.
 742  */
 743 static
 744 void
 745 hammer2_write_file_core(struct buf *bp, hammer2_trans_t *trans,
 746                         hammer2_inode_t *ip,
 747                         hammer2_cluster_t *cparent,
 748                         hammer2_key_t lbase, int ioflag, int pblksize,
 749                         int *errorp)
 750 {
 751         hammer2_cluster_t *cluster;
 752
 753         switch(HAMMER2_DEC_ALGO(ip->meta.comp_algo)) {
 754         case HAMMER2_COMP_NONE:
 755                 /*
 756                  * We have to assign physical storage to the buffer
 757                  * we intend to dirty or write now to avoid deadlocks
 758                  * in the strategy code later.
 759                  *
 760                  * This can return NOOFFSET for inode-embedded data.
 761                  * The strategy code will take care of it in that case.
 762                  */
 763                 cluster = hammer2_assign_physical(trans, ip, cparent,
 764                                                 lbase, pblksize,
 765                                                 errorp);
 766                 if (cluster->ddflag) {
 767                         hammer2_inode_data_t *wipdata;
 768
 769                         wipdata = hammer2_cluster_modify_ip(trans, ip,
 770                                                             cluster, 0);
 771                         KKASSERT(wipdata->meta.op_flags &
 772                                  HAMMER2_OPFLAG_DIRECTDATA);
 773                         KKASSERT(bp->b_loffset == 0);
 774                         bcopy(bp->b_data, wipdata->u.data,
 775                               HAMMER2_EMBEDDED_BYTES);
 776                         hammer2_cluster_modsync(cluster);
 777                 } else {
 778                         hammer2_write_bp(cluster, bp, ioflag, pblksize,
 779                                          errorp, ip->meta.check_algo);
 780                 }
 781                 if (cluster) {
 782                         hammer2_cluster_unlock(cluster);
 783                         hammer2_cluster_drop(cluster);
 784                 }
 785                 break;
 786         case HAMMER2_COMP_AUTOZERO:
 787                 /*
 788                  * Check for zero-fill only
 789                  */
 790                 hammer2_zero_check_and_write(bp, trans, ip,
 791                                     cparent, lbase,
 792                                     ioflag, pblksize, errorp,
 793                                     ip->meta.check_algo);
 794                 break;
 795         case HAMMER2_COMP_LZ4:
 796         case HAMMER2_COMP_ZLIB:
 797         default:
 798                 /*
 799                  * Check for zero-fill and attempt compression.
 800                  */
 801                 hammer2_compress_and_write(bp, trans, ip,
 802                                            cparent,
 803                                            lbase, ioflag,
 804                                            pblksize, errorp,
 805                                            ip->meta.comp_algo,
 806                                            ip->meta.check_algo);
 807                 break;
 808         }
 809 }
 810
 811 /*
 812  * Helper
 813  *
 814  * Generic function that will perform the compression in compression
 815  * write path. The compression algorithm is determined by the settings
 816  * obtained from inode.
 817  */
 818 static
 819 void
 820 hammer2_compress_and_write(struct buf *bp, hammer2_trans_t *trans,
 821         hammer2_inode_t *ip,
 822         hammer2_cluster_t *cparent,
 823         hammer2_key_t lbase, int ioflag, int pblksize,
 824         int *errorp, int comp_algo, int check_algo)
 825 {
 826         hammer2_cluster_t *cluster;
 827         hammer2_chain_t *chain;
 828         int comp_size;
 829         int comp_block_size;
 830         int i;
 831         char *comp_buffer;
 832
 833         if (test_block_zeros(bp->b_data, pblksize)) {
 834                 zero_write(bp, trans, ip, cparent, lbase, errorp);
 835                 return;
 836         }
 837
 838         comp_size = 0;
 839         comp_buffer = NULL;
 840
 841         KKASSERT(pblksize / 2 <= 32768);
 842
 843         if (ip->comp_heuristic < 8 || (ip->comp_heuristic & 7) == 0) {
 844                 z_stream strm_compress;
 845                 int comp_level;
 846                 int ret;
 847
 848                 switch(HAMMER2_DEC_ALGO(comp_algo)) {
 849                 case HAMMER2_COMP_LZ4:
 850                         comp_buffer = objcache_get(cache_buffer_write,
 851                                                    M_INTWAIT);
 852                         comp_size = LZ4_compress_limitedOutput(
 853                                         bp->b_data,
 854                                         &comp_buffer[sizeof(int)],
 855                                         pblksize,
 856                                         pblksize / 2 - sizeof(int));
 857                         /*
 858                          * We need to prefix with the size, LZ4
 859                          * doesn't do it for us.  Add the related
 860                          * overhead.
 861                          */
 862                         *(int *)comp_buffer = comp_size;
 863                         if (comp_size)
 864                                 comp_size += sizeof(int);
 865                         break;
 866                 case HAMMER2_COMP_ZLIB:
 867                         comp_level = HAMMER2_DEC_LEVEL(comp_algo);
 868                         if (comp_level == 0)
 869                                 comp_level = 6; /* default zlib compression */
 870                         else if (comp_level < 6)
 871                                 comp_level = 6;
 872                         else if (comp_level > 9)
 873                                 comp_level = 9;
 874                         ret = deflateInit(&strm_compress, comp_level);
 875                         if (ret != Z_OK) {
 876                                 kprintf("HAMMER2 ZLIB: fatal error "
 877                                         "on deflateInit.\n");
 878                         }
 879
 880                         comp_buffer = objcache_get(cache_buffer_write,
 881                                                    M_INTWAIT);
 882                         strm_compress.next_in = bp->b_data;
 883                         strm_compress.avail_in = pblksize;
 884                         strm_compress.next_out = comp_buffer;
 885                         strm_compress.avail_out = pblksize / 2;
 886                         ret = deflate(&strm_compress, Z_FINISH);
 887                         if (ret == Z_STREAM_END) {
 888                                 comp_size = pblksize / 2 -
 889                                             strm_compress.avail_out;
 890                         } else {
 891                                 comp_size = 0;
 892                         }
 893                         ret = deflateEnd(&strm_compress);
 894                         break;
 895                 default:
 896                         kprintf("Error: Unknown compression method.\n");
 897                         kprintf("Comp_method = %d.\n", comp_algo);
 898                         break;
 899                 }
 900         }
 901
 902         if (comp_size == 0) {
 903                 /*
 904                  * compression failed or turned off
 905                  */
 906                 comp_block_size = pblksize;     /* safety */
 907                 if (++ip->comp_heuristic > 128)
 908                         ip->comp_heuristic = 8;
 909         } else {
 910                 /*
 911                  * compression succeeded
 912                  */
 913                 ip->comp_heuristic = 0;
 914                 if (comp_size <= 1024) {
 915                         comp_block_size = 1024;
 916                 } else if (comp_size <= 2048) {
 917                         comp_block_size = 2048;
 918                 } else if (comp_size <= 4096) {
 919                         comp_block_size = 4096;
 920                 } else if (comp_size <= 8192) {
 921                         comp_block_size = 8192;
 922                 } else if (comp_size <= 16384) {
 923                         comp_block_size = 16384;
 924                 } else if (comp_size <= 32768) {
 925                         comp_block_size = 32768;
 926                 } else {
 927                         panic("hammer2: WRITE PATH: "
 928                               "Weird comp_size value.");
 929                         /* NOT REACHED */
 930                         comp_block_size = pblksize;
 931                 }
 932         }
 933
 934         cluster = hammer2_assign_physical(trans, ip, cparent,
 935                                           lbase, comp_block_size,
 936                                           errorp);
 937         if (*errorp) {
 938                 kprintf("WRITE PATH: An error occurred while "
 939                         "assigning physical space.\n");
 940                 KKASSERT(cluster == NULL);
 941                 goto done;
 942         }
 943
 944         if (cluster->ddflag) {
 945                 hammer2_inode_data_t *wipdata;
 946
 947                 wipdata = &hammer2_cluster_wdata(cluster)->ipdata;
 948                 KKASSERT(wipdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA);
 949                 KKASSERT(bp->b_loffset == 0);
 950                 bcopy(bp->b_data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
 951                 hammer2_cluster_modsync(cluster);
 952         } else
 953         for (i = 0; i < cluster->nchains; ++i) {
 954                 hammer2_io_t *dio;
 955                 char *bdata;
 956
 957                 /* XXX hackx */
 958
 959                 if ((cluster->array[i].flags & HAMMER2_CITEM_FEMOD) == 0)
 960                         continue;
 961                 chain = cluster->array[i].chain;        /* XXX */
 962                 if (chain == NULL)
 963                         continue;
 964                 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
 965
 966                 switch(chain->bref.type) {
 967                 case HAMMER2_BREF_TYPE_INODE:
 968                         panic("hammer2_write_bp: unexpected inode\n");
 969                         break;
 970                 case HAMMER2_BREF_TYPE_DATA:
 971                         /*
 972                          * Optimize out the read-before-write
 973                          * if possible.
 974                          */
 975                         *errorp = hammer2_io_newnz(chain->hmp,
 976                                                    chain->bref.data_off,
 977                                                    chain->bytes,
 978                                                    &dio);
 979                         if (*errorp) {
 980                                 hammer2_io_brelse(&dio);
 981                                 kprintf("hammer2: WRITE PATH: "
 982                                         "dbp bread error\n");
 983                                 break;
 984                         }
 985                         bdata = hammer2_io_data(dio, chain->bref.data_off);
 986
 987                         /*
 988                          * When loading the block make sure we don't
 989                          * leave garbage after the compressed data.
 990                          */
 991                         if (comp_size) {
 992                                 chain->bref.methods =
 993                                         HAMMER2_ENC_COMP(comp_algo) +
 994                                         HAMMER2_ENC_CHECK(check_algo);
 995                                 bcopy(comp_buffer, bdata, comp_size);
 996                                 if (comp_size != comp_block_size) {
 997                                         bzero(bdata + comp_size,
 998                                               comp_block_size - comp_size);
 999                                 }
1000                         } else {
1001                                 chain->bref.methods =
1002                                         HAMMER2_ENC_COMP(
1003                                                 HAMMER2_COMP_NONE) +
1004                                         HAMMER2_ENC_CHECK(check_algo);
1005                                 bcopy(bp->b_data, bdata, pblksize);
1006                         }
1007
1008                         /*
1009                          * The flush code doesn't calculate check codes for
1010                          * file data (doing so can result in excessive I/O),
1011                          * so we do it here.
1012                          */
1013                         hammer2_chain_setcheck(chain, bdata);
1014
1015                         /*
1016                          * Device buffer is now valid, chain is no longer in
1017                          * the initial state.
1018                          *
1019                          * (No blockref table worries with file data)
1020                          */
1021                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1022
1023                         /* Now write the related bdp. */
1024                         if (ioflag & IO_SYNC) {
1025                                 /*
1026                                  * Synchronous I/O requested.
1027                                  */
1028                                 hammer2_io_bwrite(&dio);
1029                         /*
1030                         } else if ((ioflag & IO_DIRECT) &&
1031                                    loff + n == pblksize) {
1032                                 hammer2_io_bdwrite(&dio);
1033                         */
1034                         } else if (ioflag & IO_ASYNC) {
1035                                 hammer2_io_bawrite(&dio);
1036                         } else {
1037                                 hammer2_io_bdwrite(&dio);
1038                         }
1039                         break;
1040                 default:
1041                         panic("hammer2_write_bp: bad chain type %d\n",
1042                                 chain->bref.type);
1043                         /* NOT REACHED */
1044                         break;
1045                 }
1046         }
1047 done:
1048         if (cluster) {
1049                 hammer2_cluster_unlock(cluster);
1050                 hammer2_cluster_drop(cluster);
1051         }
1052         if (comp_buffer)
1053                 objcache_put(cache_buffer_write, comp_buffer);
1054 }
1055
1056 /*
1057  * Helper
1058  *
1059  * Function that performs zero-checking and writing without compression,
1060  * it corresponds to default zero-checking path.
1061  */
1062 static
1063 void
1064 hammer2_zero_check_and_write(struct buf *bp, hammer2_trans_t *trans,
1065         hammer2_inode_t *ip,
1066         hammer2_cluster_t *cparent,
1067         hammer2_key_t lbase, int ioflag, int pblksize, int *errorp,
1068         int check_algo)
1069 {
1070         hammer2_cluster_t *cluster;
1071
1072         if (test_block_zeros(bp->b_data, pblksize)) {
1073                 zero_write(bp, trans, ip, cparent, lbase, errorp);
1074         } else {
1075                 cluster = hammer2_assign_physical(trans, ip, cparent,
1076                                                   lbase, pblksize, errorp);
1077                 hammer2_write_bp(cluster, bp, ioflag, pblksize, errorp,
1078                                  check_algo);
1079                 if (cluster) {
1080                         hammer2_cluster_unlock(cluster);
1081                         hammer2_cluster_drop(cluster);
1082                 }
1083         }
1084 }
1085
1086 /*
1087  * Helper
1088  *
1089  * A function to test whether a block of data contains only zeros,
1090  * returns TRUE (non-zero) if the block is all zeros.
1091  */
1092 static
1093 int
1094 test_block_zeros(const char *buf, size_t bytes)
1095 {
1096         size_t i;
1097
1098         for (i = 0; i < bytes; i += sizeof(long)) {
1099                 if (*(const long *)(buf + i) != 0)
1100                         return (0);
1101         }
1102         return (1);
1103 }
1104
1105 /*
1106  * Helper
1107  *
1108  * Function to "write" a block that contains only zeros.
1109  */
1110 static
1111 void
1112 zero_write(struct buf *bp, hammer2_trans_t *trans,
1113            hammer2_inode_t *ip,
1114            hammer2_cluster_t *cparent,
1115            hammer2_key_t lbase, int *errorp __unused)
1116 {
1117         hammer2_cluster_t *cluster;
1118         hammer2_key_t key_dummy;
1119
1120         cparent = hammer2_cluster_lookup_init(cparent, 0);
1121         cluster = hammer2_cluster_lookup(cparent, &key_dummy, lbase, lbase,
1122                                      HAMMER2_LOOKUP_NODATA);
1123         if (cluster) {
1124                 if (cluster->ddflag) {
1125                         hammer2_inode_data_t *wipdata;
1126
1127                         wipdata = hammer2_cluster_modify_ip(trans, ip,
1128                                                             cluster, 0);
1129                         KKASSERT(wipdata->meta.op_flags &
1130                                  HAMMER2_OPFLAG_DIRECTDATA);
1131                         KKASSERT(bp->b_loffset == 0);
1132                         bzero(wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1133                         hammer2_cluster_modsync(cluster);
1134                 } else {
1135                         hammer2_cluster_delete(trans, cparent, cluster,
1136                                                HAMMER2_DELETE_PERMANENT);
1137                 }
1138                 hammer2_cluster_unlock(cluster);
1139                 hammer2_cluster_drop(cluster);
1140         }
1141         hammer2_cluster_lookup_done(cparent);
1142 }
1143
1144 /*
1145  * Helper
1146  *
1147  * Function to write the data as it is, without performing any sort of
1148  * compression. This function is used in path without compression and
1149  * default zero-checking path.
1150  */
1151 static
1152 void
1153 hammer2_write_bp(hammer2_cluster_t *cluster, struct buf *bp, int ioflag,
1154                                 int pblksize, int *errorp, int check_algo)
1155 {
1156         hammer2_chain_t *chain;
1157         hammer2_inode_data_t *wipdata;
1158         hammer2_io_t *dio;
1159         char *bdata;
1160         int error;
1161         int i;
1162
1163         error = 0;      /* XXX TODO below */
1164
1165         for (i = 0; i < cluster->nchains; ++i) {
1166                 if ((cluster->array[i].flags & HAMMER2_CITEM_FEMOD) == 0)
1167                         continue;
1168                 chain = cluster->array[i].chain;        /* XXX */
1169                 if (chain == NULL)
1170                         continue;
1171                 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1172
1173                 switch(chain->bref.type) {
1174                 case HAMMER2_BREF_TYPE_INODE:
1175                         wipdata = &hammer2_chain_wdata(chain)->ipdata;
1176                         KKASSERT(wipdata->meta.op_flags &
1177                                  HAMMER2_OPFLAG_DIRECTDATA);
1178                         KKASSERT(bp->b_loffset == 0);
1179                         bcopy(bp->b_data, wipdata->u.data,
1180                               HAMMER2_EMBEDDED_BYTES);
1181                         error = 0;
1182                         break;
1183                 case HAMMER2_BREF_TYPE_DATA:
1184                         error = hammer2_io_newnz(chain->hmp,
1185                                                  chain->bref.data_off,
1186                                                  chain->bytes, &dio);
1187                         if (error) {
1188                                 hammer2_io_bqrelse(&dio);
1189                                 kprintf("hammer2: WRITE PATH: "
1190                                         "dbp bread error\n");
1191                                 break;
1192                         }
1193                         bdata = hammer2_io_data(dio, chain->bref.data_off);
1194
1195                         chain->bref.methods = HAMMER2_ENC_COMP(
1196                                                         HAMMER2_COMP_NONE) +
1197                                               HAMMER2_ENC_CHECK(check_algo);
1198                         bcopy(bp->b_data, bdata, chain->bytes);
1199
1200                         /*
1201                          * The flush code doesn't calculate check codes for
1202                          * file data (doing so can result in excessive I/O),
1203                          * so we do it here.
1204                          */
1205                         hammer2_chain_setcheck(chain, bdata);
1206
1207                         /*
1208                          * Device buffer is now valid, chain is no longer in
1209                          * the initial state.
1210                          *
1211                          * (No blockref table worries with file data)
1212                          */
1213                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1214
1215                         if (ioflag & IO_SYNC) {
1216                                 /*
1217                                  * Synchronous I/O requested.
1218                                  */
1219                                 hammer2_io_bwrite(&dio);
1220                         /*
1221                         } else if ((ioflag & IO_DIRECT) &&
1222                                    loff + n == pblksize) {
1223                                 hammer2_io_bdwrite(&dio);
1224                         */
1225                         } else if (ioflag & IO_ASYNC) {
1226                                 hammer2_io_bawrite(&dio);
1227                         } else {
1228                                 hammer2_io_bdwrite(&dio);
1229                         }
1230                         break;
1231                 default:
1232                         panic("hammer2_write_bp: bad chain type %d\n",
1233                               chain->bref.type);
1234                         /* NOT REACHED */
1235                         error = 0;
1236                         break;
1237                 }
1238                 KKASSERT(error == 0);   /* XXX TODO */
1239         }
1240         *errorp = error;
1241 }