sys/vfs/hammer2/hammer2_strategy.c

   1 /*
   2  * Copyright (c) 2011-2015 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@dragonflybsd.org>
   6  * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
   7  * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  *
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  * 3. Neither the name of The DragonFly Project nor the names of its
  20  *    contributors may be used to endorse or promote products derived
  21  *    from this software without specific, prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  26  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  27  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  28  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  29  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  30  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  31  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  32  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  33  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  */
  36 /*
  37  * This module handles low level logical file I/O (strategy) which backs
  38  * the logical buffer cache.
  39  *
  40  * [De]compression, zero-block, check codes, and buffer cache operations
  41  * for file data is handled here.
  42  *
  43  * Live dedup makes its home here as well.
  44  */
  45
  46 #include <sys/param.h>
  47 #include <sys/systm.h>
  48 #include <sys/kernel.h>
  49 #include <sys/fcntl.h>
  50 #include <sys/buf.h>
  51 #include <sys/proc.h>
  52 #include <sys/namei.h>
  53 #include <sys/mount.h>
  54 #include <sys/vnode.h>
  55 #include <sys/mountctl.h>
  56 #include <sys/dirent.h>
  57 #include <sys/uio.h>
  58 #include <sys/objcache.h>
  59 #include <sys/event.h>
  60 #include <sys/file.h>
  61 #include <vfs/fifofs/fifo.h>
  62
  63 #include "hammer2.h"
  64 #include "hammer2_lz4.h"
  65
  66 #include "zlib/hammer2_zlib.h"
  67
  68 struct objcache *cache_buffer_read;
  69 struct objcache *cache_buffer_write;
  70
  71 /*
  72  * Strategy code (async logical file buffer I/O from system)
  73  *
  74  * Except for the transaction init (which should normally not block),
  75  * we essentially run the strategy operation asynchronously via a XOP.
  76  *
  77  * XXX This isn't supposed to be able to deadlock against vfs_sync vfsync()
  78  *     calls but it has in the past when multiple flushes are queued.
  79  *
  80  * XXX We currently terminate the transaction once we get a quorum, otherwise
  81  *     the frontend can stall, but this can leave the remaining nodes with
  82  *     a potential flush conflict.  We need to delay flushes on those nodes
  83  *     until running transactions complete separately from the normal
  84  *     transaction sequencing.  FIXME TODO.
  85  */
  86 static void hammer2_strategy_xop_read(hammer2_thread_t *thr,
  87                                 hammer2_xop_t *arg);
  88 static void hammer2_strategy_xop_write(hammer2_thread_t *thr,
  89                                 hammer2_xop_t *arg);
  90 static int hammer2_strategy_read(struct vop_strategy_args *ap);
  91 static int hammer2_strategy_write(struct vop_strategy_args *ap);
  92 static void hammer2_strategy_read_completion(hammer2_chain_t *chain,
  93                                 char *data, struct bio *bio);
  94
  95 static hammer2_off_t hammer2_dedup_lookup(hammer2_dev_t *hmp,
  96                         char **datap, int pblksize);
  97
  98 #if 1
  99 #define TIMER(which)
 100 #else
 101 int h2timer[32];
 102 int h2last;
 103 int h2lid;
 104
 105 #define TIMER(which)    do {                            \
 106         if (h2last)                                     \
 107                 h2timer[h2lid] += (int)(ticks - h2last);\
 108         h2last = ticks;                                 \
 109         h2lid = which;                                  \
 110 } while(0)
 111 #endif
 112
 113 int
 114 hammer2_vop_strategy(struct vop_strategy_args *ap)
 115 {
 116         struct bio *biop;
 117         struct buf *bp;
 118         int error;
 119
 120         biop = ap->a_bio;
 121         bp = biop->bio_buf;
 122
 123         switch(bp->b_cmd) {
 124         case BUF_CMD_READ:
 125                 error = hammer2_strategy_read(ap);
 126                 ++hammer2_iod_file_read;
 127                 break;
 128         case BUF_CMD_WRITE:
 129                 error = hammer2_strategy_write(ap);
 130                 ++hammer2_iod_file_write;
 131                 break;
 132         default:
 133                 bp->b_error = error = EINVAL;
 134                 bp->b_flags |= B_ERROR;
 135                 biodone(biop);
 136                 break;
 137         }
 138         return (error);
 139 }
 140
 141 /*
 142  * Return the largest contiguous physical disk range for the logical
 143  * request, in bytes.
 144  *
 145  * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
 146  *
 147  * Basically disabled, the logical buffer write thread has to deal with
 148  * buffers one-at-a-time.  Note that this should not prevent cluster_read()
 149  * from reading-ahead, it simply prevents it from trying form a single
 150  * cluster buffer for the logical request.  H2 already uses 64KB buffers!
 151  */
 152 int
 153 hammer2_vop_bmap(struct vop_bmap_args *ap)
 154 {
 155         *ap->a_doffsetp = NOOFFSET;
 156         if (ap->a_runp)
 157                 *ap->a_runp = 0;
 158         if (ap->a_runb)
 159                 *ap->a_runb = 0;
 160         return (EOPNOTSUPP);
 161 }
 162
 163 /****************************************************************************
 164  *                              READ SUPPORT                                *
 165  ****************************************************************************/
 166 /*
 167  * Callback used in read path in case that a block is compressed with LZ4.
 168  */
 169 static
 170 void
 171 hammer2_decompress_LZ4_callback(const char *data, u_int bytes, struct bio *bio)
 172 {
 173         struct buf *bp;
 174         char *compressed_buffer;
 175         int compressed_size;
 176         int result;
 177
 178         bp = bio->bio_buf;
 179
 180 #if 0
 181         if bio->bio_caller_info2.index &&
 182               bio->bio_caller_info1.uvalue32 !=
 183               crc32(bp->b_data, bp->b_bufsize) --- return error
 184 #endif
 185
 186         KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
 187         compressed_size = *(const int *)data;
 188         KKASSERT((uint32_t)compressed_size <= bytes - sizeof(int));
 189
 190         compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
 191         result = LZ4_decompress_safe(__DECONST(char *, &data[sizeof(int)]),
 192                                      compressed_buffer,
 193                                      compressed_size,
 194                                      bp->b_bufsize);
 195         if (result < 0) {
 196                 kprintf("READ PATH: Error during decompression."
 197                         "bio %016jx/%d\n",
 198                         (intmax_t)bio->bio_offset, bytes);
 199                 /* make sure it isn't random garbage */
 200                 bzero(compressed_buffer, bp->b_bufsize);
 201         }
 202         KKASSERT(result <= bp->b_bufsize);
 203         bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
 204         if (result < bp->b_bufsize)
 205                 bzero(bp->b_data + result, bp->b_bufsize - result);
 206         objcache_put(cache_buffer_read, compressed_buffer);
 207         bp->b_resid = 0;
 208         bp->b_flags |= B_AGE;
 209 }
 210
 211 /*
 212  * Callback used in read path in case that a block is compressed with ZLIB.
 213  * It is almost identical to LZ4 callback, so in theory they can be unified,
 214  * but we didn't want to make changes in bio structure for that.
 215  */
 216 static
 217 void
 218 hammer2_decompress_ZLIB_callback(const char *data, u_int bytes, struct bio *bio)
 219 {
 220         struct buf *bp;
 221         char *compressed_buffer;
 222         z_stream strm_decompress;
 223         int result;
 224         int ret;
 225
 226         bp = bio->bio_buf;
 227
 228         KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
 229         strm_decompress.avail_in = 0;
 230         strm_decompress.next_in = Z_NULL;
 231
 232         ret = inflateInit(&strm_decompress);
 233
 234         if (ret != Z_OK)
 235                 kprintf("HAMMER2 ZLIB: Fatal error in inflateInit.\n");
 236
 237         compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
 238         strm_decompress.next_in = __DECONST(char *, data);
 239
 240         /* XXX supply proper size, subset of device bp */
 241         strm_decompress.avail_in = bytes;
 242         strm_decompress.next_out = compressed_buffer;
 243         strm_decompress.avail_out = bp->b_bufsize;
 244
 245         ret = inflate(&strm_decompress, Z_FINISH);
 246         if (ret != Z_STREAM_END) {
 247                 kprintf("HAMMER2 ZLIB: Fatar error during decompression.\n");
 248                 bzero(compressed_buffer, bp->b_bufsize);
 249         }
 250         bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
 251         result = bp->b_bufsize - strm_decompress.avail_out;
 252         if (result < bp->b_bufsize)
 253                 bzero(bp->b_data + result, strm_decompress.avail_out);
 254         objcache_put(cache_buffer_read, compressed_buffer);
 255         ret = inflateEnd(&strm_decompress);
 256
 257         bp->b_resid = 0;
 258         bp->b_flags |= B_AGE;
 259 }
 260
 261 /*
 262  * Logical buffer I/O, async read.
 263  */
 264 static
 265 int
 266 hammer2_strategy_read(struct vop_strategy_args *ap)
 267 {
 268         hammer2_xop_strategy_t *xop;
 269         struct buf *bp;
 270         struct bio *bio;
 271         struct bio *nbio;
 272         hammer2_inode_t *ip;
 273         hammer2_key_t lbase;
 274
 275         bio = ap->a_bio;
 276         bp = bio->bio_buf;
 277         ip = VTOI(ap->a_vp);
 278         nbio = push_bio(bio);
 279
 280         lbase = bio->bio_offset;
 281         KKASSERT(((int)lbase & HAMMER2_PBUFMASK) == 0);
 282
 283         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_STRATEGY);
 284         xop->finished = 0;
 285         xop->bio = bio;
 286         xop->lbase = lbase;
 287         hammer2_mtx_init(&xop->lock, "h2bior");
 288         hammer2_xop_start(&xop->head, hammer2_strategy_xop_read);
 289         /* asynchronous completion */
 290
 291         return(0);
 292 }
 293
 294 /*
 295  * Per-node XOP (threaded), do a synchronous lookup of the chain and
 296  * its data.  The frontend is asynchronous, so we are also responsible
 297  * for racing to terminate the frontend.
 298  */
 299 static
 300 void
 301 hammer2_strategy_xop_read(hammer2_thread_t *thr, hammer2_xop_t *arg)
 302 {
 303         hammer2_xop_strategy_t *xop = &arg->xop_strategy;
 304         hammer2_chain_t *parent;
 305         hammer2_chain_t *chain;
 306         hammer2_key_t key_dummy;
 307         hammer2_key_t lbase;
 308         struct bio *bio;
 309         struct buf *bp;
 310         int cache_index = -1;
 311         int error;
 312
 313         /*
 314          * Note that we can race completion of the bio supplied by
 315          * the front-end so we cannot access it until we determine
 316          * that we are the ones finishing it up.
 317          */
 318         TIMER(0);
 319         lbase = xop->lbase;
 320
 321         /*
 322          * This is difficult to optimize.  The logical buffer might be
 323          * partially dirty (contain dummy zero-fill pages), which would
 324          * mess up our crc calculation if we were to try a direct read.
 325          * So for now we always double-buffer through the underlying
 326          * storage.
 327          *
 328          * If not for the above problem we could conditionalize on
 329          * (1) 64KB buffer, (2) one chain (not multi-master) and
 330          * (3) !hammer2_double_buffer, and issue a direct read into the
 331          * logical buffer.
 332          */
 333         parent = hammer2_inode_chain(xop->head.ip1, thr->clindex,
 334                                      HAMMER2_RESOLVE_ALWAYS |
 335                                      HAMMER2_RESOLVE_SHARED);
 336         TIMER(1);
 337         if (parent) {
 338                 chain = hammer2_chain_lookup(&parent, &key_dummy,
 339                                              lbase, lbase,
 340                                              &cache_index,
 341                                              HAMMER2_LOOKUP_ALWAYS |
 342                                              HAMMER2_LOOKUP_SHARED);
 343                 error = chain ? chain->error : 0;
 344         } else {
 345                 error = EIO;
 346                 chain = NULL;
 347         }
 348         TIMER(2);
 349         error = hammer2_xop_feed(&xop->head, chain, thr->clindex, error);
 350         TIMER(3);
 351         if (chain) {
 352                 hammer2_chain_unlock(chain);
 353                 hammer2_chain_drop(chain);
 354         }
 355         if (parent) {
 356                 hammer2_chain_unlock(parent);
 357                 hammer2_chain_drop(parent);
 358         }
 359         chain = NULL;   /* safety */
 360         parent = NULL;  /* safety */
 361         TIMER(4);
 362
 363         /*
 364          * Race to finish the frontend.  First-to-complete.  bio is only
 365          * valid if we are determined to be the ones able to complete
 366          * the operation.
 367          */
 368         if (xop->finished)
 369                 return;
 370         hammer2_mtx_ex(&xop->lock);
 371         if (xop->finished) {
 372                 hammer2_mtx_unlock(&xop->lock);
 373                 return;
 374         }
 375         bio = xop->bio;
 376         bp = bio->bio_buf;
 377
 378         /*
 379          * Async operation has not completed and we now own the lock.
 380          * Determine if we can complete the operation by issuing the
 381          * frontend collection non-blocking.
 382          *
 383          * H2 double-buffers the data, setting B_NOTMETA on the logical
 384          * buffer hints to the OS that the logical buffer should not be
 385          * swapcached (since the device buffer can be).
 386          *
 387          * Also note that even for compressed data we would rather the
 388          * kernel cache/swapcache device buffers more and (decompressed)
 389          * logical buffers less, since that will significantly improve
 390          * the amount of end-user data that can be cached.
 391          */
 392         error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
 393         TIMER(5);
 394
 395         switch(error) {
 396         case 0:
 397                 xop->finished = 1;
 398                 hammer2_mtx_unlock(&xop->lock);
 399                 bp->b_flags |= B_NOTMETA;
 400                 chain = xop->head.cluster.focus;
 401                 hammer2_strategy_read_completion(chain, (char *)chain->data,
 402                                                  xop->bio);
 403                 biodone(bio);
 404                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 405                 break;
 406         case ENOENT:
 407                 xop->finished = 1;
 408                 hammer2_mtx_unlock(&xop->lock);
 409                 bp->b_flags |= B_NOTMETA;
 410                 bp->b_resid = 0;
 411                 bp->b_error = 0;
 412                 bzero(bp->b_data, bp->b_bcount);
 413                 biodone(bio);
 414                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 415                 break;
 416         case EINPROGRESS:
 417                 hammer2_mtx_unlock(&xop->lock);
 418                 break;
 419         default:
 420                 kprintf("strategy_xop_read: error %d loff=%016jx\n",
 421                         error, bp->b_loffset);
 422                 xop->finished = 1;
 423                 hammer2_mtx_unlock(&xop->lock);
 424                 bp->b_flags |= B_ERROR;
 425                 bp->b_error = EIO;
 426                 biodone(bio);
 427                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 428                 break;
 429         }
 430         TIMER(6);
 431 }
 432
 433 static
 434 void
 435 hammer2_strategy_read_completion(hammer2_chain_t *chain, char *data,
 436                                  struct bio *bio)
 437 {
 438         struct buf *bp = bio->bio_buf;
 439
 440         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
 441                 /*
 442                  * Copy from in-memory inode structure.
 443                  */
 444                 bcopy(((hammer2_inode_data_t *)data)->u.data,
 445                       bp->b_data, HAMMER2_EMBEDDED_BYTES);
 446                 bzero(bp->b_data + HAMMER2_EMBEDDED_BYTES,
 447                       bp->b_bcount - HAMMER2_EMBEDDED_BYTES);
 448                 bp->b_resid = 0;
 449                 bp->b_error = 0;
 450         } else if (chain->bref.type == HAMMER2_BREF_TYPE_DATA) {
 451                 /*
 452                  * Data is on-media, record for live dedup.  Release the
 453                  * chain (try to free it) when done.  The data is still
 454                  * cached by both the buffer cache in front and the
 455                  * block device behind us.  This leaves more room in the
 456                  * LRU chain cache for meta-data chains which we really
 457                  * want to retain.
 458                  *
 459                  * NOTE: Deduplication cannot be safely recorded for
 460                  *       records without a check code.
 461                  */
 462                 hammer2_dedup_record(chain, NULL, data);
 463                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 464
 465                 /*
 466                  * Decompression and copy.
 467                  */
 468                 switch (HAMMER2_DEC_COMP(chain->bref.methods)) {
 469                 case HAMMER2_COMP_LZ4:
 470                         hammer2_decompress_LZ4_callback(data, chain->bytes,
 471                                                         bio);
 472                         /* b_resid set by call */
 473                         break;
 474                 case HAMMER2_COMP_ZLIB:
 475                         hammer2_decompress_ZLIB_callback(data, chain->bytes,
 476                                                          bio);
 477                         /* b_resid set by call */
 478                         break;
 479                 case HAMMER2_COMP_NONE:
 480                         KKASSERT(chain->bytes <= bp->b_bcount);
 481                         bcopy(data, bp->b_data, chain->bytes);
 482                         if (chain->bytes < bp->b_bcount) {
 483                                 bzero(bp->b_data + chain->bytes,
 484                                       bp->b_bcount - chain->bytes);
 485                         }
 486                         bp->b_resid = 0;
 487                         bp->b_error = 0;
 488                         break;
 489                 default:
 490                         panic("hammer2_strategy_read: "
 491                               "unknown compression type");
 492                 }
 493         } else {
 494                 panic("hammer2_strategy_read: unknown bref type");
 495         }
 496 }
 497
 498 /****************************************************************************
 499  *                              WRITE SUPPORT                               *
 500  ****************************************************************************/
 501
 502 /*
 503  * Functions for compression in threads,
 504  * from hammer2_vnops.c
 505  */
 506 static void hammer2_write_file_core(char *data, hammer2_inode_t *ip,
 507                                 hammer2_chain_t **parentp,
 508                                 hammer2_key_t lbase, int ioflag, int pblksize,
 509                                 hammer2_tid_t mtid, int *errorp);
 510 static void hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
 511                                 hammer2_chain_t **parentp,
 512                                 hammer2_key_t lbase, int ioflag, int pblksize,
 513                                 hammer2_tid_t mtid, int *errorp,
 514                                 int comp_algo, int check_algo);
 515 static void hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
 516                                 hammer2_chain_t **parentp,
 517                                 hammer2_key_t lbase, int ioflag, int pblksize,
 518                                 hammer2_tid_t mtid, int *errorp,
 519                                 int check_algo);
 520 static int test_block_zeros(const char *buf, size_t bytes);
 521 static void zero_write(char *data, hammer2_inode_t *ip,
 522                                 hammer2_chain_t **parentp,
 523                                 hammer2_key_t lbase,
 524                                 hammer2_tid_t mtid, int *errorp);
 525 static void hammer2_write_bp(hammer2_chain_t *chain, char *data,
 526                                 int ioflag, int pblksize,
 527                                 hammer2_tid_t mtid, int *errorp,
 528                                 int check_algo);
 529
 530 static
 531 int
 532 hammer2_strategy_write(struct vop_strategy_args *ap)
 533 {
 534         hammer2_xop_strategy_t *xop;
 535         hammer2_pfs_t *pmp;
 536         struct bio *bio;
 537         struct buf *bp;
 538         hammer2_inode_t *ip;
 539
 540         bio = ap->a_bio;
 541         bp = bio->bio_buf;
 542         ip = VTOI(ap->a_vp);
 543         pmp = ip->pmp;
 544
 545         hammer2_lwinprog_ref(pmp);
 546         hammer2_trans_assert_strategy(pmp);
 547         hammer2_trans_init(pmp, HAMMER2_TRANS_BUFCACHE);
 548
 549         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
 550                                     HAMMER2_XOP_STRATEGY);
 551         xop->finished = 0;
 552         xop->bio = bio;
 553         xop->lbase = bio->bio_offset;
 554         hammer2_mtx_init(&xop->lock, "h2biow");
 555         hammer2_xop_start(&xop->head, hammer2_strategy_xop_write);
 556         /* asynchronous completion */
 557
 558         hammer2_lwinprog_wait(pmp, hammer2_flush_pipe);
 559
 560         return(0);
 561 }
 562
 563 /*
 564  * Per-node XOP (threaded).  Write the logical buffer to the media.
 565  *
 566  * This is a bit problematic because there may be multiple target and
 567  * any of them may be able to release the bp.  In addition, if our
 568  * particulr target is offline we don't want to block the bp (and thus
 569  * the frontend).  To accomplish this we copy the data to the per-thr
 570  * scratch buffer.
 571  */
 572 static
 573 void
 574 hammer2_strategy_xop_write(hammer2_thread_t *thr, hammer2_xop_t *arg)
 575 {
 576         hammer2_xop_strategy_t *xop = &arg->xop_strategy;
 577         hammer2_chain_t *parent;
 578         hammer2_key_t lbase;
 579         hammer2_inode_t *ip;
 580         struct bio *bio;
 581         struct buf *bp;
 582         int error;
 583         int lblksize;
 584         int pblksize;
 585         hammer2_off_t bio_offset;
 586         char *bio_data;
 587
 588         /*
 589          * We can only access the bp/bio if the frontend has not yet
 590          * completed.
 591          */
 592         if (xop->finished)
 593                 return;
 594         hammer2_mtx_sh(&xop->lock);
 595         if (xop->finished) {
 596                 hammer2_mtx_unlock(&xop->lock);
 597                 return;
 598         }
 599
 600         lbase = xop->lbase;
 601         bio = xop->bio;                 /* ephermal */
 602         bp = bio->bio_buf;              /* ephermal */
 603         ip = xop->head.ip1;             /* retained by ref */
 604         bio_offset = bio->bio_offset;
 605         bio_data = thr->scratch;
 606
 607         /* hammer2_trans_init(parent->hmp->spmp, HAMMER2_TRANS_BUFCACHE); */
 608
 609         lblksize = hammer2_calc_logical(ip, bio->bio_offset, &lbase, NULL);
 610         pblksize = hammer2_calc_physical(ip, lbase);
 611         bcopy(bp->b_data, bio_data, lblksize);
 612
 613         hammer2_mtx_unlock(&xop->lock);
 614         bp = NULL;      /* safety, illegal to access after unlock */
 615         bio = NULL;     /* safety, illegal to access after unlock */
 616
 617         /*
 618          * Actual operation
 619          */
 620         parent = hammer2_inode_chain(ip, thr->clindex, HAMMER2_RESOLVE_ALWAYS);
 621         hammer2_write_file_core(bio_data, ip, &parent,
 622                                 lbase, IO_ASYNC, pblksize,
 623                                 xop->head.mtid, &error);
 624         if (parent) {
 625                 hammer2_chain_unlock(parent);
 626                 hammer2_chain_drop(parent);
 627                 parent = NULL;  /* safety */
 628         }
 629         hammer2_xop_feed(&xop->head, NULL, thr->clindex, error);
 630
 631         /*
 632          * Try to complete the operation on behalf of the front-end.
 633          */
 634         if (xop->finished)
 635                 return;
 636         hammer2_mtx_ex(&xop->lock);
 637         if (xop->finished) {
 638                 hammer2_mtx_unlock(&xop->lock);
 639                 return;
 640         }
 641
 642         /*
 643          * Async operation has not completed and we now own the lock.
 644          * Determine if we can complete the operation by issuing the
 645          * frontend collection non-blocking.
 646          *
 647          * H2 double-buffers the data, setting B_NOTMETA on the logical
 648          * buffer hints to the OS that the logical buffer should not be
 649          * swapcached (since the device buffer can be).
 650          */
 651         error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
 652
 653         if (error == EINPROGRESS) {
 654                 hammer2_mtx_unlock(&xop->lock);
 655                 return;
 656         }
 657
 658         /*
 659          * Async operation has completed.
 660          */
 661         xop->finished = 1;
 662         hammer2_mtx_unlock(&xop->lock);
 663
 664         bio = xop->bio;         /* now owned by us */
 665         bp = bio->bio_buf;      /* now owned by us */
 666
 667         if (error == ENOENT || error == 0) {
 668                 bp->b_flags |= B_NOTMETA;
 669                 bp->b_resid = 0;
 670                 bp->b_error = 0;
 671                 biodone(bio);
 672         } else {
 673                 kprintf("strategy_xop_write: error %d loff=%016jx\n",
 674                         error, bp->b_loffset);
 675                 bp->b_flags |= B_ERROR;
 676                 bp->b_error = EIO;
 677                 biodone(bio);
 678         }
 679         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 680         hammer2_trans_assert_strategy(ip->pmp);
 681         hammer2_lwinprog_drop(ip->pmp);
 682         hammer2_trans_done(ip->pmp);
 683 }
 684
 685 /*
 686  * Wait for pending I/O to complete
 687  */
 688 void
 689 hammer2_bioq_sync(hammer2_pfs_t *pmp)
 690 {
 691         hammer2_lwinprog_wait(pmp, 0);
 692 }
 693
 694 /*
 695  * Create a new cluster at (cparent, lbase) and assign physical storage,
 696  * returning a cluster suitable for I/O.  The cluster will be in a modified
 697  * state.
 698  *
 699  * cparent can wind up being anything.
 700  *
 701  * If datap is not NULL, *datap points to the real data we intend to write.
 702  * If we can dedup the storage location we set *datap to NULL to indicate
 703  * to the caller that a dedup occurred.
 704  *
 705  * NOTE: Special case for data embedded in inode.
 706  */
 707 static
 708 hammer2_chain_t *
 709 hammer2_assign_physical(hammer2_inode_t *ip, hammer2_chain_t **parentp,
 710                         hammer2_key_t lbase, int pblksize,
 711                         hammer2_tid_t mtid, char **datap, int *errorp)
 712 {
 713         hammer2_chain_t *chain;
 714         hammer2_key_t key_dummy;
 715         hammer2_off_t dedup_off;
 716         int pradix = hammer2_getradix(pblksize);
 717         int cache_index = -1;
 718
 719         /*
 720          * Locate the chain associated with lbase, return a locked chain.
 721          * However, do not instantiate any data reference (which utilizes a
 722          * device buffer) because we will be using direct IO via the
 723          * logical buffer cache buffer.
 724          */
 725         *errorp = 0;
 726         KKASSERT(pblksize >= HAMMER2_ALLOC_MIN);
 727 retry:
 728         TIMER(30);
 729         chain = hammer2_chain_lookup(parentp, &key_dummy,
 730                                      lbase, lbase,
 731                                      &cache_index,
 732                                      HAMMER2_LOOKUP_NODATA);
 733
 734         /*
 735          * The lookup code should not return a DELETED chain to us, unless
 736          * its a short-file embedded in the inode.  Then it is possible for
 737          * the lookup to return a deleted inode.
 738          */
 739         if (chain && (chain->flags & HAMMER2_CHAIN_DELETED) &&
 740             chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
 741                 kprintf("assign physical deleted chain @ "
 742                         "%016jx (%016jx.%02x) ip %016jx\n",
 743                         lbase, chain->bref.data_off, chain->bref.type,
 744                         ip->meta.inum);
 745                 Debugger("bleh");
 746         }
 747
 748         if (chain == NULL) {
 749                 /*
 750                  * We found a hole, create a new chain entry.
 751                  *
 752                  * NOTE: DATA chains are created without device backing
 753                  *       store (nor do we want any).
 754                  */
 755                 dedup_off = hammer2_dedup_lookup((*parentp)->hmp, datap,
 756                                                  pblksize);
 757                 *errorp = hammer2_chain_create(parentp, &chain,
 758                                                ip->pmp,
 759                                        HAMMER2_ENC_CHECK(ip->meta.check_algo) |
 760                                        HAMMER2_ENC_COMP(HAMMER2_COMP_NONE),
 761                                                lbase, HAMMER2_PBUFRADIX,
 762                                                HAMMER2_BREF_TYPE_DATA,
 763                                                pblksize, mtid,
 764                                                dedup_off, 0);
 765                 if (chain == NULL) {
 766                         panic("hammer2_chain_create: par=%p error=%d\n",
 767                               *parentp, *errorp);
 768                         goto retry;
 769                 }
 770                 /*ip->delta_dcount += pblksize;*/
 771         } else {
 772                 switch (chain->bref.type) {
 773                 case HAMMER2_BREF_TYPE_INODE:
 774                         /*
 775                          * The data is embedded in the inode, which requires
 776                          * a bit more finess.
 777                          */
 778                         hammer2_chain_modify_ip(ip, chain, mtid, 0);
 779                         break;
 780                 case HAMMER2_BREF_TYPE_DATA:
 781                         dedup_off = hammer2_dedup_lookup(chain->hmp, datap,
 782                                                          pblksize);
 783                         if (chain->bytes != pblksize) {
 784                                 hammer2_chain_resize(chain,
 785                                                      mtid, dedup_off,
 786                                                      pradix,
 787                                                      HAMMER2_MODIFY_OPTDATA);
 788                         }
 789
 790                         /*
 791                          * DATA buffers must be marked modified whether the
 792                          * data is in a logical buffer or not.  We also have
 793                          * to make this call to fixup the chain data pointers
 794                          * after resizing in case this is an encrypted or
 795                          * compressed buffer.
 796                          */
 797                         hammer2_chain_modify(chain, mtid, dedup_off,
 798                                              HAMMER2_MODIFY_OPTDATA);
 799                         break;
 800                 default:
 801                         panic("hammer2_assign_physical: bad type");
 802                         /* NOT REACHED */
 803                         break;
 804                 }
 805         }
 806         TIMER(31);
 807         return (chain);
 808 }
 809
 810 /*
 811  * hammer2_write_file_core() - hammer2_write_thread() helper
 812  *
 813  * The core write function which determines which path to take
 814  * depending on compression settings.  We also have to locate the
 815  * related chains so we can calculate and set the check data for
 816  * the blockref.
 817  */
 818 static
 819 void
 820 hammer2_write_file_core(char *data, hammer2_inode_t *ip,
 821                         hammer2_chain_t **parentp,
 822                         hammer2_key_t lbase, int ioflag, int pblksize,
 823                         hammer2_tid_t mtid, int *errorp)
 824 {
 825         hammer2_chain_t *chain;
 826         char *bdata;
 827
 828         *errorp = 0;
 829
 830         switch(HAMMER2_DEC_ALGO(ip->meta.comp_algo)) {
 831         case HAMMER2_COMP_NONE:
 832                 /*
 833                  * We have to assign physical storage to the buffer
 834                  * we intend to dirty or write now to avoid deadlocks
 835                  * in the strategy code later.
 836                  *
 837                  * This can return NOOFFSET for inode-embedded data.
 838                  * The strategy code will take care of it in that case.
 839                  */
 840                 bdata = data;
 841                 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
 842                                                 mtid, &bdata, errorp);
 843                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
 844                         hammer2_inode_data_t *wipdata;
 845
 846                         wipdata = &chain->data->ipdata;
 847                         KKASSERT(wipdata->meta.op_flags &
 848                                  HAMMER2_OPFLAG_DIRECTDATA);
 849                         bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
 850                         ++hammer2_iod_file_wembed;
 851                 } else if (bdata == NULL) {
 852                         /*
 853                          * Copy of data already present on-media.
 854                          */
 855                         chain->bref.methods =
 856                                 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
 857                                 HAMMER2_ENC_CHECK(ip->meta.check_algo);
 858                         hammer2_chain_setcheck(chain, data);
 859                 } else {
 860                         hammer2_write_bp(chain, data, ioflag, pblksize,
 861                                          mtid, errorp, ip->meta.check_algo);
 862                 }
 863                 if (chain) {
 864                         hammer2_chain_unlock(chain);
 865                         hammer2_chain_drop(chain);
 866                 }
 867                 break;
 868         case HAMMER2_COMP_AUTOZERO:
 869                 /*
 870                  * Check for zero-fill only
 871                  */
 872                 hammer2_zero_check_and_write(data, ip, parentp,
 873                                              lbase, ioflag, pblksize,
 874                                              mtid, errorp,
 875                                              ip->meta.check_algo);
 876                 break;
 877         case HAMMER2_COMP_LZ4:
 878         case HAMMER2_COMP_ZLIB:
 879         default:
 880                 /*
 881                  * Check for zero-fill and attempt compression.
 882                  */
 883                 hammer2_compress_and_write(data, ip, parentp,
 884                                            lbase, ioflag, pblksize,
 885                                            mtid, errorp,
 886                                            ip->meta.comp_algo,
 887                                            ip->meta.check_algo);
 888                 break;
 889         }
 890 }
 891
 892 /*
 893  * Helper
 894  *
 895  * Generic function that will perform the compression in compression
 896  * write path. The compression algorithm is determined by the settings
 897  * obtained from inode.
 898  */
 899 static
 900 void
 901 hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
 902         hammer2_chain_t **parentp,
 903         hammer2_key_t lbase, int ioflag, int pblksize,
 904         hammer2_tid_t mtid, int *errorp, int comp_algo, int check_algo)
 905 {
 906         hammer2_chain_t *chain;
 907         int comp_size;
 908         int comp_block_size;
 909         char *comp_buffer;
 910         char *bdata;
 911
 912         /*
 913          * An all-zeros write creates a hole unless the check code
 914          * is disabled.  When the check code is disabled all writes
 915          * are done in-place, including any all-zeros writes.
 916          *
 917          * NOTE: A snapshot will still force a copy-on-write
 918          *       (see the HAMMER2_CHECK_NONE in hammer2_chain.c).
 919          */
 920         if (check_algo != HAMMER2_CHECK_NONE &&
 921             test_block_zeros(data, pblksize)) {
 922                 zero_write(data, ip, parentp, lbase, mtid, errorp);
 923                 return;
 924         }
 925
 926         /*
 927          * Compression requested.  Try to compress the block.  We store
 928          * the data normally if we cannot sufficiently compress it.
 929          *
 930          * We have a heuristic to detect files which are mostly
 931          * uncompressable and avoid the compression attempt in that
 932          * case.  If the compression heuristic is turned off, we always
 933          * try to compress.
 934          */
 935         comp_size = 0;
 936         comp_buffer = NULL;
 937
 938         KKASSERT(pblksize / 2 <= 32768);
 939
 940         if (ip->comp_heuristic < 8 || (ip->comp_heuristic & 7) == 0 ||
 941             hammer2_always_compress) {
 942                 z_stream strm_compress;
 943                 int comp_level;
 944                 int ret;
 945
 946                 switch(HAMMER2_DEC_ALGO(comp_algo)) {
 947                 case HAMMER2_COMP_LZ4:
 948                         comp_buffer = objcache_get(cache_buffer_write,
 949                                                    M_INTWAIT);
 950                         comp_size = LZ4_compress_limitedOutput(
 951                                         data,
 952                                         &comp_buffer[sizeof(int)],
 953                                         pblksize,
 954                                         pblksize / 2 - sizeof(int));
 955                         /*
 956                          * We need to prefix with the size, LZ4
 957                          * doesn't do it for us.  Add the related
 958                          * overhead.
 959                          */
 960                         *(int *)comp_buffer = comp_size;
 961                         if (comp_size)
 962                                 comp_size += sizeof(int);
 963                         break;
 964                 case HAMMER2_COMP_ZLIB:
 965                         comp_level = HAMMER2_DEC_LEVEL(comp_algo);
 966                         if (comp_level == 0)
 967                                 comp_level = 6; /* default zlib compression */
 968                         else if (comp_level < 6)
 969                                 comp_level = 6;
 970                         else if (comp_level > 9)
 971                                 comp_level = 9;
 972                         ret = deflateInit(&strm_compress, comp_level);
 973                         if (ret != Z_OK) {
 974                                 kprintf("HAMMER2 ZLIB: fatal error "
 975                                         "on deflateInit.\n");
 976                         }
 977
 978                         comp_buffer = objcache_get(cache_buffer_write,
 979                                                    M_INTWAIT);
 980                         strm_compress.next_in = data;
 981                         strm_compress.avail_in = pblksize;
 982                         strm_compress.next_out = comp_buffer;
 983                         strm_compress.avail_out = pblksize / 2;
 984                         ret = deflate(&strm_compress, Z_FINISH);
 985                         if (ret == Z_STREAM_END) {
 986                                 comp_size = pblksize / 2 -
 987                                             strm_compress.avail_out;
 988                         } else {
 989                                 comp_size = 0;
 990                         }
 991                         ret = deflateEnd(&strm_compress);
 992                         break;
 993                 default:
 994                         kprintf("Error: Unknown compression method.\n");
 995                         kprintf("Comp_method = %d.\n", comp_algo);
 996                         break;
 997                 }
 998         }
 999
1000         if (comp_size == 0) {
1001                 /*
1002                  * compression failed or turned off
1003                  */
1004                 comp_block_size = pblksize;     /* safety */
1005                 if (++ip->comp_heuristic > 128)
1006                         ip->comp_heuristic = 8;
1007         } else {
1008                 /*
1009                  * compression succeeded
1010                  */
1011                 ip->comp_heuristic = 0;
1012                 if (comp_size <= 1024) {
1013                         comp_block_size = 1024;
1014                 } else if (comp_size <= 2048) {
1015                         comp_block_size = 2048;
1016                 } else if (comp_size <= 4096) {
1017                         comp_block_size = 4096;
1018                 } else if (comp_size <= 8192) {
1019                         comp_block_size = 8192;
1020                 } else if (comp_size <= 16384) {
1021                         comp_block_size = 16384;
1022                 } else if (comp_size <= 32768) {
1023                         comp_block_size = 32768;
1024                 } else {
1025                         panic("hammer2: WRITE PATH: "
1026                               "Weird comp_size value.");
1027                         /* NOT REACHED */
1028                         comp_block_size = pblksize;
1029                 }
1030
1031                 /*
1032                  * Must zero the remainder or dedup (which operates on a
1033                  * physical block basis) will not find matches.
1034                  */
1035                 if (comp_size < comp_block_size) {
1036                         bzero(comp_buffer + comp_size,
1037                               comp_block_size - comp_size);
1038                 }
1039         }
1040
1041         /*
1042          * Assign physical storage, data will be set to NULL if a live-dedup
1043          * was successful.
1044          */
1045         bdata = comp_size ? comp_buffer : data;
1046         chain = hammer2_assign_physical(ip, parentp, lbase, comp_block_size,
1047                                         mtid, &bdata, errorp);
1048
1049         if (*errorp) {
1050                 kprintf("WRITE PATH: An error occurred while "
1051                         "assigning physical space.\n");
1052                 KKASSERT(chain == NULL);
1053                 goto done;
1054         }
1055
1056         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1057                 hammer2_inode_data_t *wipdata;
1058
1059                 hammer2_chain_modify_ip(ip, chain, mtid, 0);
1060                 wipdata = &chain->data->ipdata;
1061                 KKASSERT(wipdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA);
1062                 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1063                 ++hammer2_iod_file_wembed;
1064         } else if (bdata == NULL) {
1065                 /*
1066                  * Live deduplication, a copy of the data is already present
1067                  * on the media.
1068                  */
1069                 if (comp_size) {
1070                         chain->bref.methods =
1071                                 HAMMER2_ENC_COMP(comp_algo) +
1072                                 HAMMER2_ENC_CHECK(check_algo);
1073                 } else {
1074                         chain->bref.methods =
1075                                 HAMMER2_ENC_COMP(
1076                                         HAMMER2_COMP_NONE) +
1077                                 HAMMER2_ENC_CHECK(check_algo);
1078                 }
1079                 bdata = comp_size ? comp_buffer : data;
1080                 hammer2_chain_setcheck(chain, bdata);
1081                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1082         } else {
1083                 hammer2_io_t *dio;
1084
1085                 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1086
1087                 switch(chain->bref.type) {
1088                 case HAMMER2_BREF_TYPE_INODE:
1089                         panic("hammer2_write_bp: unexpected inode\n");
1090                         break;
1091                 case HAMMER2_BREF_TYPE_DATA:
1092                         /*
1093                          * Optimize out the read-before-write
1094                          * if possible.
1095                          */
1096                         *errorp = hammer2_io_newnz(chain->hmp,
1097                                                    chain->bref.type,
1098                                                    chain->bref.data_off,
1099                                                    chain->bytes,
1100                                                    &dio);
1101                         if (*errorp) {
1102                                 hammer2_io_brelse(&dio);
1103                                 kprintf("hammer2: WRITE PATH: "
1104                                         "dbp bread error\n");
1105                                 break;
1106                         }
1107                         bdata = hammer2_io_data(dio, chain->bref.data_off);
1108
1109                         /*
1110                          * When loading the block make sure we don't
1111                          * leave garbage after the compressed data.
1112                          */
1113                         if (comp_size) {
1114                                 chain->bref.methods =
1115                                         HAMMER2_ENC_COMP(comp_algo) +
1116                                         HAMMER2_ENC_CHECK(check_algo);
1117                                 bcopy(comp_buffer, bdata, comp_size);
1118                         } else {
1119                                 chain->bref.methods =
1120                                         HAMMER2_ENC_COMP(
1121                                                 HAMMER2_COMP_NONE) +
1122                                         HAMMER2_ENC_CHECK(check_algo);
1123                                 bcopy(data, bdata, pblksize);
1124                         }
1125
1126                         /*
1127                          * The flush code doesn't calculate check codes for
1128                          * file data (doing so can result in excessive I/O),
1129                          * so we do it here.
1130                          */
1131                         hammer2_chain_setcheck(chain, bdata);
1132
1133                         /*
1134                          * Device buffer is now valid, chain is no longer in
1135                          * the initial state.
1136                          *
1137                          * (No blockref table worries with file data)
1138                          */
1139                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1140                         hammer2_dedup_record(chain, dio, bdata);
1141
1142                         /* Now write the related bdp. */
1143                         if (ioflag & IO_SYNC) {
1144                                 /*
1145                                  * Synchronous I/O requested.
1146                                  */
1147                                 hammer2_io_bwrite(&dio);
1148                         /*
1149                         } else if ((ioflag & IO_DIRECT) &&
1150                                    loff + n == pblksize) {
1151                                 hammer2_io_bdwrite(&dio);
1152                         */
1153                         } else if (ioflag & IO_ASYNC) {
1154                                 hammer2_io_bawrite(&dio);
1155                         } else {
1156                                 hammer2_io_bdwrite(&dio);
1157                         }
1158                         break;
1159                 default:
1160                         panic("hammer2_write_bp: bad chain type %d\n",
1161                                 chain->bref.type);
1162                         /* NOT REACHED */
1163                         break;
1164                 }
1165         }
1166 done:
1167         if (chain) {
1168                 hammer2_chain_unlock(chain);
1169                 hammer2_chain_drop(chain);
1170         }
1171         if (comp_buffer)
1172                 objcache_put(cache_buffer_write, comp_buffer);
1173 }
1174
1175 /*
1176  * Helper
1177  *
1178  * Function that performs zero-checking and writing without compression,
1179  * it corresponds to default zero-checking path.
1180  */
1181 static
1182 void
1183 hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
1184         hammer2_chain_t **parentp,
1185         hammer2_key_t lbase, int ioflag, int pblksize,
1186         hammer2_tid_t mtid, int *errorp,
1187         int check_algo)
1188 {
1189         hammer2_chain_t *chain;
1190         char *bdata;
1191
1192         if (check_algo != HAMMER2_CHECK_NONE &&
1193             test_block_zeros(data, pblksize)) {
1194                 /*
1195                  * An all-zeros write creates a hole unless the check code
1196                  * is disabled.  When the check code is disabled all writes
1197                  * are done in-place, including any all-zeros writes.
1198                  *
1199                  * NOTE: A snapshot will still force a copy-on-write
1200                  *       (see the HAMMER2_CHECK_NONE in hammer2_chain.c).
1201                  */
1202                 zero_write(data, ip, parentp, lbase, mtid, errorp);
1203         } else {
1204                 /*
1205                  * Normal write
1206                  */
1207                 bdata = data;
1208                 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
1209                                                 mtid, &bdata, errorp);
1210                 if (bdata) {
1211                         hammer2_write_bp(chain, data, ioflag, pblksize,
1212                                          mtid, errorp, check_algo);
1213                 } else {
1214                         /* dedup occurred */
1215                         chain->bref.methods =
1216                                 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
1217                                 HAMMER2_ENC_CHECK(check_algo);
1218                         hammer2_chain_setcheck(chain, data);
1219                 }
1220                 if (chain) {
1221                         hammer2_chain_unlock(chain);
1222                         hammer2_chain_drop(chain);
1223                 }
1224         }
1225 }
1226
1227 /*
1228  * Helper
1229  *
1230  * A function to test whether a block of data contains only zeros,
1231  * returns TRUE (non-zero) if the block is all zeros.
1232  */
1233 static
1234 int
1235 test_block_zeros(const char *buf, size_t bytes)
1236 {
1237         size_t i;
1238
1239         for (i = 0; i < bytes; i += sizeof(long)) {
1240                 if (*(const long *)(buf + i) != 0)
1241                         return (0);
1242         }
1243         return (1);
1244 }
1245
1246 /*
1247  * Helper
1248  *
1249  * Function to "write" a block that contains only zeros.
1250  */
1251 static
1252 void
1253 zero_write(char *data, hammer2_inode_t *ip,
1254            hammer2_chain_t **parentp,
1255            hammer2_key_t lbase, hammer2_tid_t mtid, int *errorp)
1256 {
1257         hammer2_chain_t *chain;
1258         hammer2_key_t key_dummy;
1259         int cache_index = -1;
1260
1261         *errorp = 0;
1262         chain = hammer2_chain_lookup(parentp, &key_dummy,
1263                                      lbase, lbase,
1264                                      &cache_index,
1265                                      HAMMER2_LOOKUP_NODATA);
1266         if (chain) {
1267                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1268                         hammer2_inode_data_t *wipdata;
1269
1270                         hammer2_chain_modify_ip(ip, chain, mtid, 0);
1271                         wipdata = &chain->data->ipdata;
1272                         KKASSERT(wipdata->meta.op_flags &
1273                                  HAMMER2_OPFLAG_DIRECTDATA);
1274                         bzero(wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1275                         ++hammer2_iod_file_wembed;
1276                 } else {
1277                         hammer2_chain_delete(*parentp, chain,
1278                                              mtid, HAMMER2_DELETE_PERMANENT);
1279                         ++hammer2_iod_file_wzero;
1280                 }
1281                 hammer2_chain_unlock(chain);
1282                 hammer2_chain_drop(chain);
1283         } else {
1284                 ++hammer2_iod_file_wzero;
1285         }
1286 }
1287
1288 /*
1289  * Helper
1290  *
1291  * Function to write the data as it is, without performing any sort of
1292  * compression. This function is used in path without compression and
1293  * default zero-checking path.
1294  */
1295 static
1296 void
1297 hammer2_write_bp(hammer2_chain_t *chain, char *data, int ioflag,
1298                  int pblksize,
1299                  hammer2_tid_t mtid, int *errorp, int check_algo)
1300 {
1301         hammer2_inode_data_t *wipdata;
1302         hammer2_io_t *dio;
1303         char *bdata;
1304         int error;
1305
1306         error = 0;      /* XXX TODO below */
1307
1308         KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1309
1310         switch(chain->bref.type) {
1311         case HAMMER2_BREF_TYPE_INODE:
1312                 wipdata = &chain->data->ipdata;
1313                 KKASSERT(wipdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA);
1314                 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1315                 error = 0;
1316                 ++hammer2_iod_file_wembed;
1317                 break;
1318         case HAMMER2_BREF_TYPE_DATA:
1319                 error = hammer2_io_newnz(chain->hmp,
1320                                          chain->bref.type,
1321                                          chain->bref.data_off,
1322                                          chain->bytes, &dio);
1323                 if (error) {
1324                         hammer2_io_bqrelse(&dio);
1325                         kprintf("hammer2: WRITE PATH: "
1326                                 "dbp bread error\n");
1327                         break;
1328                 }
1329                 bdata = hammer2_io_data(dio, chain->bref.data_off);
1330
1331                 chain->bref.methods = HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
1332                                       HAMMER2_ENC_CHECK(check_algo);
1333                 bcopy(data, bdata, chain->bytes);
1334
1335                 /*
1336                  * The flush code doesn't calculate check codes for
1337                  * file data (doing so can result in excessive I/O),
1338                  * so we do it here.
1339                  */
1340                 hammer2_chain_setcheck(chain, bdata);
1341
1342                 /*
1343                  * Device buffer is now valid, chain is no longer in
1344                  * the initial state.
1345                  *
1346                  * (No blockref table worries with file data)
1347                  */
1348                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1349                 hammer2_dedup_record(chain, dio, bdata);
1350
1351                 if (ioflag & IO_SYNC) {
1352                         /*
1353                          * Synchronous I/O requested.
1354                          */
1355                         hammer2_io_bwrite(&dio);
1356                 /*
1357                 } else if ((ioflag & IO_DIRECT) &&
1358                            loff + n == pblksize) {
1359                         hammer2_io_bdwrite(&dio);
1360                 */
1361                 } else if (ioflag & IO_ASYNC) {
1362                         hammer2_io_bawrite(&dio);
1363                 } else {
1364                         hammer2_io_bdwrite(&dio);
1365                 }
1366                 break;
1367         default:
1368                 panic("hammer2_write_bp: bad chain type %d\n",
1369                       chain->bref.type);
1370                 /* NOT REACHED */
1371                 error = 0;
1372                 break;
1373         }
1374         KKASSERT(error == 0);   /* XXX TODO */
1375         *errorp = error;
1376 }
1377
1378 /*
1379  * LIVE DEDUP HEURISTICS
1380  *
1381  * Record media and crc information for possible dedup operation.  Note
1382  * that the dedup mask bits must also be set in the related DIO for a dedup
1383  * to be fully validated (which is handled in the freemap allocation code).
1384  *
1385  * WARNING! This code is SMP safe but the heuristic allows SMP collisions.
1386  *          All fields must be loaded into locals and validated.
1387  *
1388  * WARNING! Should only be used for file data and directory entries,
1389  *          hammer2_chain_modify() only checks for the dedup case on data
1390  *          chains.  Also, dedup data can only be recorded for committed
1391  *          chains (so NOT strategy writes which can undergo further
1392  *          modification after the fact!).
1393  */
1394 void
1395 hammer2_dedup_record(hammer2_chain_t *chain, hammer2_io_t *dio, char *data)
1396 {
1397         hammer2_dev_t *hmp;
1398         hammer2_dedup_t *dedup;
1399         uint64_t crc;
1400         uint64_t mask;
1401         int best = 0;
1402         int i;
1403         int dticks;
1404
1405         /*
1406          * We can only record a dedup if we have media data to test against.
1407          * If dedup is not enabled, return early, which allows a chain to
1408          * remain marked MODIFIED (which might have benefits in special
1409          * situations, though typically it does not).
1410          */
1411         if (hammer2_dedup_enable == 0)
1412                 return;
1413         if (dio == NULL) {
1414                 dio = chain->dio;
1415                 if (dio == NULL)
1416                         return;
1417         }
1418
1419         hmp = chain->hmp;
1420
1421         switch(HAMMER2_DEC_CHECK(chain->bref.methods)) {
1422         case HAMMER2_CHECK_ISCSI32:
1423                 /*
1424                  * XXX use the built-in crc (the dedup lookup sequencing
1425                  * needs to be fixed so the check code is already present
1426                  * when dedup_lookup is called)
1427                  */
1428 #if 0
1429                 crc = (uint64_t)(uint32_t)chain->bref.check.iscsi32.value;
1430 #endif
1431                 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1432                 break;
1433         case HAMMER2_CHECK_XXHASH64:
1434                 crc = chain->bref.check.xxhash64.value;
1435                 break;
1436         case HAMMER2_CHECK_SHA192:
1437                 /*
1438                  * XXX use the built-in crc (the dedup lookup sequencing
1439                  * needs to be fixed so the check code is already present
1440                  * when dedup_lookup is called)
1441                  */
1442 #if 0
1443                 crc = ((uint64_t *)chain->bref.check.sha192.data)[0] ^
1444                       ((uint64_t *)chain->bref.check.sha192.data)[1] ^
1445                       ((uint64_t *)chain->bref.check.sha192.data)[2];
1446 #endif
1447                 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1448                 break;
1449         default:
1450                 /*
1451                  * Cannot dedup without a check code
1452                  *
1453                  * NOTE: In particular, CHECK_NONE allows a sector to be
1454                  *       overwritten without copy-on-write, recording
1455                  *       a dedup block for a CHECK_NONE object would be
1456                  *       a disaster!
1457                  */
1458                 return;
1459         }
1460
1461         atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEDUPABLE);
1462
1463         dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1464         for (i = 0; i < 4; ++i) {
1465                 if (dedup[i].data_crc == crc) {
1466                         best = i;
1467                         break;
1468                 }
1469                 dticks = (int)(dedup[i].ticks - dedup[best].ticks);
1470                 if (dticks < 0 || dticks > hz * 60 * 30)
1471                         best = i;
1472         }
1473         dedup += best;
1474         if (hammer2_debug & 0x40000) {
1475                 kprintf("REC %04x %016jx %016jx\n",
1476                         (int)(dedup - hmp->heur_dedup),
1477                         crc,
1478                         chain->bref.data_off);
1479         }
1480         dedup->ticks = ticks;
1481         dedup->data_off = chain->bref.data_off;
1482         dedup->data_crc = crc;
1483
1484         /*
1485          * Set the valid bits for the dedup only after we know the data
1486          * buffer has been updated.  The alloc bits were set (and the valid
1487          * bits cleared) when the media was allocated.
1488          *
1489          * This is done in two stages becuase the bulkfree code can race
1490          * the gap between allocation and data population.  Both masks must
1491          * be set before a bcmp/dedup operation is able to use the block.
1492          */
1493         mask = hammer2_dedup_mask(dio, chain->bref.data_off, chain->bytes);
1494         atomic_set_64(&dio->dedup_valid, mask);
1495
1496 #if 0
1497         /*
1498          * XXX removed. MODIFIED is an integral part of the flush code,
1499          * lets not just clear it
1500          */
1501         /*
1502          * Once we record the dedup the chain must be marked clean to
1503          * prevent reuse of the underlying block.   Remember that this
1504          * write occurs when the buffer cache is flushed (i.e. on sync(),
1505          * fsync(), filesystem periodic sync, or when the kernel needs to
1506          * flush a buffer), and not whenever the user write()s.
1507          */
1508         if (chain->flags & HAMMER2_CHAIN_MODIFIED) {
1509                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_MODIFIED);
1510                 atomic_add_long(&hammer2_count_modified_chains, -1);
1511                 if (chain->pmp)
1512                         hammer2_pfs_memory_wakeup(chain->pmp);
1513         }
1514 #endif
1515 }
1516
1517 static
1518 hammer2_off_t
1519 hammer2_dedup_lookup(hammer2_dev_t *hmp, char **datap, int pblksize)
1520 {
1521         hammer2_dedup_t *dedup;
1522         hammer2_io_t *dio;
1523         hammer2_off_t off;
1524         uint64_t crc;
1525         uint64_t mask;
1526         char *data;
1527         char *dtmp;
1528         int i;
1529
1530         if (hammer2_dedup_enable == 0)
1531                 return 0;
1532         data = *datap;
1533         if (data == NULL)
1534                 return 0;
1535
1536         /*
1537          * XXX use the built-in crc (the dedup lookup sequencing
1538          * needs to be fixed so the check code is already present
1539          * when dedup_lookup is called)
1540          */
1541         crc = XXH64(data, pblksize, XXH_HAMMER2_SEED);
1542         dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1543
1544         if (hammer2_debug & 0x40000) {
1545                 kprintf("LOC %04x/4 %016jx\n",
1546                         (int)(dedup - hmp->heur_dedup),
1547                         crc);
1548         }
1549
1550         for (i = 0; i < 4; ++i) {
1551                 off = dedup[i].data_off;
1552                 cpu_ccfence();
1553                 if (dedup[i].data_crc != crc)
1554                         continue;
1555                 if ((1 << (int)(off & HAMMER2_OFF_MASK_RADIX)) != pblksize)
1556                         continue;
1557                 dio = hammer2_io_getquick(hmp, off, pblksize, 0);
1558                 if (dio) {
1559                         dtmp = hammer2_io_data(dio, off),
1560                         mask = hammer2_dedup_mask(dio, off, pblksize);
1561                         if ((dio->dedup_alloc & mask) == mask &&
1562                             (dio->dedup_valid & mask) == mask &&
1563                             bcmp(data, dtmp, pblksize) == 0) {
1564                                 if (hammer2_debug & 0x40000) {
1565                                         kprintf("DEDUP SUCCESS %016jx\n",
1566                                                 (intmax_t)off);
1567                                 }
1568                                 hammer2_io_putblk(&dio);
1569                                 *datap = NULL;
1570                                 dedup[i].ticks = ticks;   /* update use */
1571                                 atomic_add_long(&hammer2_iod_file_wdedup,
1572                                                 pblksize);
1573
1574                                 return off;             /* RETURN */
1575                         }
1576                         hammer2_io_putblk(&dio);
1577                 }
1578         }
1579         return 0;
1580 }
1581
1582 /*
1583  * Poof.  Races are ok, if someone gets in and reuses a dedup offset
1584  * before or while we are clearing it they will also recover the freemap
1585  * entry (set it to fully allocated), so a bulkfree race can only set it
1586  * to a possibly-free state.
1587  *
1588  * XXX ok, well, not really sure races are ok but going to run with it
1589  *     for the moment.
1590  */
1591 void
1592 hammer2_dedup_clear(hammer2_dev_t *hmp)
1593 {
1594         int i;
1595
1596         for (i = 0; i < HAMMER2_DEDUP_HEUR_SIZE; ++i) {
1597                 hmp->heur_dedup[i].data_off = 0;
1598                 hmp->heur_dedup[i].ticks = ticks - 1;
1599         }
1600 }