sys/vfs/hammer2/hammer2_strategy.c

   1 /*
   2  * Copyright (c) 2011-2015 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@dragonflybsd.org>
   6  * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
   7  * by Daniel Flores (GSOC 2013 - mentored by Matthew Dillon, compression)
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  *
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in
  17  *    the documentation and/or other materials provided with the
  18  *    distribution.
  19  * 3. Neither the name of The DragonFly Project nor the names of its
  20  *    contributors may be used to endorse or promote products derived
  21  *    from this software without specific, prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  26  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  27  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  28  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  29  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  30  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  31  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  32  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  33  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  */
  36 /*
  37  * This module handles low level logical file I/O (strategy) which backs
  38  * the logical buffer cache.
  39  *
  40  * [De]compression, zero-block, check codes, and buffer cache operations
  41  * for file data is handled here.
  42  *
  43  * Live dedup makes its home here as well.
  44  */
  45
  46 #include <sys/param.h>
  47 #include <sys/systm.h>
  48 #include <sys/kernel.h>
  49 #include <sys/fcntl.h>
  50 #include <sys/buf.h>
  51 #include <sys/proc.h>
  52 #include <sys/namei.h>
  53 #include <sys/mount.h>
  54 #include <sys/vnode.h>
  55 #include <sys/mountctl.h>
  56 #include <sys/dirent.h>
  57 #include <sys/uio.h>
  58 #include <sys/objcache.h>
  59 #include <sys/event.h>
  60 #include <sys/file.h>
  61 #include <vfs/fifofs/fifo.h>
  62
  63 #include "hammer2.h"
  64 #include "hammer2_lz4.h"
  65
  66 #include "zlib/hammer2_zlib.h"
  67
  68 struct objcache *cache_buffer_read;
  69 struct objcache *cache_buffer_write;
  70
  71 /*
  72  * Strategy code (async logical file buffer I/O from system)
  73  *
  74  * Except for the transaction init (which should normally not block),
  75  * we essentially run the strategy operation asynchronously via a XOP.
  76  *
  77  * XXX This isn't supposed to be able to deadlock against vfs_sync vfsync()
  78  *     calls but it has in the past when multiple flushes are queued.
  79  *
  80  * XXX We currently terminate the transaction once we get a quorum, otherwise
  81  *     the frontend can stall, but this can leave the remaining nodes with
  82  *     a potential flush conflict.  We need to delay flushes on those nodes
  83  *     until running transactions complete separately from the normal
  84  *     transaction sequencing.  FIXME TODO.
  85  */
  86 static void hammer2_strategy_xop_read(hammer2_thread_t *thr,
  87                                 hammer2_xop_t *arg);
  88 static void hammer2_strategy_xop_write(hammer2_thread_t *thr,
  89                                 hammer2_xop_t *arg);
  90 static int hammer2_strategy_read(struct vop_strategy_args *ap);
  91 static int hammer2_strategy_write(struct vop_strategy_args *ap);
  92 static void hammer2_strategy_read_completion(hammer2_chain_t *chain,
  93                                 char *data, struct bio *bio);
  94
  95 static hammer2_off_t hammer2_dedup_lookup(hammer2_dev_t *hmp,
  96                         char **datap, int pblksize);
  97
  98 int h2timer[32];
  99 int h2last;
 100 int h2lid;
 101
 102 #define TIMER(which)    do {                            \
 103         if (h2last)                                     \
 104                 h2timer[h2lid] += (int)(ticks - h2last);\
 105         h2last = ticks;                                 \
 106         h2lid = which;                                  \
 107 } while(0)
 108
 109 int
 110 hammer2_vop_strategy(struct vop_strategy_args *ap)
 111 {
 112         struct bio *biop;
 113         struct buf *bp;
 114         int error;
 115
 116         biop = ap->a_bio;
 117         bp = biop->bio_buf;
 118
 119         switch(bp->b_cmd) {
 120         case BUF_CMD_READ:
 121                 error = hammer2_strategy_read(ap);
 122                 ++hammer2_iod_file_read;
 123                 break;
 124         case BUF_CMD_WRITE:
 125                 error = hammer2_strategy_write(ap);
 126                 ++hammer2_iod_file_write;
 127                 break;
 128         default:
 129                 bp->b_error = error = EINVAL;
 130                 bp->b_flags |= B_ERROR;
 131                 biodone(biop);
 132                 break;
 133         }
 134         return (error);
 135 }
 136
 137 /*
 138  * Return the largest contiguous physical disk range for the logical
 139  * request, in bytes.
 140  *
 141  * (struct vnode *vp, off_t loffset, off_t *doffsetp, int *runp, int *runb)
 142  *
 143  * Basically disabled, the logical buffer write thread has to deal with
 144  * buffers one-at-a-time.  Note that this should not prevent cluster_read()
 145  * from reading-ahead, it simply prevents it from trying form a single
 146  * cluster buffer for the logical request.  H2 already uses 64KB buffers!
 147  */
 148 int
 149 hammer2_vop_bmap(struct vop_bmap_args *ap)
 150 {
 151         *ap->a_doffsetp = NOOFFSET;
 152         if (ap->a_runp)
 153                 *ap->a_runp = 0;
 154         if (ap->a_runb)
 155                 *ap->a_runb = 0;
 156         return (EOPNOTSUPP);
 157 }
 158
 159 /****************************************************************************
 160  *                              READ SUPPORT                                *
 161  ****************************************************************************/
 162 /*
 163  * Callback used in read path in case that a block is compressed with LZ4.
 164  */
 165 static
 166 void
 167 hammer2_decompress_LZ4_callback(const char *data, u_int bytes, struct bio *bio)
 168 {
 169         struct buf *bp;
 170         char *compressed_buffer;
 171         int compressed_size;
 172         int result;
 173
 174         bp = bio->bio_buf;
 175
 176 #if 0
 177         if bio->bio_caller_info2.index &&
 178               bio->bio_caller_info1.uvalue32 !=
 179               crc32(bp->b_data, bp->b_bufsize) --- return error
 180 #endif
 181
 182         KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
 183         compressed_size = *(const int *)data;
 184         KKASSERT((uint32_t)compressed_size <= bytes - sizeof(int));
 185
 186         compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
 187         result = LZ4_decompress_safe(__DECONST(char *, &data[sizeof(int)]),
 188                                      compressed_buffer,
 189                                      compressed_size,
 190                                      bp->b_bufsize);
 191         if (result < 0) {
 192                 kprintf("READ PATH: Error during decompression."
 193                         "bio %016jx/%d\n",
 194                         (intmax_t)bio->bio_offset, bytes);
 195                 /* make sure it isn't random garbage */
 196                 bzero(compressed_buffer, bp->b_bufsize);
 197         }
 198         KKASSERT(result <= bp->b_bufsize);
 199         bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
 200         if (result < bp->b_bufsize)
 201                 bzero(bp->b_data + result, bp->b_bufsize - result);
 202         objcache_put(cache_buffer_read, compressed_buffer);
 203         bp->b_resid = 0;
 204         bp->b_flags |= B_AGE;
 205 }
 206
 207 /*
 208  * Callback used in read path in case that a block is compressed with ZLIB.
 209  * It is almost identical to LZ4 callback, so in theory they can be unified,
 210  * but we didn't want to make changes in bio structure for that.
 211  */
 212 static
 213 void
 214 hammer2_decompress_ZLIB_callback(const char *data, u_int bytes, struct bio *bio)
 215 {
 216         struct buf *bp;
 217         char *compressed_buffer;
 218         z_stream strm_decompress;
 219         int result;
 220         int ret;
 221
 222         bp = bio->bio_buf;
 223
 224         KKASSERT(bp->b_bufsize <= HAMMER2_PBUFSIZE);
 225         strm_decompress.avail_in = 0;
 226         strm_decompress.next_in = Z_NULL;
 227
 228         ret = inflateInit(&strm_decompress);
 229
 230         if (ret != Z_OK)
 231                 kprintf("HAMMER2 ZLIB: Fatal error in inflateInit.\n");
 232
 233         compressed_buffer = objcache_get(cache_buffer_read, M_INTWAIT);
 234         strm_decompress.next_in = __DECONST(char *, data);
 235
 236         /* XXX supply proper size, subset of device bp */
 237         strm_decompress.avail_in = bytes;
 238         strm_decompress.next_out = compressed_buffer;
 239         strm_decompress.avail_out = bp->b_bufsize;
 240
 241         ret = inflate(&strm_decompress, Z_FINISH);
 242         if (ret != Z_STREAM_END) {
 243                 kprintf("HAMMER2 ZLIB: Fatar error during decompression.\n");
 244                 bzero(compressed_buffer, bp->b_bufsize);
 245         }
 246         bcopy(compressed_buffer, bp->b_data, bp->b_bufsize);
 247         result = bp->b_bufsize - strm_decompress.avail_out;
 248         if (result < bp->b_bufsize)
 249                 bzero(bp->b_data + result, strm_decompress.avail_out);
 250         objcache_put(cache_buffer_read, compressed_buffer);
 251         ret = inflateEnd(&strm_decompress);
 252
 253         bp->b_resid = 0;
 254         bp->b_flags |= B_AGE;
 255 }
 256
 257 /*
 258  * Logical buffer I/O, async read.
 259  */
 260 static
 261 int
 262 hammer2_strategy_read(struct vop_strategy_args *ap)
 263 {
 264         hammer2_xop_strategy_t *xop;
 265         struct buf *bp;
 266         struct bio *bio;
 267         struct bio *nbio;
 268         hammer2_inode_t *ip;
 269         hammer2_key_t lbase;
 270
 271         bio = ap->a_bio;
 272         bp = bio->bio_buf;
 273         ip = VTOI(ap->a_vp);
 274         nbio = push_bio(bio);
 275
 276         lbase = bio->bio_offset;
 277         KKASSERT(((int)lbase & HAMMER2_PBUFMASK) == 0);
 278
 279         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_STRATEGY);
 280         xop->finished = 0;
 281         xop->bio = bio;
 282         xop->lbase = lbase;
 283         hammer2_mtx_init(&xop->lock, "h2bior");
 284         hammer2_xop_start(&xop->head, hammer2_strategy_xop_read);
 285         /* asynchronous completion */
 286
 287         return(0);
 288 }
 289
 290 /*
 291  * Per-node XOP (threaded), do a synchronous lookup of the chain and
 292  * its data.  The frontend is asynchronous, so we are also responsible
 293  * for racing to terminate the frontend.
 294  */
 295 static
 296 void
 297 hammer2_strategy_xop_read(hammer2_thread_t *thr, hammer2_xop_t *arg)
 298 {
 299         hammer2_xop_strategy_t *xop = &arg->xop_strategy;
 300         hammer2_chain_t *parent;
 301         hammer2_chain_t *chain;
 302         hammer2_key_t key_dummy;
 303         hammer2_key_t lbase;
 304         struct bio *bio;
 305         struct buf *bp;
 306         int cache_index = -1;
 307         int error;
 308
 309         /*
 310          * Note that we can race completion of the bio supplied by
 311          * the front-end so we cannot access it until we determine
 312          * that we are the ones finishing it up.
 313          */
 314         TIMER(0);
 315         lbase = xop->lbase;
 316
 317         /*
 318          * This is difficult to optimize.  The logical buffer might be
 319          * partially dirty (contain dummy zero-fill pages), which would
 320          * mess up our crc calculation if we were to try a direct read.
 321          * So for now we always double-buffer through the underlying
 322          * storage.
 323          *
 324          * If not for the above problem we could conditionalize on
 325          * (1) 64KB buffer, (2) one chain (not multi-master) and
 326          * (3) !hammer2_double_buffer, and issue a direct read into the
 327          * logical buffer.
 328          */
 329         parent = hammer2_inode_chain(xop->head.ip1, thr->clindex,
 330                                      HAMMER2_RESOLVE_ALWAYS |
 331                                      HAMMER2_RESOLVE_SHARED);
 332         TIMER(1);
 333         if (parent) {
 334                 chain = hammer2_chain_lookup(&parent, &key_dummy,
 335                                              lbase, lbase,
 336                                              &cache_index,
 337                                              HAMMER2_LOOKUP_ALWAYS |
 338                                              HAMMER2_LOOKUP_SHARED);
 339                 error = chain ? chain->error : 0;
 340         } else {
 341                 error = EIO;
 342                 chain = NULL;
 343         }
 344         TIMER(2);
 345         error = hammer2_xop_feed(&xop->head, chain, thr->clindex, error);
 346         TIMER(3);
 347         if (chain) {
 348                 hammer2_chain_unlock(chain);
 349                 hammer2_chain_drop(chain);
 350         }
 351         if (parent) {
 352                 hammer2_chain_unlock(parent);
 353                 hammer2_chain_drop(parent);
 354         }
 355         chain = NULL;   /* safety */
 356         parent = NULL;  /* safety */
 357         TIMER(4);
 358
 359         /*
 360          * Race to finish the frontend.  First-to-complete.  bio is only
 361          * valid if we are determined to be the ones able to complete
 362          * the operation.
 363          */
 364         if (xop->finished)
 365                 return;
 366         hammer2_mtx_ex(&xop->lock);
 367         if (xop->finished) {
 368                 hammer2_mtx_unlock(&xop->lock);
 369                 return;
 370         }
 371         bio = xop->bio;
 372         bp = bio->bio_buf;
 373
 374         /*
 375          * Async operation has not completed and we now own the lock.
 376          * Determine if we can complete the operation by issuing the
 377          * frontend collection non-blocking.
 378          *
 379          * H2 double-buffers the data, setting B_NOTMETA on the logical
 380          * buffer hints to the OS that the logical buffer should not be
 381          * swapcached (since the device buffer can be).
 382          *
 383          * Also note that even for compressed data we would rather the
 384          * kernel cache/swapcache device buffers more and (decompressed)
 385          * logical buffers less, since that will significantly improve
 386          * the amount of end-user data that can be cached.
 387          */
 388         error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
 389         TIMER(5);
 390
 391         switch(error) {
 392         case 0:
 393                 xop->finished = 1;
 394                 hammer2_mtx_unlock(&xop->lock);
 395                 bp->b_flags |= B_NOTMETA;
 396                 chain = xop->head.cluster.focus;
 397                 hammer2_strategy_read_completion(chain, (char *)chain->data,
 398                                                  xop->bio);
 399                 biodone(bio);
 400                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 401                 break;
 402         case ENOENT:
 403                 xop->finished = 1;
 404                 hammer2_mtx_unlock(&xop->lock);
 405                 bp->b_flags |= B_NOTMETA;
 406                 bp->b_resid = 0;
 407                 bp->b_error = 0;
 408                 bzero(bp->b_data, bp->b_bcount);
 409                 biodone(bio);
 410                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 411                 break;
 412         case EINPROGRESS:
 413                 hammer2_mtx_unlock(&xop->lock);
 414                 break;
 415         default:
 416                 kprintf("strategy_xop_read: error %d loff=%016jx\n",
 417                         error, bp->b_loffset);
 418                 xop->finished = 1;
 419                 hammer2_mtx_unlock(&xop->lock);
 420                 bp->b_flags |= B_ERROR;
 421                 bp->b_error = EIO;
 422                 biodone(bio);
 423                 hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 424                 break;
 425         }
 426         TIMER(6);
 427 }
 428
 429 static
 430 void
 431 hammer2_strategy_read_completion(hammer2_chain_t *chain, char *data,
 432                                  struct bio *bio)
 433 {
 434         struct buf *bp = bio->bio_buf;
 435
 436         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
 437                 /*
 438                  * Copy from in-memory inode structure.
 439                  */
 440                 bcopy(((hammer2_inode_data_t *)data)->u.data,
 441                       bp->b_data, HAMMER2_EMBEDDED_BYTES);
 442                 bzero(bp->b_data + HAMMER2_EMBEDDED_BYTES,
 443                       bp->b_bcount - HAMMER2_EMBEDDED_BYTES);
 444                 bp->b_resid = 0;
 445                 bp->b_error = 0;
 446         } else if (chain->bref.type == HAMMER2_BREF_TYPE_DATA) {
 447                 /*
 448                  * Data is on-media, record for live dedup.  Release the
 449                  * chain (try to free it) when done.  The data is still
 450                  * cached by both the buffer cache in front and the
 451                  * block device behind us.  This leaves more room in the
 452                  * LRU chain cache for meta-data chains which we really
 453                  * want to retain.
 454                  */
 455                 hammer2_dedup_record(chain, data);
 456                 atomic_set_int(&chain->flags, HAMMER2_CHAIN_RELEASE);
 457
 458                 /*
 459                  * Decompression and copy.
 460                  */
 461                 switch (HAMMER2_DEC_COMP(chain->bref.methods)) {
 462                 case HAMMER2_COMP_LZ4:
 463                         hammer2_decompress_LZ4_callback(data, chain->bytes,
 464                                                         bio);
 465                         /* b_resid set by call */
 466                         break;
 467                 case HAMMER2_COMP_ZLIB:
 468                         hammer2_decompress_ZLIB_callback(data, chain->bytes,
 469                                                          bio);
 470                         /* b_resid set by call */
 471                         break;
 472                 case HAMMER2_COMP_NONE:
 473                         KKASSERT(chain->bytes <= bp->b_bcount);
 474                         bcopy(data, bp->b_data, chain->bytes);
 475                         if (chain->bytes < bp->b_bcount) {
 476                                 bzero(bp->b_data + chain->bytes,
 477                                       bp->b_bcount - chain->bytes);
 478                         }
 479                         bp->b_resid = 0;
 480                         bp->b_error = 0;
 481                         break;
 482                 default:
 483                         panic("hammer2_strategy_read: "
 484                               "unknown compression type");
 485                 }
 486         } else {
 487                 panic("hammer2_strategy_read: unknown bref type");
 488         }
 489 }
 490
 491 /****************************************************************************
 492  *                              WRITE SUPPORT                               *
 493  ****************************************************************************/
 494
 495 /*
 496  * Functions for compression in threads,
 497  * from hammer2_vnops.c
 498  */
 499 static void hammer2_write_file_core(char *data, hammer2_inode_t *ip,
 500                                 hammer2_chain_t **parentp,
 501                                 hammer2_key_t lbase, int ioflag, int pblksize,
 502                                 hammer2_tid_t mtid, int *errorp);
 503 static void hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
 504                                 hammer2_chain_t **parentp,
 505                                 hammer2_key_t lbase, int ioflag, int pblksize,
 506                                 hammer2_tid_t mtid, int *errorp,
 507                                 int comp_algo, int check_algo);
 508 static void hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
 509                                 hammer2_chain_t **parentp,
 510                                 hammer2_key_t lbase, int ioflag, int pblksize,
 511                                 hammer2_tid_t mtid, int *errorp,
 512                                 int check_algo);
 513 static int test_block_zeros(const char *buf, size_t bytes);
 514 static void zero_write(char *data, hammer2_inode_t *ip,
 515                                 hammer2_chain_t **parentp,
 516                                 hammer2_key_t lbase,
 517                                 hammer2_tid_t mtid, int *errorp);
 518 static void hammer2_write_bp(hammer2_chain_t *chain, char *data,
 519                                 int ioflag, int pblksize,
 520                                 hammer2_tid_t mtid, int *errorp,
 521                                 int check_algo);
 522
 523 static
 524 int
 525 hammer2_strategy_write(struct vop_strategy_args *ap)
 526 {
 527         hammer2_xop_strategy_t *xop;
 528         hammer2_pfs_t *pmp;
 529         struct bio *bio;
 530         struct buf *bp;
 531         hammer2_inode_t *ip;
 532
 533         bio = ap->a_bio;
 534         bp = bio->bio_buf;
 535         ip = VTOI(ap->a_vp);
 536         pmp = ip->pmp;
 537
 538         hammer2_lwinprog_ref(pmp);
 539         hammer2_trans_assert_strategy(pmp);
 540         hammer2_trans_init(pmp, HAMMER2_TRANS_BUFCACHE);
 541
 542         xop = hammer2_xop_alloc(ip, HAMMER2_XOP_MODIFYING |
 543                                     HAMMER2_XOP_STRATEGY);
 544         xop->finished = 0;
 545         xop->bio = bio;
 546         xop->lbase = bio->bio_offset;
 547         hammer2_mtx_init(&xop->lock, "h2biow");
 548         hammer2_xop_start(&xop->head, hammer2_strategy_xop_write);
 549         /* asynchronous completion */
 550
 551         hammer2_lwinprog_wait(pmp, hammer2_flush_pipe);
 552
 553         return(0);
 554 }
 555
 556 /*
 557  * Per-node XOP (threaded).  Write the logical buffer to the media.
 558  *
 559  * This is a bit problematic because there may be multiple target and
 560  * any of them may be able to release the bp.  In addition, if our
 561  * particulr target is offline we don't want to block the bp (and thus
 562  * the frontend).  To accomplish this we copy the data to the per-thr
 563  * scratch buffer.
 564  */
 565 static
 566 void
 567 hammer2_strategy_xop_write(hammer2_thread_t *thr, hammer2_xop_t *arg)
 568 {
 569         hammer2_xop_strategy_t *xop = &arg->xop_strategy;
 570         hammer2_chain_t *parent;
 571         hammer2_key_t lbase;
 572         hammer2_inode_t *ip;
 573         struct bio *bio;
 574         struct buf *bp;
 575         int error;
 576         int lblksize;
 577         int pblksize;
 578         hammer2_off_t bio_offset;
 579         char *bio_data;
 580
 581         /*
 582          * We can only access the bp/bio if the frontend has not yet
 583          * completed.
 584          */
 585         if (xop->finished)
 586                 return;
 587         hammer2_mtx_sh(&xop->lock);
 588         if (xop->finished) {
 589                 hammer2_mtx_unlock(&xop->lock);
 590                 return;
 591         }
 592
 593         lbase = xop->lbase;
 594         bio = xop->bio;                 /* ephermal */
 595         bp = bio->bio_buf;              /* ephermal */
 596         ip = xop->head.ip1;             /* retained by ref */
 597         bio_offset = bio->bio_offset;
 598         bio_data = thr->scratch;
 599
 600         /* hammer2_trans_init(parent->hmp->spmp, HAMMER2_TRANS_BUFCACHE); */
 601
 602         lblksize = hammer2_calc_logical(ip, bio->bio_offset, &lbase, NULL);
 603         pblksize = hammer2_calc_physical(ip, lbase);
 604         bcopy(bp->b_data, bio_data, lblksize);
 605
 606         hammer2_mtx_unlock(&xop->lock);
 607         bp = NULL;      /* safety, illegal to access after unlock */
 608         bio = NULL;     /* safety, illegal to access after unlock */
 609
 610         /*
 611          * Actual operation
 612          */
 613         parent = hammer2_inode_chain(ip, thr->clindex, HAMMER2_RESOLVE_ALWAYS);
 614         hammer2_write_file_core(bio_data, ip, &parent,
 615                                 lbase, IO_ASYNC, pblksize,
 616                                 xop->head.mtid, &error);
 617         if (parent) {
 618                 hammer2_chain_unlock(parent);
 619                 hammer2_chain_drop(parent);
 620                 parent = NULL;  /* safety */
 621         }
 622         hammer2_xop_feed(&xop->head, NULL, thr->clindex, error);
 623
 624         /*
 625          * Try to complete the operation on behalf of the front-end.
 626          */
 627         if (xop->finished)
 628                 return;
 629         hammer2_mtx_ex(&xop->lock);
 630         if (xop->finished) {
 631                 hammer2_mtx_unlock(&xop->lock);
 632                 return;
 633         }
 634
 635         /*
 636          * Async operation has not completed and we now own the lock.
 637          * Determine if we can complete the operation by issuing the
 638          * frontend collection non-blocking.
 639          *
 640          * H2 double-buffers the data, setting B_NOTMETA on the logical
 641          * buffer hints to the OS that the logical buffer should not be
 642          * swapcached (since the device buffer can be).
 643          */
 644         error = hammer2_xop_collect(&xop->head, HAMMER2_XOP_COLLECT_NOWAIT);
 645
 646         if (error == EINPROGRESS) {
 647                 hammer2_mtx_unlock(&xop->lock);
 648                 return;
 649         }
 650
 651         /*
 652          * Async operation has completed.
 653          */
 654         xop->finished = 1;
 655         hammer2_mtx_unlock(&xop->lock);
 656
 657         bio = xop->bio;         /* now owned by us */
 658         bp = bio->bio_buf;      /* now owned by us */
 659
 660         if (error == ENOENT || error == 0) {
 661                 bp->b_flags |= B_NOTMETA;
 662                 bp->b_resid = 0;
 663                 bp->b_error = 0;
 664                 biodone(bio);
 665         } else {
 666                 kprintf("strategy_xop_write: error %d loff=%016jx\n",
 667                         error, bp->b_loffset);
 668                 bp->b_flags |= B_ERROR;
 669                 bp->b_error = EIO;
 670                 biodone(bio);
 671         }
 672         hammer2_xop_retire(&xop->head, HAMMER2_XOPMASK_VOP);
 673         hammer2_trans_assert_strategy(ip->pmp);
 674         hammer2_lwinprog_drop(ip->pmp);
 675         hammer2_trans_done(ip->pmp);
 676 }
 677
 678 /*
 679  * Wait for pending I/O to complete
 680  */
 681 void
 682 hammer2_bioq_sync(hammer2_pfs_t *pmp)
 683 {
 684         hammer2_lwinprog_wait(pmp, 0);
 685 }
 686
 687 /*
 688  * Create a new cluster at (cparent, lbase) and assign physical storage,
 689  * returning a cluster suitable for I/O.  The cluster will be in a modified
 690  * state.
 691  *
 692  * cparent can wind up being anything.
 693  *
 694  * If datap is not NULL, *datap points to the real data we intend to write.
 695  * If we can dedup the storage location we set *datap to NULL to indicate
 696  * to the caller that a dedup occurred.
 697  *
 698  * NOTE: Special case for data embedded in inode.
 699  */
 700 static
 701 hammer2_chain_t *
 702 hammer2_assign_physical(hammer2_inode_t *ip, hammer2_chain_t **parentp,
 703                         hammer2_key_t lbase, int pblksize,
 704                         hammer2_tid_t mtid, char **datap, int *errorp)
 705 {
 706         hammer2_chain_t *chain;
 707         hammer2_key_t key_dummy;
 708         hammer2_off_t dedup_off;
 709         int pradix = hammer2_getradix(pblksize);
 710         int cache_index = -1;
 711
 712         /*
 713          * Locate the chain associated with lbase, return a locked chain.
 714          * However, do not instantiate any data reference (which utilizes a
 715          * device buffer) because we will be using direct IO via the
 716          * logical buffer cache buffer.
 717          */
 718         *errorp = 0;
 719         KKASSERT(pblksize >= HAMMER2_ALLOC_MIN);
 720 retry:
 721         TIMER(30);
 722         chain = hammer2_chain_lookup(parentp, &key_dummy,
 723                                      lbase, lbase,
 724                                      &cache_index,
 725                                      HAMMER2_LOOKUP_NODATA);
 726
 727         /*
 728          * The lookup code should not return a DELETED chain to us, unless
 729          * its a short-file embedded in the inode.  Then it is possible for
 730          * the lookup to return a deleted inode.
 731          */
 732         if (chain && (chain->flags & HAMMER2_CHAIN_DELETED) &&
 733             chain->bref.type != HAMMER2_BREF_TYPE_INODE) {
 734                 kprintf("assign physical deleted chain @ "
 735                         "%016jx (%016jx.%02x) ip %016jx\n",
 736                         lbase, chain->bref.data_off, chain->bref.type,
 737                         ip->meta.inum);
 738                 Debugger("bleh");
 739         }
 740
 741         if (chain == NULL) {
 742                 /*
 743                  * We found a hole, create a new chain entry.
 744                  *
 745                  * NOTE: DATA chains are created without device backing
 746                  *       store (nor do we want any).
 747                  */
 748                 dedup_off = hammer2_dedup_lookup((*parentp)->hmp, datap,
 749                                                  pblksize);
 750                 *errorp = hammer2_chain_create(parentp, &chain,
 751                                                ip->pmp,
 752                                        HAMMER2_ENC_CHECK(ip->meta.check_algo) |
 753                                        HAMMER2_ENC_COMP(HAMMER2_COMP_NONE),
 754                                                lbase, HAMMER2_PBUFRADIX,
 755                                                HAMMER2_BREF_TYPE_DATA,
 756                                                pblksize, mtid,
 757                                                dedup_off, 0);
 758                 if (chain == NULL) {
 759                         panic("hammer2_chain_create: par=%p error=%d\n",
 760                               *parentp, *errorp);
 761                         goto retry;
 762                 }
 763                 /*ip->delta_dcount += pblksize;*/
 764         } else {
 765                 switch (chain->bref.type) {
 766                 case HAMMER2_BREF_TYPE_INODE:
 767                         /*
 768                          * The data is embedded in the inode, which requires
 769                          * a bit more finess.
 770                          */
 771                         hammer2_chain_modify_ip(ip, chain, mtid, 0);
 772                         break;
 773                 case HAMMER2_BREF_TYPE_DATA:
 774                         dedup_off = hammer2_dedup_lookup(chain->hmp, datap,
 775                                                          pblksize);
 776                         if (chain->bytes != pblksize) {
 777                                 hammer2_chain_resize(chain,
 778                                                      mtid, dedup_off,
 779                                                      pradix,
 780                                                      HAMMER2_MODIFY_OPTDATA);
 781                         }
 782
 783                         /*
 784                          * DATA buffers must be marked modified whether the
 785                          * data is in a logical buffer or not.  We also have
 786                          * to make this call to fixup the chain data pointers
 787                          * after resizing in case this is an encrypted or
 788                          * compressed buffer.
 789                          */
 790                         hammer2_chain_modify(chain, mtid, dedup_off,
 791                                              HAMMER2_MODIFY_OPTDATA);
 792                         break;
 793                 default:
 794                         panic("hammer2_assign_physical: bad type");
 795                         /* NOT REACHED */
 796                         break;
 797                 }
 798         }
 799         TIMER(31);
 800         return (chain);
 801 }
 802
 803 /*
 804  * hammer2_write_file_core() - hammer2_write_thread() helper
 805  *
 806  * The core write function which determines which path to take
 807  * depending on compression settings.  We also have to locate the
 808  * related chains so we can calculate and set the check data for
 809  * the blockref.
 810  */
 811 static
 812 void
 813 hammer2_write_file_core(char *data, hammer2_inode_t *ip,
 814                         hammer2_chain_t **parentp,
 815                         hammer2_key_t lbase, int ioflag, int pblksize,
 816                         hammer2_tid_t mtid, int *errorp)
 817 {
 818         hammer2_chain_t *chain;
 819         char *bdata;
 820
 821         *errorp = 0;
 822
 823         switch(HAMMER2_DEC_ALGO(ip->meta.comp_algo)) {
 824         case HAMMER2_COMP_NONE:
 825                 /*
 826                  * We have to assign physical storage to the buffer
 827                  * we intend to dirty or write now to avoid deadlocks
 828                  * in the strategy code later.
 829                  *
 830                  * This can return NOOFFSET for inode-embedded data.
 831                  * The strategy code will take care of it in that case.
 832                  */
 833                 bdata = data;
 834                 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
 835                                                 mtid, &bdata, errorp);
 836                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
 837                         hammer2_inode_data_t *wipdata;
 838
 839                         wipdata = &chain->data->ipdata;
 840                         KKASSERT(wipdata->meta.op_flags &
 841                                  HAMMER2_OPFLAG_DIRECTDATA);
 842                         bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
 843                         ++hammer2_iod_file_wembed;
 844                 } else if (bdata == NULL) {
 845                         /*
 846                          * Copy of data already present on-media.
 847                          */
 848                         chain->bref.methods =
 849                                 HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
 850                                 HAMMER2_ENC_CHECK(ip->meta.check_algo);
 851                         hammer2_chain_setcheck(chain, data);
 852                 } else {
 853                         hammer2_write_bp(chain, data, ioflag, pblksize,
 854                                          mtid, errorp, ip->meta.check_algo);
 855                 }
 856                 if (chain) {
 857                         hammer2_chain_unlock(chain);
 858                         hammer2_chain_drop(chain);
 859                 }
 860                 break;
 861         case HAMMER2_COMP_AUTOZERO:
 862                 /*
 863                  * Check for zero-fill only
 864                  */
 865                 hammer2_zero_check_and_write(data, ip, parentp,
 866                                              lbase, ioflag, pblksize,
 867                                              mtid, errorp,
 868                                              ip->meta.check_algo);
 869                 break;
 870         case HAMMER2_COMP_LZ4:
 871         case HAMMER2_COMP_ZLIB:
 872         default:
 873                 /*
 874                  * Check for zero-fill and attempt compression.
 875                  */
 876                 hammer2_compress_and_write(data, ip, parentp,
 877                                            lbase, ioflag, pblksize,
 878                                            mtid, errorp,
 879                                            ip->meta.comp_algo,
 880                                            ip->meta.check_algo);
 881                 break;
 882         }
 883 }
 884
 885 /*
 886  * Helper
 887  *
 888  * Generic function that will perform the compression in compression
 889  * write path. The compression algorithm is determined by the settings
 890  * obtained from inode.
 891  */
 892 static
 893 void
 894 hammer2_compress_and_write(char *data, hammer2_inode_t *ip,
 895         hammer2_chain_t **parentp,
 896         hammer2_key_t lbase, int ioflag, int pblksize,
 897         hammer2_tid_t mtid, int *errorp, int comp_algo, int check_algo)
 898 {
 899         hammer2_chain_t *chain;
 900         int comp_size;
 901         int comp_block_size;
 902         char *comp_buffer;
 903         char *bdata;
 904
 905         if (test_block_zeros(data, pblksize)) {
 906                 zero_write(data, ip, parentp, lbase, mtid, errorp);
 907                 return;
 908         }
 909
 910         comp_size = 0;
 911         comp_buffer = NULL;
 912
 913         KKASSERT(pblksize / 2 <= 32768);
 914
 915         if (ip->comp_heuristic < 8 || (ip->comp_heuristic & 7) == 0) {
 916                 z_stream strm_compress;
 917                 int comp_level;
 918                 int ret;
 919
 920                 switch(HAMMER2_DEC_ALGO(comp_algo)) {
 921                 case HAMMER2_COMP_LZ4:
 922                         comp_buffer = objcache_get(cache_buffer_write,
 923                                                    M_INTWAIT);
 924                         comp_size = LZ4_compress_limitedOutput(
 925                                         data,
 926                                         &comp_buffer[sizeof(int)],
 927                                         pblksize,
 928                                         pblksize / 2 - sizeof(int));
 929                         /*
 930                          * We need to prefix with the size, LZ4
 931                          * doesn't do it for us.  Add the related
 932                          * overhead.
 933                          */
 934                         *(int *)comp_buffer = comp_size;
 935                         if (comp_size)
 936                                 comp_size += sizeof(int);
 937                         break;
 938                 case HAMMER2_COMP_ZLIB:
 939                         comp_level = HAMMER2_DEC_LEVEL(comp_algo);
 940                         if (comp_level == 0)
 941                                 comp_level = 6; /* default zlib compression */
 942                         else if (comp_level < 6)
 943                                 comp_level = 6;
 944                         else if (comp_level > 9)
 945                                 comp_level = 9;
 946                         ret = deflateInit(&strm_compress, comp_level);
 947                         if (ret != Z_OK) {
 948                                 kprintf("HAMMER2 ZLIB: fatal error "
 949                                         "on deflateInit.\n");
 950                         }
 951
 952                         comp_buffer = objcache_get(cache_buffer_write,
 953                                                    M_INTWAIT);
 954                         strm_compress.next_in = data;
 955                         strm_compress.avail_in = pblksize;
 956                         strm_compress.next_out = comp_buffer;
 957                         strm_compress.avail_out = pblksize / 2;
 958                         ret = deflate(&strm_compress, Z_FINISH);
 959                         if (ret == Z_STREAM_END) {
 960                                 comp_size = pblksize / 2 -
 961                                             strm_compress.avail_out;
 962                         } else {
 963                                 comp_size = 0;
 964                         }
 965                         ret = deflateEnd(&strm_compress);
 966                         break;
 967                 default:
 968                         kprintf("Error: Unknown compression method.\n");
 969                         kprintf("Comp_method = %d.\n", comp_algo);
 970                         break;
 971                 }
 972         }
 973
 974         if (comp_size == 0) {
 975                 /*
 976                  * compression failed or turned off
 977                  */
 978                 comp_block_size = pblksize;     /* safety */
 979                 if (++ip->comp_heuristic > 128)
 980                         ip->comp_heuristic = 8;
 981         } else {
 982                 /*
 983                  * compression succeeded
 984                  */
 985                 ip->comp_heuristic = 0;
 986                 if (comp_size <= 1024) {
 987                         comp_block_size = 1024;
 988                 } else if (comp_size <= 2048) {
 989                         comp_block_size = 2048;
 990                 } else if (comp_size <= 4096) {
 991                         comp_block_size = 4096;
 992                 } else if (comp_size <= 8192) {
 993                         comp_block_size = 8192;
 994                 } else if (comp_size <= 16384) {
 995                         comp_block_size = 16384;
 996                 } else if (comp_size <= 32768) {
 997                         comp_block_size = 32768;
 998                 } else {
 999                         panic("hammer2: WRITE PATH: "
1000                               "Weird comp_size value.");
1001                         /* NOT REACHED */
1002                         comp_block_size = pblksize;
1003                 }
1004
1005                 /*
1006                  * Must zero the remainder or dedup (which operates on a
1007                  * physical block basis) will not find matches.
1008                  */
1009                 if (comp_size < comp_block_size) {
1010                         bzero(comp_buffer + comp_size,
1011                               comp_block_size - comp_size);
1012                 }
1013         }
1014
1015         /*
1016          * Assign physical storage, data will be set to NULL if a live-dedup
1017          * was successful.
1018          */
1019         bdata = comp_size ? comp_buffer : data;
1020         chain = hammer2_assign_physical(ip, parentp, lbase, comp_block_size,
1021                                         mtid, &bdata, errorp);
1022
1023         if (*errorp) {
1024                 kprintf("WRITE PATH: An error occurred while "
1025                         "assigning physical space.\n");
1026                 KKASSERT(chain == NULL);
1027                 goto done;
1028         }
1029
1030         if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1031                 hammer2_inode_data_t *wipdata;
1032
1033                 hammer2_chain_modify_ip(ip, chain, mtid, 0);
1034                 wipdata = &chain->data->ipdata;
1035                 KKASSERT(wipdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA);
1036                 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1037                 ++hammer2_iod_file_wembed;
1038         } else if (bdata == NULL) {
1039                 /*
1040                  * Live deduplication, a copy of the data is already present
1041                  * on the media.
1042                  */
1043                 if (comp_size) {
1044                         chain->bref.methods =
1045                                 HAMMER2_ENC_COMP(comp_algo) +
1046                                 HAMMER2_ENC_CHECK(check_algo);
1047                 } else {
1048                         chain->bref.methods =
1049                                 HAMMER2_ENC_COMP(
1050                                         HAMMER2_COMP_NONE) +
1051                                 HAMMER2_ENC_CHECK(check_algo);
1052                 }
1053                 bdata = comp_size ? comp_buffer : data;
1054                 hammer2_chain_setcheck(chain, bdata);
1055                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1056         } else {
1057                 hammer2_io_t *dio;
1058
1059                 KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1060
1061                 switch(chain->bref.type) {
1062                 case HAMMER2_BREF_TYPE_INODE:
1063                         panic("hammer2_write_bp: unexpected inode\n");
1064                         break;
1065                 case HAMMER2_BREF_TYPE_DATA:
1066                         /*
1067                          * Optimize out the read-before-write
1068                          * if possible.
1069                          */
1070                         *errorp = hammer2_io_newnz(chain->hmp,
1071                                                    chain->bref.type,
1072                                                    chain->bref.data_off,
1073                                                    chain->bytes,
1074                                                    &dio);
1075                         if (*errorp) {
1076                                 hammer2_io_brelse(&dio);
1077                                 kprintf("hammer2: WRITE PATH: "
1078                                         "dbp bread error\n");
1079                                 break;
1080                         }
1081                         bdata = hammer2_io_data(dio, chain->bref.data_off);
1082
1083                         /*
1084                          * When loading the block make sure we don't
1085                          * leave garbage after the compressed data.
1086                          */
1087                         if (comp_size) {
1088                                 chain->bref.methods =
1089                                         HAMMER2_ENC_COMP(comp_algo) +
1090                                         HAMMER2_ENC_CHECK(check_algo);
1091                                 bcopy(comp_buffer, bdata, comp_size);
1092                         } else {
1093                                 chain->bref.methods =
1094                                         HAMMER2_ENC_COMP(
1095                                                 HAMMER2_COMP_NONE) +
1096                                         HAMMER2_ENC_CHECK(check_algo);
1097                                 bcopy(data, bdata, pblksize);
1098                         }
1099
1100                         /*
1101                          * The flush code doesn't calculate check codes for
1102                          * file data (doing so can result in excessive I/O),
1103                          * so we do it here.
1104                          */
1105                         hammer2_chain_setcheck(chain, bdata);
1106                         hammer2_dedup_record(chain, bdata);
1107
1108                         /*
1109                          * Device buffer is now valid, chain is no longer in
1110                          * the initial state.
1111                          *
1112                          * (No blockref table worries with file data)
1113                          */
1114                         atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1115
1116                         /* Now write the related bdp. */
1117                         if (ioflag & IO_SYNC) {
1118                                 /*
1119                                  * Synchronous I/O requested.
1120                                  */
1121                                 hammer2_io_bwrite(&dio);
1122                         /*
1123                         } else if ((ioflag & IO_DIRECT) &&
1124                                    loff + n == pblksize) {
1125                                 hammer2_io_bdwrite(&dio);
1126                         */
1127                         } else if (ioflag & IO_ASYNC) {
1128                                 hammer2_io_bawrite(&dio);
1129                         } else {
1130                                 hammer2_io_bdwrite(&dio);
1131                         }
1132                         break;
1133                 default:
1134                         panic("hammer2_write_bp: bad chain type %d\n",
1135                                 chain->bref.type);
1136                         /* NOT REACHED */
1137                         break;
1138                 }
1139         }
1140 done:
1141         if (chain) {
1142                 hammer2_chain_unlock(chain);
1143                 hammer2_chain_drop(chain);
1144         }
1145         if (comp_buffer)
1146                 objcache_put(cache_buffer_write, comp_buffer);
1147 }
1148
1149 /*
1150  * Helper
1151  *
1152  * Function that performs zero-checking and writing without compression,
1153  * it corresponds to default zero-checking path.
1154  */
1155 static
1156 void
1157 hammer2_zero_check_and_write(char *data, hammer2_inode_t *ip,
1158         hammer2_chain_t **parentp,
1159         hammer2_key_t lbase, int ioflag, int pblksize,
1160         hammer2_tid_t mtid, int *errorp,
1161         int check_algo)
1162 {
1163         hammer2_chain_t *chain;
1164
1165         if (test_block_zeros(data, pblksize)) {
1166                 zero_write(data, ip, parentp, lbase, mtid, errorp);
1167         } else {
1168                 chain = hammer2_assign_physical(ip, parentp, lbase, pblksize,
1169                                                 mtid, &data, errorp);
1170                 if (data) {
1171                         hammer2_write_bp(chain, data, ioflag, pblksize,
1172                                          mtid, errorp, check_algo);
1173                 } /* else dedup occurred */
1174                 if (chain) {
1175                         hammer2_chain_unlock(chain);
1176                         hammer2_chain_drop(chain);
1177                 }
1178         }
1179 }
1180
1181 /*
1182  * Helper
1183  *
1184  * A function to test whether a block of data contains only zeros,
1185  * returns TRUE (non-zero) if the block is all zeros.
1186  */
1187 static
1188 int
1189 test_block_zeros(const char *buf, size_t bytes)
1190 {
1191         size_t i;
1192
1193         for (i = 0; i < bytes; i += sizeof(long)) {
1194                 if (*(const long *)(buf + i) != 0)
1195                         return (0);
1196         }
1197         return (1);
1198 }
1199
1200 /*
1201  * Helper
1202  *
1203  * Function to "write" a block that contains only zeros.
1204  */
1205 static
1206 void
1207 zero_write(char *data, hammer2_inode_t *ip,
1208            hammer2_chain_t **parentp,
1209            hammer2_key_t lbase, hammer2_tid_t mtid, int *errorp)
1210 {
1211         hammer2_chain_t *chain;
1212         hammer2_key_t key_dummy;
1213         int cache_index = -1;
1214
1215         *errorp = 0;
1216         chain = hammer2_chain_lookup(parentp, &key_dummy,
1217                                      lbase, lbase,
1218                                      &cache_index,
1219                                      HAMMER2_LOOKUP_NODATA);
1220         if (chain) {
1221                 if (chain->bref.type == HAMMER2_BREF_TYPE_INODE) {
1222                         hammer2_inode_data_t *wipdata;
1223
1224                         hammer2_chain_modify_ip(ip, chain, mtid, 0);
1225                         wipdata = &chain->data->ipdata;
1226                         KKASSERT(wipdata->meta.op_flags &
1227                                  HAMMER2_OPFLAG_DIRECTDATA);
1228                         bzero(wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1229                         ++hammer2_iod_file_wembed;
1230                 } else {
1231                         hammer2_chain_delete(*parentp, chain,
1232                                              mtid, HAMMER2_DELETE_PERMANENT);
1233                         ++hammer2_iod_file_wzero;
1234                 }
1235                 hammer2_chain_unlock(chain);
1236                 hammer2_chain_drop(chain);
1237         } else {
1238                 ++hammer2_iod_file_wzero;
1239         }
1240 }
1241
1242 /*
1243  * Helper
1244  *
1245  * Function to write the data as it is, without performing any sort of
1246  * compression. This function is used in path without compression and
1247  * default zero-checking path.
1248  */
1249 static
1250 void
1251 hammer2_write_bp(hammer2_chain_t *chain, char *data, int ioflag,
1252                  int pblksize,
1253                  hammer2_tid_t mtid, int *errorp, int check_algo)
1254 {
1255         hammer2_inode_data_t *wipdata;
1256         hammer2_io_t *dio;
1257         char *bdata;
1258         int error;
1259
1260         error = 0;      /* XXX TODO below */
1261
1262         KKASSERT(chain->flags & HAMMER2_CHAIN_MODIFIED);
1263
1264         switch(chain->bref.type) {
1265         case HAMMER2_BREF_TYPE_INODE:
1266                 wipdata = &chain->data->ipdata;
1267                 KKASSERT(wipdata->meta.op_flags & HAMMER2_OPFLAG_DIRECTDATA);
1268                 bcopy(data, wipdata->u.data, HAMMER2_EMBEDDED_BYTES);
1269                 error = 0;
1270                 ++hammer2_iod_file_wembed;
1271                 break;
1272         case HAMMER2_BREF_TYPE_DATA:
1273                 error = hammer2_io_newnz(chain->hmp,
1274                                          chain->bref.type,
1275                                          chain->bref.data_off,
1276                                          chain->bytes, &dio);
1277                 if (error) {
1278                         hammer2_io_bqrelse(&dio);
1279                         kprintf("hammer2: WRITE PATH: "
1280                                 "dbp bread error\n");
1281                         break;
1282                 }
1283                 bdata = hammer2_io_data(dio, chain->bref.data_off);
1284
1285                 chain->bref.methods = HAMMER2_ENC_COMP(HAMMER2_COMP_NONE) +
1286                                       HAMMER2_ENC_CHECK(check_algo);
1287                 bcopy(data, bdata, chain->bytes);
1288
1289                 /*
1290                  * The flush code doesn't calculate check codes for
1291                  * file data (doing so can result in excessive I/O),
1292                  * so we do it here.
1293                  */
1294                 hammer2_chain_setcheck(chain, bdata);
1295                 hammer2_dedup_record(chain, bdata);
1296
1297                 /*
1298                  * Device buffer is now valid, chain is no longer in
1299                  * the initial state.
1300                  *
1301                  * (No blockref table worries with file data)
1302                  */
1303                 atomic_clear_int(&chain->flags, HAMMER2_CHAIN_INITIAL);
1304
1305                 if (ioflag & IO_SYNC) {
1306                         /*
1307                          * Synchronous I/O requested.
1308                          */
1309                         hammer2_io_bwrite(&dio);
1310                 /*
1311                 } else if ((ioflag & IO_DIRECT) &&
1312                            loff + n == pblksize) {
1313                         hammer2_io_bdwrite(&dio);
1314                 */
1315                 } else if (ioflag & IO_ASYNC) {
1316                         hammer2_io_bawrite(&dio);
1317                 } else {
1318                         hammer2_io_bdwrite(&dio);
1319                 }
1320                 break;
1321         default:
1322                 panic("hammer2_write_bp: bad chain type %d\n",
1323                       chain->bref.type);
1324                 /* NOT REACHED */
1325                 error = 0;
1326                 break;
1327         }
1328         KKASSERT(error == 0);   /* XXX TODO */
1329         *errorp = error;
1330 }
1331
1332 /*
1333  * LIVE DEDUP HEURISTIC
1334  *
1335  * WARNING! This code is SMP safe but the heuristic allows SMP collisions.
1336  *          All fields must be loaded into locals and validated.
1337  *
1338  * WARNING! Should only be used for file data, hammer2_chain_modify() only
1339  *          checks for the dedup case on data chains.  Also, dedup data can
1340  *          only be recorded for committed chains (so NOT strategy writes
1341  *          which can undergo further modification after the fact!).
1342  */
1343 void
1344 hammer2_dedup_record(hammer2_chain_t *chain, char *data)
1345 {
1346         hammer2_dev_t *hmp;
1347         hammer2_dedup_t *dedup;
1348         uint64_t crc;
1349         int best = 0;
1350         int i;
1351         int dticks;
1352
1353         if (hammer2_dedup_enable == 0)
1354                 return;
1355
1356         /*
1357          * Only committed data can be recorded for de-duplication, otherwise
1358          * the contents may change out from under us.  So, on read if the
1359          * chain is not modified, and on flush when the chain is committed.
1360          */
1361         if ((chain->flags &
1362             (HAMMER2_CHAIN_MODIFIED | HAMMER2_CHAIN_INITIAL)) == 0) {
1363                 return;
1364         }
1365
1366
1367         hmp = chain->hmp;
1368
1369         switch(HAMMER2_DEC_CHECK(chain->bref.methods)) {
1370         case HAMMER2_CHECK_ISCSI32:
1371                 /*
1372                  * XXX use the built-in crc (the dedup lookup sequencing
1373                  * needs to be fixed so the check code is already present
1374                  * when dedup_lookup is called)
1375                  */
1376 #if 0
1377                 crc = (uint64_t)(uint32_t)chain->bref.check.iscsi32.value;
1378 #endif
1379                 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1380                 break;
1381         case HAMMER2_CHECK_XXHASH64:
1382                 crc = chain->bref.check.xxhash64.value;
1383                 break;
1384         case HAMMER2_CHECK_SHA192:
1385                 /*
1386                  * XXX use the built-in crc (the dedup lookup sequencing
1387                  * needs to be fixed so the check code is already present
1388                  * when dedup_lookup is called)
1389                  */
1390 #if 0
1391                 crc = ((uint64_t *)chain->bref.check.sha192.data)[0] ^
1392                       ((uint64_t *)chain->bref.check.sha192.data)[1] ^
1393                       ((uint64_t *)chain->bref.check.sha192.data)[2];
1394 #endif
1395                 crc = XXH64(data, chain->bytes, XXH_HAMMER2_SEED);
1396                 break;
1397         default:
1398                 /*
1399                  * Cannot dedup without a check code
1400                  *
1401                  * NOTE: In particular, CHECK_NONE allows a sector to be
1402                  *       overwritten without copy-on-write, recording
1403                  *       a dedup block for a CHECK_NONE object would be
1404                  *       a disaster!
1405                  */
1406                 return;
1407         }
1408         dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1409         for (i = 0; i < 4; ++i) {
1410                 if (dedup[i].data_crc == crc) {
1411                         best = i;
1412                         break;
1413                 }
1414                 dticks = (int)(dedup[i].ticks - dedup[best].ticks);
1415                 if (dticks < 0 || dticks > hz * 60 * 30)
1416                         best = i;
1417         }
1418         dedup += best;
1419         if (hammer2_debug & 0x40000) {
1420                 kprintf("REC %04x %016jx %016jx\n",
1421                         (int)(dedup - hmp->heur_dedup),
1422                         crc,
1423                         chain->bref.data_off);
1424         }
1425         dedup->ticks = ticks;
1426         dedup->data_off = chain->bref.data_off;
1427         dedup->data_crc = crc;
1428         atomic_set_int(&chain->flags, HAMMER2_CHAIN_DEDUP);
1429 }
1430
1431 static
1432 hammer2_off_t
1433 hammer2_dedup_lookup(hammer2_dev_t *hmp, char **datap, int pblksize)
1434 {
1435         hammer2_dedup_t *dedup;
1436         hammer2_io_t *dio;
1437         hammer2_off_t off;
1438         uint64_t crc;
1439         char *data;
1440         int i;
1441
1442         if (hammer2_dedup_enable == 0)
1443                 return 0;
1444         data = *datap;
1445         if (data == NULL)
1446                 return 0;
1447
1448         /*
1449          * XXX use the built-in crc (the dedup lookup sequencing
1450          * needs to be fixed so the check code is already present
1451          * when dedup_lookup is called)
1452          */
1453         crc = XXH64(data, pblksize, XXH_HAMMER2_SEED);
1454         dedup = &hmp->heur_dedup[crc & (HAMMER2_DEDUP_HEUR_MASK & ~3)];
1455
1456         if (hammer2_debug & 0x40000) {
1457                 kprintf("LOC %04x/4 %016jx\n",
1458                         (int)(dedup - hmp->heur_dedup),
1459                         crc);
1460         }
1461
1462         for (i = 0; i < 4; ++i) {
1463                 off = dedup[i].data_off;
1464                 cpu_ccfence();
1465                 if (dedup[i].data_crc != crc)
1466                         continue;
1467                 if ((1 << (int)(off & HAMMER2_OFF_MASK_RADIX)) != pblksize)
1468                         continue;
1469                 dio = hammer2_io_getquick(hmp, off, pblksize);
1470                 if (dio &&
1471                     bcmp(data, hammer2_io_data(dio, off), pblksize) == 0) {
1472                         /*
1473                          * Make sure the INVALOK flag is cleared to prevent
1474                          * the possibly-dirty bp from being invalidated now
1475                          * that we are using it as part of a de-dup operation.
1476                          */
1477                         if (hammer2_debug & 0x40000) {
1478                                 kprintf("DEDUP SUCCESS %016jx\n",
1479                                         (intmax_t)off);
1480                         }
1481                         atomic_clear_64(&dio->refs, HAMMER2_DIO_INVALOK);
1482                         hammer2_io_putblk(&dio);
1483                         *datap = NULL;
1484                         dedup[i].ticks = ticks; /* update use */
1485                         ++hammer2_iod_file_wdedup;
1486
1487                         return off;             /* RETURN */
1488                 }
1489                 if (dio)
1490                         hammer2_io_putblk(&dio);
1491         }
1492         return 0;
1493 }
1494
1495 /*
1496  * Poof.  Races are ok, if someone gets in and reuses a dedup offset
1497  * before or while we are clearing it they will also recover the freemap
1498  * entry (set it to fully allocated), so a bulkfree race can only set it
1499  * to a possibly-free state.
1500  *
1501  * XXX ok, well, not really sure races are ok but going to run with it
1502  *     for the moment.
1503  */
1504 void
1505 hammer2_dedup_clear(hammer2_dev_t *hmp)
1506 {
1507         int i;
1508
1509         for (i = 0; i < HAMMER2_DEDUP_HEUR_SIZE; ++i) {
1510                 hmp->heur_dedup[i].data_off = 0;
1511                 hmp->heur_dedup[i].ticks = ticks - 1;
1512         }
1513 }