sys/vfs/hammer/hammer_flusher.c

   1 /*
   2  * Copyright (c) 2008 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * $DragonFly: src/sys/vfs/hammer/hammer_flusher.c,v 1.29 2008/06/23 07:31:14 dillon Exp $
  35  */
  36 /*
  37  * HAMMER dependancy flusher thread
  38  *
  39  * Meta data updates create buffer dependancies which are arranged as a
  40  * hierarchy of lists.
  41  */
  42
  43 #include "hammer.h"
  44
  45 static void hammer_flusher_master_thread(void *arg);
  46 static void hammer_flusher_slave_thread(void *arg);
  47 static void hammer_flusher_clean_loose_ios(hammer_mount_t hmp);
  48 static void hammer_flusher_flush(hammer_mount_t hmp);
  49 static void hammer_flusher_flush_inode(hammer_inode_t ip,
  50                                         hammer_transaction_t trans);
  51 static int hammer_must_finalize_undo(hammer_mount_t hmp);
  52 static void hammer_flusher_finalize(hammer_transaction_t trans, int final);
  53
  54 /*
  55  * Support structures for the flusher threads.
  56  */
  57 struct hammer_flusher_info {
  58         struct hammer_mount *hmp;
  59         thread_t        td;
  60         int             startit;
  61         hammer_inode_t  work_array[HAMMER_FLUSH_GROUP_SIZE];
  62 };
  63
  64 typedef struct hammer_flusher_info *hammer_flusher_info_t;
  65
  66 /*
  67  * Sync all inodes pending on the flusher.  This routine may have to be
  68  * called twice to get them all as some may be queued to a later flush group.
  69  */
  70 void
  71 hammer_flusher_sync(hammer_mount_t hmp)
  72 {
  73         int seq;
  74
  75         if (hmp->flusher.td) {
  76                 seq = hmp->flusher.next;
  77                 if (hmp->flusher.signal++ == 0)
  78                         wakeup(&hmp->flusher.signal);
  79                 while ((int)(seq - hmp->flusher.done) > 0)
  80                         tsleep(&hmp->flusher.done, 0, "hmrfls", 0);
  81         }
  82 }
  83
  84 /*
  85  * Sync all inodes pending on the flusher - return immediately.
  86  */
  87 void
  88 hammer_flusher_async(hammer_mount_t hmp)
  89 {
  90         if (hmp->flusher.td) {
  91                 if (hmp->flusher.signal++ == 0)
  92                         wakeup(&hmp->flusher.signal);
  93         }
  94 }
  95
  96 void
  97 hammer_flusher_create(hammer_mount_t hmp)
  98 {
  99         hammer_flusher_info_t info;
 100         int i;
 101
 102         hmp->flusher.signal = 0;
 103         hmp->flusher.act = 0;
 104         hmp->flusher.done = 0;
 105         hmp->flusher.next = 1;
 106         hmp->flusher.count = 0;
 107         hammer_ref(&hmp->flusher.finalize_lock);
 108
 109         lwkt_create(hammer_flusher_master_thread, hmp,
 110                     &hmp->flusher.td, NULL, 0, -1, "hammer-M");
 111         for (i = 0; i < HAMMER_MAX_FLUSHERS; ++i) {
 112                 info = kmalloc(sizeof(*info), M_HAMMER, M_WAITOK|M_ZERO);
 113                 info->hmp = hmp;
 114                 ++hmp->flusher.count;
 115                 hmp->flusher.info[i] = info;
 116                 lwkt_create(hammer_flusher_slave_thread, info,
 117                             &info->td, NULL, 0, -1, "hammer-S%d", i);
 118         }
 119 }
 120
 121 void
 122 hammer_flusher_destroy(hammer_mount_t hmp)
 123 {
 124         hammer_flusher_info_t info;
 125         int i;
 126
 127         /*
 128          * Kill the master
 129          */
 130         hmp->flusher.exiting = 1;
 131         while (hmp->flusher.td) {
 132                 ++hmp->flusher.signal;
 133                 wakeup(&hmp->flusher.signal);
 134                 tsleep(&hmp->flusher.exiting, 0, "hmrwex", hz);
 135         }
 136
 137         /*
 138          * Kill the slaves
 139          */
 140         for (i = 0; i < HAMMER_MAX_FLUSHERS; ++i) {
 141                 if ((info = hmp->flusher.info[i]) != NULL) {
 142                         KKASSERT(info->startit == 0);
 143                         info->startit = -1;
 144                         wakeup(&info->startit);
 145                         while (info->td) {
 146                                 tsleep(&info->td, 0, "hmrwwc", 0);
 147                         }
 148                         hmp->flusher.info[i] = NULL;
 149                         kfree(info, M_HAMMER);
 150                         --hmp->flusher.count;
 151                 }
 152         }
 153         KKASSERT(hmp->flusher.count == 0);
 154 }
 155
 156 /*
 157  * The master flusher thread manages the flusher sequence id and
 158  * synchronization with the slave work threads.
 159  */
 160 static void
 161 hammer_flusher_master_thread(void *arg)
 162 {
 163         hammer_mount_t hmp = arg;
 164
 165         for (;;) {
 166                 while (hmp->flusher.group_lock)
 167                         tsleep(&hmp->flusher.group_lock, 0, "hmrhld", 0);
 168                 hmp->flusher.act = hmp->flusher.next;
 169                 ++hmp->flusher.next;
 170                 hammer_flusher_clean_loose_ios(hmp);
 171                 hammer_flusher_flush(hmp);
 172                 hmp->flusher.done = hmp->flusher.act;
 173                 wakeup(&hmp->flusher.done);
 174
 175                 /*
 176                  * Wait for activity.
 177                  */
 178                 if (hmp->flusher.exiting && TAILQ_EMPTY(&hmp->flush_list))
 179                         break;
 180
 181                 /*
 182                  * This is a hack until we can dispose of frontend buffer
 183                  * cache buffers on the frontend.
 184                  */
 185                 while (hmp->flusher.signal == 0)
 186                         tsleep(&hmp->flusher.signal, 0, "hmrwwa", 0);
 187                 hmp->flusher.signal = 0;
 188         }
 189
 190         /*
 191          * And we are done.
 192          */
 193         hmp->flusher.td = NULL;
 194         wakeup(&hmp->flusher.exiting);
 195         lwkt_exit();
 196 }
 197
 198 /*
 199  * The slave flusher thread pulls work off the master flush_list until no
 200  * work is left.
 201  */
 202 static void
 203 hammer_flusher_slave_thread(void *arg)
 204 {
 205         hammer_flusher_info_t info;
 206         hammer_mount_t hmp;
 207         hammer_inode_t ip;
 208         int c;
 209         int i;
 210         int n;
 211
 212         info = arg;
 213         hmp = info->hmp;
 214
 215         for (;;) {
 216                 while (info->startit == 0)
 217                         tsleep(&info->startit, 0, "hmrssw", 0);
 218                 if (info->startit < 0)
 219                         break;
 220                 info->startit = 0;
 221
 222                 /*
 223                  * Try to pull out around ~64 inodes at a time to flush.
 224                  * The idea is to try to avoid deadlocks between the slaves.
 225                  */
 226                 n = c = 0;
 227                 while ((ip = TAILQ_FIRST(&hmp->flush_list)) != NULL) {
 228                         if (ip->flush_group != hmp->flusher.act)
 229                                 break;
 230                         TAILQ_REMOVE(&hmp->flush_list, ip, flush_entry);
 231                         info->work_array[n++] = ip;
 232                         c += ip->rsv_recs;
 233                         if (n < HAMMER_FLUSH_GROUP_SIZE &&
 234                             c < HAMMER_FLUSH_GROUP_SIZE * 8) {
 235                                 continue;
 236                         }
 237                         for (i = 0; i < n; ++i){
 238                                 hammer_flusher_flush_inode(info->work_array[i],
 239                                                         &hmp->flusher.trans);
 240                         }
 241                         n = c = 0;
 242                 }
 243                 for (i = 0; i < n; ++i) {
 244                         hammer_flusher_flush_inode(info->work_array[i],
 245                                                    &hmp->flusher.trans);
 246                 }
 247                 if (--hmp->flusher.running == 0)
 248                         wakeup(&hmp->flusher.running);
 249         }
 250         info->td = NULL;
 251         wakeup(&info->td);
 252         lwkt_exit();
 253 }
 254
 255 static void
 256 hammer_flusher_clean_loose_ios(hammer_mount_t hmp)
 257 {
 258         hammer_buffer_t buffer;
 259         hammer_io_t io;
 260         int panic_count = 1000000;
 261
 262         /*
 263          * loose ends - buffers without bp's aren't tracked by the kernel
 264          * and can build up, so clean them out.  This can occur when an
 265          * IO completes on a buffer with no references left.
 266          */
 267         crit_enter();   /* biodone() race */
 268         while ((io = TAILQ_FIRST(&hmp->lose_list)) != NULL) {
 269                 KKASSERT(--panic_count > 0);
 270                 KKASSERT(io->mod_list == &hmp->lose_list);
 271                 TAILQ_REMOVE(&hmp->lose_list, io, mod_entry);
 272                 io->mod_list = NULL;
 273                 if (io->lock.refs == 0)
 274                         ++hammer_count_refedbufs;
 275                 hammer_ref(&io->lock);
 276                 buffer = (void *)io;
 277                 hammer_rel_buffer(buffer, 0);
 278         }
 279         crit_exit();
 280 }
 281
 282 /*
 283  * Flush all inodes in the current flush group.
 284  */
 285 static void
 286 hammer_flusher_flush(hammer_mount_t hmp)
 287 {
 288         hammer_flusher_info_t info;
 289         hammer_reserve_t resv;
 290         int i;
 291         int n;
 292
 293         hammer_start_transaction_fls(&hmp->flusher.trans, hmp);
 294
 295         /*
 296          * Start work threads.
 297          */
 298         i = 0;
 299         n = hmp->count_iqueued / HAMMER_FLUSH_GROUP_SIZE;
 300         if (TAILQ_FIRST(&hmp->flush_list)) {
 301                 for (i = 0; i <= n; ++i) {
 302                         if (i == HAMMER_MAX_FLUSHERS ||
 303                             hmp->flusher.info[i] == NULL) {
 304                                 break;
 305                         }
 306                         info = hmp->flusher.info[i];
 307                         if (info->startit == 0) {
 308                                 ++hmp->flusher.running;
 309                                 info->startit = 1;
 310                                 wakeup(&info->startit);
 311                         }
 312                 }
 313         }
 314         while (hmp->flusher.running)
 315                 tsleep(&hmp->flusher.running, 0, "hmrfcc", 0);
 316
 317         hammer_flusher_finalize(&hmp->flusher.trans, 1);
 318         hmp->flusher.tid = hmp->flusher.trans.tid;
 319
 320         /*
 321          * Clean up any freed big-blocks (typically zone-2).
 322          * resv->flush_group is typically set several flush groups ahead
 323          * of the free to ensure that the freed block is not reused until
 324          * it can no longer be reused.
 325          */
 326         while ((resv = TAILQ_FIRST(&hmp->delay_list)) != NULL) {
 327                 if (resv->flush_group != hmp->flusher.act)
 328                         break;
 329                 hammer_reserve_clrdelay(hmp, resv);
 330         }
 331         hammer_done_transaction(&hmp->flusher.trans);
 332 }
 333
 334 /*
 335  * Flush a single inode that is part of a flush group.
 336  */
 337 static
 338 void
 339 hammer_flusher_flush_inode(hammer_inode_t ip, hammer_transaction_t trans)
 340 {
 341         hammer_mount_t hmp = ip->hmp;
 342
 343         hammer_lock_sh(&hmp->flusher.finalize_lock);
 344         ip->error = hammer_sync_inode(ip);
 345         hammer_flush_inode_done(ip);
 346         hammer_unlock(&hmp->flusher.finalize_lock);
 347         while (hmp->flusher.finalize_want)
 348                 tsleep(&hmp->flusher.finalize_want, 0, "hmrsxx", 0);
 349         if (hammer_must_finalize_undo(hmp)) {
 350                 hmp->flusher.finalize_want = 1;
 351                 hammer_lock_ex(&hmp->flusher.finalize_lock);
 352                 kprintf("HAMMER: Warning: UNDO area too small!\n");
 353                 hammer_flusher_finalize(trans, 1);
 354                 hammer_unlock(&hmp->flusher.finalize_lock);
 355                 hmp->flusher.finalize_want = 0;
 356                 wakeup(&hmp->flusher.finalize_want);
 357         } else if (trans->hmp->locked_dirty_count +
 358                    trans->hmp->io_running_count > hammer_limit_dirtybufs) {
 359                 hmp->flusher.finalize_want = 1;
 360                 hammer_lock_ex(&hmp->flusher.finalize_lock);
 361                 hammer_flusher_finalize(trans, 0);
 362                 hammer_unlock(&hmp->flusher.finalize_lock);
 363                 hmp->flusher.finalize_want = 0;
 364                 wakeup(&hmp->flusher.finalize_want);
 365         }
 366 }
 367
 368 /*
 369  * If the UNDO area gets over half full we have to flush it.  We can't
 370  * afford the UNDO area becoming completely full as that would break
 371  * the crash recovery atomicy.
 372  */
 373 static
 374 int
 375 hammer_must_finalize_undo(hammer_mount_t hmp)
 376 {
 377         if (hammer_undo_space(hmp) < hammer_undo_max(hmp) / 2) {
 378                 hkprintf("*");
 379                 return(1);
 380         } else {
 381                 return(0);
 382         }
 383 }
 384
 385 /*
 386  * Flush all pending UNDOs, wait for write completion, update the volume
 387  * header with the new UNDO end position, and flush it.  Then
 388  * asynchronously flush the meta-data.
 389  *
 390  * If this is the last finalization in a flush group we also synchronize
 391  * our cached blockmap and set hmp->flusher_undo_start and our cached undo
 392  * fifo first_offset so the next flush resets the FIFO pointers.
 393  */
 394 static
 395 void
 396 hammer_flusher_finalize(hammer_transaction_t trans, int final)
 397 {
 398         hammer_volume_t root_volume;
 399         hammer_blockmap_t cundomap, dundomap;
 400         hammer_mount_t hmp;
 401         hammer_io_t io;
 402         int count;
 403         int i;
 404
 405         hmp = trans->hmp;
 406         root_volume = trans->rootvol;
 407
 408         /*
 409          * Flush data buffers.  This can occur asynchronously and at any
 410          * time.  We must interlock against the frontend direct-data write
 411          * but do not have to acquire the sync-lock yet.
 412          */
 413         count = 0;
 414         while ((io = TAILQ_FIRST(&hmp->data_list)) != NULL) {
 415                 if (io->lock.refs == 0)
 416                         ++hammer_count_refedbufs;
 417                 hammer_ref(&io->lock);
 418                 hammer_io_write_interlock(io);
 419                 KKASSERT(io->type != HAMMER_STRUCTURE_VOLUME);
 420                 hammer_io_flush(io);
 421                 hammer_io_done_interlock(io);
 422                 hammer_rel_buffer((hammer_buffer_t)io, 0);
 423                 ++count;
 424         }
 425
 426         /*
 427          * The sync-lock is required for the remaining sequence.  This lock
 428          * prevents meta-data from being modified.
 429          */
 430         hammer_sync_lock_ex(trans);
 431
 432         /*
 433          * If we have been asked to finalize the volume header sync the
 434          * cached blockmap to the on-disk blockmap.  Generate an UNDO
 435          * record for the update.
 436          */
 437         if (final) {
 438                 cundomap = &hmp->blockmap[0];
 439                 dundomap = &root_volume->ondisk->vol0_blockmap[0];
 440                 if (root_volume->io.modified) {
 441                         hammer_modify_volume(trans, root_volume,
 442                                              dundomap, sizeof(hmp->blockmap));
 443                         for (i = 0; i < HAMMER_MAX_ZONES; ++i)
 444                                 hammer_crc_set_blockmap(&cundomap[i]);
 445                         bcopy(cundomap, dundomap, sizeof(hmp->blockmap));
 446                         hammer_modify_volume_done(root_volume);
 447                 }
 448         }
 449
 450         /*
 451          * Flush UNDOs
 452          */
 453         count = 0;
 454         while ((io = TAILQ_FIRST(&hmp->undo_list)) != NULL) {
 455                 KKASSERT(io->modify_refs == 0);
 456                 if (io->lock.refs == 0)
 457                         ++hammer_count_refedbufs;
 458                 hammer_ref(&io->lock);
 459                 KKASSERT(io->type != HAMMER_STRUCTURE_VOLUME);
 460                 hammer_io_flush(io);
 461                 hammer_rel_buffer((hammer_buffer_t)io, 0);
 462                 ++count;
 463         }
 464
 465         /*
 466          * Wait for I/Os to complete
 467          */
 468         hammer_flusher_clean_loose_ios(hmp);
 469         hammer_io_wait_all(hmp, "hmrfl1");
 470
 471         /*
 472          * Update the on-disk volume header with new UNDO FIFO end position
 473          * (do not generate new UNDO records for this change).  We have to
 474          * do this for the UNDO FIFO whether (final) is set or not.
 475          *
 476          * Also update the on-disk next_tid field.  This does not require
 477          * an UNDO.  However, because our TID is generated before we get
 478          * the sync lock another sync may have beat us to the punch.
 479          *
 480          * The volume header will be flushed out synchronously.
 481          */
 482         dundomap = &root_volume->ondisk->vol0_blockmap[HAMMER_ZONE_UNDO_INDEX];
 483         cundomap = &hmp->blockmap[HAMMER_ZONE_UNDO_INDEX];
 484
 485         if (dundomap->first_offset != cundomap->first_offset ||
 486             dundomap->next_offset != cundomap->next_offset) {
 487                 hammer_modify_volume(NULL, root_volume, NULL, 0);
 488                 dundomap->first_offset = cundomap->first_offset;
 489                 dundomap->next_offset = cundomap->next_offset;
 490                 hammer_crc_set_blockmap(dundomap);
 491                 hammer_crc_set_volume(root_volume->ondisk);
 492                 if (root_volume->ondisk->vol0_next_tid < trans->tid)
 493                         root_volume->ondisk->vol0_next_tid = trans->tid;
 494                 hammer_modify_volume_done(root_volume);
 495         }
 496
 497         if (root_volume->io.modified) {
 498                 hammer_io_flush(&root_volume->io);
 499         }
 500
 501         /*
 502          * Wait for I/Os to complete
 503          */
 504         hammer_flusher_clean_loose_ios(hmp);
 505         hammer_io_wait_all(hmp, "hmrfl2");
 506
 507         /*
 508          * Flush meta-data.  The meta-data will be undone if we crash
 509          * so we can safely flush it asynchronously.
 510          *
 511          * Repeated catchups will wind up flushing this update's meta-data
 512          * and the UNDO buffers for the next update simultaniously.  This
 513          * is ok.
 514          */
 515         count = 0;
 516         while ((io = TAILQ_FIRST(&hmp->meta_list)) != NULL) {
 517                 KKASSERT(io->modify_refs == 0);
 518                 if (io->lock.refs == 0)
 519                         ++hammer_count_refedbufs;
 520                 hammer_ref(&io->lock);
 521                 KKASSERT(io->type != HAMMER_STRUCTURE_VOLUME);
 522                 hammer_io_flush(io);
 523                 hammer_rel_buffer((hammer_buffer_t)io, 0);
 524                 ++count;
 525         }
 526
 527         /*
 528          * If this is the final finalization for the flush group set
 529          * up for the next sequence by setting a new first_offset in
 530          * our cached blockmap and
 531          * clearing the undo history.
 532          */
 533         if (final) {
 534                 cundomap = &hmp->blockmap[HAMMER_ZONE_UNDO_INDEX];
 535                 cundomap->first_offset = cundomap->next_offset;
 536                 hammer_clear_undo_history(hmp);
 537         }
 538
 539         hammer_sync_unlock(trans);
 540 }
 541