sys/dev/disk/dm/crypt/dm_target_crypt.c

   1 /*
   2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Alex Hornung <ahornung@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * This file implements initial version of device-mapper crypt target.
  37  */
  38 #include <sys/endian.h>
  39
  40 #include <sys/bio.h>
  41 #include <sys/globaldata.h>
  42 #include <sys/kerneldump.h>
  43 #include <sys/malloc.h>
  44 #include <sys/mpipe.h>
  45 #include <sys/md5.h>
  46 #include <sys/mutex2.h>
  47 #include <crypto/sha1.h>
  48 #include <crypto/sha2/sha2.h>
  49 #include <opencrypto/cryptodev.h>
  50 #include <opencrypto/rmd160.h>
  51 #include <machine/cpufunc.h>
  52 #include <cpu/atomic.h>
  53
  54 #include <sys/ktr.h>
  55
  56 #include <dev/disk/dm/dm.h>
  57 MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
  58
  59 KTR_INFO_MASTER(dmcrypt);
  60
  61 #if !defined(KTR_DMCRYPT)
  62 #define KTR_DMCRYPT     KTR_ALL
  63 #endif
  64
  65 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_dispatch, 0,
  66     "crypto_dispatch(%p)", struct cryptop *crp);
  67 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypt_strategy, 0,
  68     "crypt_strategy(b_cmd = %d, bp = %p)", int cmd, struct buf *bp);
  69 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_start, 1,
  70     "crypto_write_start(crp = %p, bp = %p, sector = %d/%d)",
  71     struct cryptop *crp, struct buf *bp, int i, int sectors);
  72 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_write_done, 1,
  73     "crypto_cb_write_done(crp = %p, bp = %p, n = %d)",
  74     struct cryptop *crp, struct buf *bp, int n);
  75 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_write_done, 1,
  76     "bio_write_done(bp = %p)", struct buf *bp);
  77 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_retry, 1,
  78     "crypto_write_retry(crp = %p)", struct buf *bp);
  79 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_read_done, 2,
  80     "bio_read_done(bp = %p)", struct buf *bp);
  81 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_read_start, 2,
  82     "crypto_read_start(crp = %p, bp = %p, sector = %d/%d)",
  83     struct cryptop *crp, struct buf *bp, int i, int sectors);
  84 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_read_done, 2,
  85     "crypto_cb_read_done(crp = %p, bp = %p, n = %d)",
  86     struct cryptop *crp, struct buf *bp, int n);
  87
  88 struct target_crypt_config;
  89
  90 typedef void dispatch_t(void *);
  91 typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
  92     void *);
  93
  94 typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
  95 typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
  96
  97 struct iv_generator {
  98         const char      *name;
  99         ivgen_ctor_t    *ctor;
 100         ivgen_dtor_t    *dtor;
 101         ivgen_t         *gen_iv;
 102 };
 103
 104 struct essiv_ivgen_priv {
 105         struct cryptoini        crypto_session;
 106         struct objcache *crp_crd_cache;
 107         u_int64_t       crypto_sid;
 108         size_t          keyhash_len;
 109         u_int8_t        crypto_keyhash[SHA512_DIGEST_LENGTH];
 110 };
 111
 112 typedef struct target_crypt_config {
 113         size_t  params_len;
 114         dm_pdev_t *pdev;
 115         char    *status_str;
 116         int     crypto_alg;
 117         int     crypto_klen;
 118         u_int8_t        crypto_key[512>>3];
 119
 120         u_int64_t       crypto_sid;
 121         u_int64_t       block_offset;
 122         int64_t         iv_offset;
 123         SHA512_CTX      essivsha512_ctx;
 124
 125         struct cryptoini        crypto_session;
 126
 127         struct iv_generator     *ivgen;
 128         void    *ivgen_priv;
 129
 130         struct malloc_pipe      read_mpipe;
 131         struct malloc_pipe      write_mpipe;
 132 } dm_target_crypt_config_t;
 133
 134 struct dmtc_helper {
 135         dm_target_crypt_config_t *priv;
 136         caddr_t free_addr;
 137         caddr_t orig_buf;
 138         caddr_t data_buf;
 139 };
 140
 141 struct dmtc_dump_helper {
 142         dm_target_crypt_config_t *priv;
 143         void *data;
 144         size_t length;
 145         off_t offset;
 146
 147         int sectors;
 148         int *ident;
 149
 150         struct cryptodesc crd[128];
 151         struct cryptop crp[128];
 152         u_char space[65536];
 153 };
 154
 155 #define DMTC_BUF_SIZE_WRITE \
 156     MAXPHYS + sizeof(struct dmtc_helper) + \
 157     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 158 #define DMTC_BUF_SIZE_READ \
 159     sizeof(struct dmtc_helper) + \
 160     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 161
 162 static void dmtc_crypto_dispatch(void *arg);
 163 static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
 164                                 struct dmtc_dump_helper *dump_helper);
 165 static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
 166                                 struct bio *bio);
 167 static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
 168                                 struct bio *bio);
 169 static void dmtc_bio_read_done(struct bio *bio);
 170 static void dmtc_bio_write_done(struct bio *bio);
 171 static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
 172 static int dmtc_crypto_cb_read_done(struct cryptop *crp);
 173 static int dmtc_crypto_cb_write_done(struct cryptop *crp);
 174
 175 static ivgen_ctor_t     essiv_ivgen_ctor;
 176 static ivgen_dtor_t     essiv_ivgen_dtor;
 177 static ivgen_t          essiv_ivgen;
 178 static ivgen_t          plain_ivgen;
 179 static ivgen_t          plain64_ivgen;
 180
 181 static struct iv_generator ivgens[] = {
 182         { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
 183             .gen_iv = essiv_ivgen },
 184         { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
 185         { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
 186         { NULL, NULL, NULL, NULL }
 187 };
 188
 189 struct objcache_malloc_args essiv_ivgen_malloc_args = {
 190                 2*sizeof(void *) + (sizeof(struct cryptodesc) +
 191                 sizeof(struct cryptop)), M_DMCRYPT };
 192
 193 static void
 194 dmtc_init_mpipe(struct target_crypt_config *priv)
 195 {
 196         int nmax;
 197
 198         nmax = (physmem*2/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
 199
 200         if (nmax < 2)
 201                 nmax = 2;
 202
 203         kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
 204
 205         mpipe_init(&priv->write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
 206                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 207         mpipe_init(&priv->read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
 208                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 209 }
 210
 211 static void
 212 dmtc_destroy_mpipe(struct target_crypt_config *priv)
 213 {
 214         mpipe_done(&priv->write_mpipe);
 215         mpipe_done(&priv->read_mpipe);
 216 }
 217
 218 /*
 219  * Overwrite private information (in buf) to avoid leaking it
 220  */
 221 static void
 222 dmtc_crypto_clear(void *buf, size_t len)
 223 {
 224         memset(buf, 0xFF, len);
 225         bzero(buf, len);
 226 }
 227
 228 /*
 229  * ESSIV IV Generator Routines
 230  */
 231 static int
 232 essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
 233 {
 234         struct essiv_ivgen_priv *ivpriv;
 235         u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
 236         unsigned int klen, hashlen;
 237         int error;
 238
 239         klen = (priv->crypto_klen >> 3);
 240
 241         if (iv_hash == NULL)
 242                 return EINVAL;
 243
 244         if (!strcmp(iv_hash, "sha1")) {
 245                 SHA1_CTX ctx;
 246
 247                 hashlen = SHA1_RESULTLEN;
 248                 SHA1Init(&ctx);
 249                 SHA1Update(&ctx, priv->crypto_key, klen);
 250                 SHA1Final(crypto_keyhash, &ctx);
 251         } else if (!strcmp(iv_hash, "sha256")) {
 252                 SHA256_CTX ctx;
 253
 254                 hashlen = SHA256_DIGEST_LENGTH;
 255                 SHA256_Init(&ctx);
 256                 SHA256_Update(&ctx, priv->crypto_key, klen);
 257                 SHA256_Final(crypto_keyhash, &ctx);
 258         } else if (!strcmp(iv_hash, "sha384")) {
 259                 SHA384_CTX ctx;
 260
 261                 hashlen = SHA384_DIGEST_LENGTH;
 262                 SHA384_Init(&ctx);
 263                 SHA384_Update(&ctx, priv->crypto_key, klen);
 264                 SHA384_Final(crypto_keyhash, &ctx);
 265         } else if (!strcmp(iv_hash, "sha512")) {
 266                 SHA512_CTX ctx;
 267
 268                 hashlen = SHA512_DIGEST_LENGTH;
 269                 SHA512_Init(&ctx);
 270                 SHA512_Update(&ctx, priv->crypto_key, klen);
 271                 SHA512_Final(crypto_keyhash, &ctx);
 272         } else if (!strcmp(iv_hash, "md5")) {
 273                 MD5_CTX ctx;
 274
 275                 hashlen = MD5_DIGEST_LENGTH;
 276                 MD5Init(&ctx);
 277                 MD5Update(&ctx, priv->crypto_key, klen);
 278                 MD5Final(crypto_keyhash, &ctx);
 279         } else if (!strcmp(iv_hash, "rmd160") ||
 280                    !strcmp(iv_hash, "ripemd160")) {
 281                 RMD160_CTX ctx;
 282
 283                 hashlen = 160/8;
 284                 RMD160Init(&ctx);
 285                 RMD160Update(&ctx, priv->crypto_key, klen);
 286                 RMD160Final(crypto_keyhash, &ctx);
 287         } else {
 288                 return EINVAL;
 289         }
 290
 291         /* Convert hashlen to bits */
 292         hashlen <<= 3;
 293
 294         ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
 295             M_WAITOK | M_ZERO);
 296         memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
 297         ivpriv->keyhash_len = sizeof(crypto_keyhash);
 298         dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
 299
 300         ivpriv->crypto_session.cri_alg = priv->crypto_alg;
 301         ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
 302         ivpriv->crypto_session.cri_klen = hashlen;
 303         ivpriv->crypto_session.cri_mlen = 0;
 304         ivpriv->crypto_session.cri_next = NULL;
 305
 306         /*
 307          * XXX: in principle we also need to check if the block size of the
 308          *      cipher is a valid iv size for the block cipher.
 309          */
 310
 311         error = crypto_newsession(&ivpriv->crypto_sid,
 312                                   &ivpriv->crypto_session,
 313                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 314         if (error) {
 315                 kprintf("dm_target_crypt: Error during crypto_newsession "
 316                         "for essiv_ivgen, error = %d\n",
 317                         error);
 318                 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 319                 kfree(ivpriv, M_DMCRYPT);
 320                 return ENOTSUP;
 321         }
 322
 323         ivpriv->crp_crd_cache = objcache_create(
 324             "dmcrypt-essiv-cache", 0, 0,
 325             NULL, NULL, NULL,
 326             objcache_malloc_alloc,
 327             objcache_malloc_free,
 328             &essiv_ivgen_malloc_args );
 329
 330         *p_ivpriv = ivpriv;
 331         return 0;
 332 }
 333
 334 static int
 335 essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
 336 {
 337         struct essiv_ivgen_priv *ivpriv;
 338
 339         ivpriv = (struct essiv_ivgen_priv *)arg;
 340         KKASSERT(ivpriv != NULL);
 341
 342         crypto_freesession(ivpriv->crypto_sid);
 343
 344         objcache_destroy(ivpriv->crp_crd_cache);
 345
 346         dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 347         kfree(ivpriv, M_DMCRYPT);
 348
 349         return 0;
 350 }
 351
 352 static int
 353 essiv_ivgen_done(struct cryptop *crp)
 354 {
 355         struct essiv_ivgen_priv *ivpriv;
 356         void *free_addr;
 357         void *opaque;
 358
 359
 360         if (crp->crp_etype == EAGAIN)
 361                 return crypto_dispatch(crp);
 362
 363         if (crp->crp_etype != 0) {
 364                 kprintf("dm_target_crypt: essiv_ivgen_done, "
 365                         "crp->crp_etype = %d\n", crp->crp_etype);
 366         }
 367
 368         free_addr = crp->crp_opaque;
 369         /*
 370          * In-memory structure is:
 371          * |  ivpriv  |  opaque  |     crp     |      crd      |
 372          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 373          */
 374         ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
 375         crp->crp_opaque += sizeof(void *);
 376         opaque = *((void **)crp->crp_opaque);
 377
 378         objcache_put(ivpriv->crp_crd_cache, free_addr);
 379         dmtc_crypto_dispatch(opaque);
 380         return 0;
 381 }
 382
 383 static void
 384 essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 385             size_t iv_len, off_t sector, void *opaque)
 386 {
 387         struct essiv_ivgen_priv *ivpriv;
 388         struct cryptodesc *crd;
 389         struct cryptop *crp;
 390         caddr_t space, alloc_addr;
 391         int error;
 392
 393         ivpriv = priv->ivgen_priv;
 394         KKASSERT(ivpriv != NULL);
 395
 396         /*
 397          * In-memory structure is:
 398          * |  ivpriv  |  opaque  |     crp     |      crd      |
 399          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 400          */
 401         alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
 402         *((struct essiv_ivgen_priv **)space) = ivpriv;
 403         space += sizeof(void *);
 404         *((void **)space) = opaque;
 405         space += sizeof(void *);
 406         crp = (struct cryptop *)space;
 407         space += sizeof(struct cryptop);
 408         crd = (struct cryptodesc *)space;
 409
 410         bzero(iv, iv_len);
 411         bzero(crd, sizeof(struct cryptodesc));
 412         bzero(crp, sizeof(struct cryptop));
 413         *((off_t *)iv) = htole64(sector + priv->iv_offset);
 414         crp->crp_buf = (caddr_t)iv;
 415
 416         crp->crp_sid = ivpriv->crypto_sid;
 417         crp->crp_ilen = crp->crp_olen = iv_len;
 418
 419         crp->crp_opaque = alloc_addr;
 420
 421         crp->crp_callback = essiv_ivgen_done;
 422
 423         crp->crp_desc = crd;
 424         crp->crp_etype = 0;
 425         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
 426
 427         crd->crd_alg = priv->crypto_alg;
 428 #if 0
 429         crd->crd_key = (caddr_t)priv->crypto_keyhash;
 430         crd->crd_klen = priv->crypto_klen;
 431 #endif
 432
 433         bzero(crd->crd_iv, sizeof(crd->crd_iv));
 434
 435         crd->crd_skip = 0;
 436         crd->crd_len = iv_len;
 437         crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 438         crd->crd_flags |= CRD_F_ENCRYPT;
 439         crd->crd_next = NULL;
 440
 441         error = crypto_dispatch(crp);
 442         if (error)
 443                 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
 444 }
 445
 446
 447 static void
 448 plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 449             size_t iv_len, off_t sector, void *opaque)
 450 {
 451         bzero(iv, iv_len);
 452         *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
 453         dmtc_crypto_dispatch(opaque);
 454 }
 455
 456 static void
 457 plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 458     size_t iv_len, off_t sector, void *opaque)
 459 {
 460         bzero(iv, iv_len);
 461         *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
 462         dmtc_crypto_dispatch(opaque);
 463 }
 464
 465 #if 0
 466 static void
 467 geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 468            size_t iv_len, off_t sector, void *opaque)
 469 {
 470
 471         SHA512_CTX      ctx512;
 472         u_int8_t        md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
 473
 474         memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
 475         SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
 476         SHA512_Final(md, &ctx512);
 477
 478         memcpy(iv, md, iv_len);
 479         dmtc_crypto_dispatch(opaque);
 480 }
 481 #endif
 482
 483 /*
 484  * Init function called from dm_table_load_ioctl.
 485  * cryptsetup actually passes us this:
 486  * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
 487  */
 488 static int
 489 hex2key(char *hex, size_t key_len, u_int8_t *key)
 490 {
 491         char hex_buf[3];
 492         size_t key_idx;
 493
 494         hex_buf[2] = 0;
 495         for (key_idx = 0; key_idx < key_len; ++key_idx) {
 496                 hex_buf[0] = *hex++;
 497                 hex_buf[1] = *hex++;
 498                 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
 499         }
 500         hex_buf[0] = 0;
 501         hex_buf[1] = 0;
 502
 503         return 0;
 504 }
 505
 506 static int
 507 dm_target_crypt_init(dm_table_entry_t *table_en, int argc, char **argv)
 508 {
 509         dm_target_crypt_config_t *priv;
 510         size_t len;
 511         char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
 512         char *status_str;
 513         int i, klen, error;
 514         uint64_t iv_offset, block_offset;
 515
 516         if (argc != 5) {
 517                 kprintf("dm_target_crypt: not enough arguments, "
 518                         "need exactly 5\n");
 519                 return EINVAL;
 520         }
 521
 522         len = 0;
 523         for (i = 0; i < argc; i++) {
 524                 len += strlen(argv[i]);
 525                 len++;
 526         }
 527         /* len is strlen() of input string +1 */
 528         status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
 529
 530         crypto_alg = strsep(&argv[0], "-");
 531         crypto_mode = strsep(&argv[0], "-");
 532         iv_opt = strsep(&argv[0], "-");
 533         iv_mode = strsep(&iv_opt, ":");
 534         key = argv[1];
 535         iv_offset = strtouq(argv[2], NULL, 0);
 536         dev = argv[3];
 537         block_offset = strtouq(argv[4], NULL, 0);
 538         /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
 539         klen = strlen(key) << 2;
 540
 541 #if 0
 542         kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
 543                 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
 544                 "block_offset=%ju\n",
 545                 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
 546                 block_offset);
 547 #endif
 548
 549         priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
 550
 551         /* Insert dmp to global pdev list */
 552         if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
 553                 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
 554                 kfree(status_str, M_DMCRYPT);
 555                 return ENOENT;
 556         }
 557
 558         /*
 559          * This code checks for valid combinations of algorithm and mode.
 560          * Currently supported options are:
 561          *
 562          * *-cbc
 563          * aes-xts
 564          * twofish-xts
 565          * serpent-xts
 566          */
 567         if ((strcmp(crypto_mode, "cbc") != 0) &&
 568             !((strcmp(crypto_mode, "xts") == 0) &&
 569             ((strcmp(crypto_alg, "aes") == 0) ||
 570             (strcmp(crypto_alg, "twofish") == 0) ||
 571             (strcmp(crypto_alg, "serpent") == 0))))
 572         {
 573                 kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
 574                     " aes-xts, twofish-xts and serpent-xts, "
 575                     "invalid mode '%s-%s'\n",
 576                     crypto_alg, crypto_mode);
 577                 goto notsup;
 578         }
 579
 580         if (!strcmp(crypto_alg, "aes")) {
 581                 if (!strcmp(crypto_mode, "xts")) {
 582                         priv->crypto_alg = CRYPTO_AES_XTS;
 583                         if (klen != 256 && klen != 512)
 584                                 goto notsup;
 585                 } else if (!strcmp(crypto_mode, "cbc")) {
 586                         priv->crypto_alg = CRYPTO_AES_CBC;
 587                         if (klen != 128 && klen != 192 && klen != 256)
 588                                 goto notsup;
 589                 } else {
 590                         goto notsup;
 591                 }
 592                 priv->crypto_klen = klen;
 593         } else if (!strcmp(crypto_alg, "twofish")) {
 594                 if (!strcmp(crypto_mode, "xts")) {
 595                         priv->crypto_alg = CRYPTO_TWOFISH_XTS;
 596                         if (klen != 256 && klen != 512)
 597                                 goto notsup;
 598                 } else if (!strcmp(crypto_mode, "cbc")) {
 599                         priv->crypto_alg = CRYPTO_TWOFISH_CBC;
 600                         if (klen != 128 && klen != 192 && klen != 256)
 601                                 goto notsup;
 602                 } else {
 603                         goto notsup;
 604                 }
 605                 priv->crypto_klen = klen;
 606         } else if (!strcmp(crypto_alg, "serpent")) {
 607                 if (!strcmp(crypto_mode, "xts")) {
 608                         priv->crypto_alg = CRYPTO_SERPENT_XTS;
 609                         if (klen != 256 && klen != 512)
 610                                 goto notsup;
 611                 } else if (!strcmp(crypto_mode, "cbc")) {
 612                         priv->crypto_alg = CRYPTO_SERPENT_CBC;
 613                         if (klen != 128 && klen != 192 && klen != 256)
 614                                 goto notsup;
 615                 } else {
 616                         goto notsup;
 617                 }
 618                 priv->crypto_klen = klen;
 619         } else if (!strcmp(crypto_alg, "blowfish")) {
 620                 priv->crypto_alg = CRYPTO_BLF_CBC;
 621                 if (klen < 128 || klen > 448 || (klen % 8) != 0)
 622                         goto notsup;
 623                 priv->crypto_klen = klen;
 624         } else if (!strcmp(crypto_alg, "3des") ||
 625                    !strncmp(crypto_alg, "des3", 4)) {
 626                 priv->crypto_alg = CRYPTO_3DES_CBC;
 627                 if (klen != 168)
 628                         goto notsup;
 629                 priv->crypto_klen = 168;
 630         } else if (!strcmp(crypto_alg, "camellia")) {
 631                 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
 632                 if (klen != 128 && klen != 192 && klen != 256)
 633                         goto notsup;
 634                 priv->crypto_klen = klen;
 635         } else if (!strcmp(crypto_alg, "skipjack")) {
 636                 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
 637                 if (klen != 80)
 638                         goto notsup;
 639                 priv->crypto_klen = 80;
 640         } else if (!strcmp(crypto_alg, "cast5")) {
 641                 priv->crypto_alg = CRYPTO_CAST_CBC;
 642                 if (klen != 128)
 643                         goto notsup;
 644                 priv->crypto_klen = 128;
 645         } else if (!strcmp(crypto_alg, "null")) {
 646                 priv->crypto_alg = CRYPTO_NULL_CBC;
 647                 if (klen != 128)
 648                         goto notsup;
 649                 priv->crypto_klen = 128;
 650         } else {
 651                 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
 652                         crypto_alg);
 653                 goto notsup;
 654         }
 655
 656         /* Save length of param string */
 657         priv->params_len = len;
 658         priv->block_offset = block_offset;
 659         priv->iv_offset = iv_offset - block_offset;
 660
 661         dm_table_add_deps(table_en, priv->pdev);
 662
 663         dm_table_init_target(table_en, DM_CRYPTO_DEV, priv);
 664
 665         error = hex2key(key, priv->crypto_klen >> 3,
 666                         (u_int8_t *)priv->crypto_key);
 667
 668         if (error) {
 669                 kprintf("dm_target_crypt: hex2key failed, "
 670                         "invalid key format\n");
 671                 goto notsup;
 672         }
 673
 674         /* Handle cmd */
 675         for(i = 0; ivgens[i].name != NULL; i++) {
 676                 if (!strcmp(iv_mode, ivgens[i].name))
 677                         break;
 678         }
 679
 680         if (ivgens[i].name == NULL) {
 681                 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
 682                         iv_mode);
 683                 goto notsup;
 684         }
 685
 686         /* Call our ivgen constructor */
 687         if (ivgens[i].ctor != NULL) {
 688                 error = ivgens[i].ctor(priv, iv_opt,
 689                     &priv->ivgen_priv);
 690                 if (error) {
 691                         kprintf("dm_target_crypt: ctor for '%s' failed\n",
 692                             ivgens[i].name);
 693                         goto notsup;
 694                 }
 695         }
 696
 697         priv->ivgen = &ivgens[i];
 698
 699         priv->crypto_session.cri_alg = priv->crypto_alg;
 700         priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
 701         priv->crypto_session.cri_klen = priv->crypto_klen;
 702         priv->crypto_session.cri_mlen = 0;
 703         priv->crypto_session.cri_next = NULL;
 704
 705         error = crypto_newsession(&priv->crypto_sid,
 706                                   &priv->crypto_session,
 707                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 708         if (error) {
 709                 kprintf("dm_target_crypt: Error during crypto_newsession, "
 710                         "error = %d\n",
 711                         error);
 712                 goto notsup;
 713         }
 714
 715         memset(key, '0', strlen(key));
 716         if (iv_opt) {
 717                 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
 718                     crypto_alg, crypto_mode, iv_mode, iv_opt,
 719                     key, iv_offset, dev, block_offset);
 720         } else {
 721                 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
 722                     crypto_alg, crypto_mode, iv_mode,
 723                     key, iv_offset, dev, block_offset);
 724         }
 725         priv->status_str = status_str;
 726
 727         /* Initialize mpipes */
 728         dmtc_init_mpipe(priv);
 729
 730         return 0;
 731
 732 notsup:
 733         kprintf("dm_target_crypt: ENOTSUP\n");
 734         kfree(status_str, M_DMCRYPT);
 735         return ENOTSUP;
 736 }
 737
 738 /* Table routine called to get params string. */
 739 static char *
 740 dm_target_crypt_table(void *target_config)
 741 {
 742         dm_target_crypt_config_t *priv;
 743         char *params;
 744
 745         priv = target_config;
 746
 747         params = dm_alloc_string(DM_MAX_PARAMS_SIZE);
 748
 749         ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
 750             priv->status_str);
 751
 752         return params;
 753 }
 754
 755 static int
 756 dm_target_crypt_destroy(dm_table_entry_t *table_en)
 757 {
 758         dm_target_crypt_config_t *priv;
 759
 760         /*
 761          * Disconnect the crypt config before unbusying the target.
 762          */
 763         priv = table_en->target_config;
 764         if (priv == NULL)
 765                 return 0;
 766         dm_pdev_decr(priv->pdev);
 767
 768         /*
 769          * Clean up the crypt config
 770          *
 771          * Overwrite the private information before freeing memory to
 772          * avoid leaking it.
 773          */
 774         if (priv->status_str) {
 775                 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
 776                 kfree(priv->status_str, M_DMCRYPT);
 777                 crypto_freesession(priv->crypto_sid);
 778         }
 779
 780         if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
 781                 priv->ivgen->dtor(priv, priv->ivgen_priv);
 782         }
 783
 784         /* Destroy mpipes */
 785         dmtc_destroy_mpipe(priv);
 786
 787         dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
 788         kfree(priv, M_DMCRYPT);
 789
 790         return 0;
 791 }
 792
 793 /************************************************************************
 794  *                      STRATEGY SUPPORT FUNCTIONS                      *
 795  ************************************************************************
 796  *
 797  * READ PATH:   doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
 798  * WRITE PATH:  crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
 799  */
 800
 801 /*
 802  * Wrapper around crypto_dispatch() to match dispatch_t type
 803  */
 804 static void
 805 dmtc_crypto_dispatch(void *arg)
 806 {
 807         struct cryptop *crp;
 808
 809         crp = (struct cryptop *)arg;
 810         KKASSERT(crp != NULL);
 811         KTR_LOG(dmcrypt_crypto_dispatch, crp);
 812         crypto_dispatch(crp);
 813 }
 814
 815 /*
 816  * Start IO operation, called from dmstrategy routine.
 817  */
 818 static int
 819 dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
 820 {
 821         struct bio *bio;
 822
 823         dm_target_crypt_config_t *priv;
 824         priv = table_en->target_config;
 825
 826         /* Get rid of stuff we can't really handle */
 827         if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
 828                 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
 829                         kprintf("dm_target_crypt_strategy: can't really "
 830                                 "handle bp->b_bcount = %d\n",
 831                                 bp->b_bcount);
 832                         bp->b_error = EINVAL;
 833                         bp->b_flags |= B_ERROR | B_INVAL;
 834                         biodone(&bp->b_bio1);
 835                         return 0;
 836                 }
 837         }
 838
 839         KTR_LOG(dmcrypt_crypt_strategy, bp->b_cmd, bp);
 840
 841         switch (bp->b_cmd) {
 842         case BUF_CMD_READ:
 843                 bio = push_bio(&bp->b_bio1);
 844                 bio->bio_offset = bp->b_bio1.bio_offset +
 845                                   priv->block_offset * DEV_BSIZE;
 846                 bio->bio_caller_info1.ptr = priv;
 847                 bio->bio_done = dmtc_bio_read_done;
 848                 vn_strategy(priv->pdev->pdev_vnode, bio);
 849                 break;
 850         case BUF_CMD_WRITE:
 851                 bio = push_bio(&bp->b_bio1);
 852                 bio->bio_offset = bp->b_bio1.bio_offset +
 853                                   priv->block_offset * DEV_BSIZE;
 854                 bio->bio_caller_info1.ptr = priv;
 855                 dmtc_crypto_write_start(priv, bio);
 856                 break;
 857         default:
 858                 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
 859                 break;
 860         }
 861         return 0;
 862 }
 863
 864 /*
 865  * STRATEGY READ PATH PART 1/3 (after read BIO completes)
 866  */
 867 static void
 868 dmtc_bio_read_done(struct bio *bio)
 869 {
 870         struct bio *obio;
 871
 872         dm_target_crypt_config_t *priv;
 873
 874         KTR_LOG(dmcrypt_bio_read_done, bio->bio_buf);
 875
 876         /*
 877          * If a read error occurs we shortcut the operation, otherwise
 878          * go on to stage 2.
 879          */
 880         if (bio->bio_buf->b_flags & B_ERROR) {
 881                 obio = pop_bio(bio);
 882                 biodone(obio);
 883         } else {
 884                 priv = bio->bio_caller_info1.ptr;
 885                 dmtc_crypto_read_start(priv, bio);
 886         }
 887 }
 888
 889 /*
 890  * STRATEGY READ PATH PART 2/3
 891  */
 892 static void
 893 dmtc_crypto_read_retry(void *arg1, void *arg2)
 894 {
 895         dm_target_crypt_config_t *priv = arg1;
 896         struct bio *bio = arg2;
 897
 898         dmtc_crypto_read_start(priv, bio);
 899 }
 900
 901 static void
 902 dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
 903 {
 904         struct dmtc_helper *dmtc;
 905         struct cryptodesc *crd;
 906         struct cryptop *crp;
 907         int i, bytes, sectors, sz;
 908         off_t isector;
 909         u_char *ptr, *space;
 910
 911         /*
 912          * Note: b_resid no good after read I/O, it will be 0, use
 913          *       b_bcount.
 914          */
 915         bytes = bio->bio_buf->b_bcount;
 916         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
 917         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
 918         sz = sectors * (sizeof(*crp) + sizeof(*crd));
 919
 920         /*
 921          * For reads with bogus page we can't decrypt in place as stuff
 922          * can get ripped out from under us.
 923          *
 924          * XXX actually it looks like we can, and in any case the initial
 925          * read already completed and threw crypted data into the buffer
 926          * cache buffer.  Disable for now.
 927          */
 928         space = mpipe_alloc_callback(&priv->read_mpipe,
 929                                      dmtc_crypto_read_retry, priv, bio);
 930         if (space == NULL)
 931                 return;
 932
 933         dmtc = (struct dmtc_helper *)space;
 934         dmtc->free_addr = space;
 935         space += sizeof(struct dmtc_helper);
 936         dmtc->orig_buf = NULL;
 937         dmtc->data_buf = bio->bio_buf->b_data;
 938         dmtc->priv = priv;
 939         bio->bio_caller_info2.ptr = dmtc;
 940         bio->bio_buf->b_error = 0;
 941
 942         /*
 943          * Load crypto descriptors (crp/crd loop)
 944          */
 945         bzero(space, sz);
 946         ptr = space;
 947         bio->bio_caller_info3.value = sectors;
 948         cpu_sfence();
 949 #if 0
 950         kprintf("Read, bytes = %d (b_bcount), "
 951                 "sectors = %d (bio = %p, b_cmd = %d)\n",
 952                 bytes, sectors, bio, bio->bio_buf->b_cmd);
 953 #endif
 954         for (i = 0; i < sectors; i++) {
 955                 crp = (struct cryptop *)ptr;
 956                 ptr += sizeof(*crp);
 957                 crd = (struct cryptodesc *)ptr;
 958                 ptr += sizeof (*crd);
 959
 960                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
 961
 962                 crp->crp_sid = priv->crypto_sid;
 963                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
 964
 965                 crp->crp_opaque = (void *)bio;
 966
 967                 crp->crp_callback = dmtc_crypto_cb_read_done;
 968                 crp->crp_desc = crd;
 969                 crp->crp_etype = 0;
 970                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
 971                                  CRYPTO_F_BATCH;
 972
 973                 crd->crd_alg = priv->crypto_alg;
 974 #if 0
 975                 crd->crd_key = (caddr_t)priv->crypto_key;
 976                 crd->crd_klen = priv->crypto_klen;
 977 #endif
 978
 979                 crd->crd_skip = 0;
 980                 crd->crd_len = DEV_BSIZE /* XXX */;
 981                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 982                 crd->crd_next = NULL;
 983
 984                 crd->crd_flags &= ~CRD_F_ENCRYPT;
 985
 986                 KTR_LOG(dmcrypt_crypto_read_start, crp, bio->bio_buf, i,
 987                     sectors);
 988
 989                 /*
 990                  * Note: last argument is used to generate salt(?) and is
 991                  *       a 64 bit value, but the original code passed an
 992                  *       int.  Changing it now will break pre-existing
 993                  *       crypt volumes.
 994                  */
 995                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
 996                                     isector + i, crp);
 997         }
 998 }
 999
1000 /*
1001  * STRATEGY READ PATH PART 3/3
1002  */
1003 static int
1004 dmtc_crypto_cb_read_done(struct cryptop *crp)
1005 {
1006         struct dmtc_helper *dmtc;
1007         struct bio *bio, *obio;
1008         int n;
1009
1010         if (crp->crp_etype == EAGAIN)
1011                 return crypto_dispatch(crp);
1012
1013         bio = (struct bio *)crp->crp_opaque;
1014         KKASSERT(bio != NULL);
1015
1016         /*
1017          * Cumulative error
1018          */
1019         if (crp->crp_etype) {
1020                 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1021                         "crp_etype = %d\n",
1022                         crp->crp_etype);
1023                 bio->bio_buf->b_error = crp->crp_etype;
1024         }
1025
1026         /*
1027          * On the last chunk of the decryption we do any required copybacks
1028          * and complete the I/O.
1029          */
1030         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1031 #if 0
1032         kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1033 #endif
1034
1035         KTR_LOG(dmcrypt_crypto_cb_read_done, crp, bio->bio_buf, n);
1036
1037         if (n == 1) {
1038                 /*
1039                  * For the B_HASBOGUS case we didn't decrypt in place,
1040                  * so we need to copy stuff back into the buf.
1041                  *
1042                  * (disabled for now).
1043                  */
1044                 dmtc = bio->bio_caller_info2.ptr;
1045                 if (bio->bio_buf->b_error) {
1046                         bio->bio_buf->b_flags |= B_ERROR;
1047                 }
1048 #if 0
1049                 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
1050                         memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1051                                bio->bio_buf->b_bcount);
1052                 }
1053 #endif
1054                 mpipe_free(&dmtc->priv->read_mpipe, dmtc->free_addr);
1055                 obio = pop_bio(bio);
1056                 biodone(obio);
1057         }
1058         return 0;
1059 }
1060 /* END OF STRATEGY READ SECTION */
1061
1062 /*
1063  * STRATEGY WRITE PATH PART 1/3
1064  */
1065
1066 static void
1067 dmtc_crypto_write_retry(void *arg1, void *arg2)
1068 {
1069         dm_target_crypt_config_t *priv = arg1;
1070         struct bio *bio = arg2;
1071
1072         KTR_LOG(dmcrypt_crypto_write_retry, bio->bio_buf);
1073
1074         dmtc_crypto_write_start(priv, bio);
1075 }
1076
1077 static void
1078 dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1079 {
1080         struct dmtc_helper *dmtc;
1081         struct cryptodesc *crd;
1082         struct cryptop *crp;
1083         int i, bytes, sectors, sz;
1084         off_t isector;
1085         u_char *ptr, *space;
1086
1087         /*
1088          * Use b_bcount for consistency
1089          */
1090         bytes = bio->bio_buf->b_bcount;
1091
1092         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
1093         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1094         sz = sectors * (sizeof(*crp) + sizeof(*crd));
1095
1096         /*
1097          * For writes and reads with bogus page don't decrypt in place.
1098          */
1099         space = mpipe_alloc_callback(&priv->write_mpipe,
1100                                      dmtc_crypto_write_retry, priv, bio);
1101         if (space == NULL)
1102                 return;
1103
1104         dmtc = (struct dmtc_helper *)space;
1105         dmtc->free_addr = space;
1106         space += sizeof(struct dmtc_helper);
1107         memcpy(space + sz, bio->bio_buf->b_data, bytes);
1108
1109         bio->bio_caller_info2.ptr = dmtc;
1110         bio->bio_buf->b_error = 0;
1111
1112         dmtc->orig_buf = bio->bio_buf->b_data;
1113         dmtc->data_buf = space + sz;
1114         dmtc->priv = priv;
1115
1116         /*
1117          * Load crypto descriptors (crp/crd loop)
1118          */
1119         bzero(space, sz);
1120         ptr = space;
1121         bio->bio_caller_info3.value = sectors;
1122         cpu_sfence();
1123 #if 0
1124         kprintf("Write, bytes = %d (b_bcount), "
1125                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1126                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1127 #endif
1128         for (i = 0; i < sectors; i++) {
1129                 crp = (struct cryptop *)ptr;
1130                 ptr += sizeof(*crp);
1131                 crd = (struct cryptodesc *)ptr;
1132                 ptr += sizeof (*crd);
1133
1134                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1135
1136                 crp->crp_sid = priv->crypto_sid;
1137                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1138
1139                 crp->crp_opaque = (void *)bio;
1140
1141                 crp->crp_callback = dmtc_crypto_cb_write_done;
1142                 crp->crp_desc = crd;
1143                 crp->crp_etype = 0;
1144                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1145                                  CRYPTO_F_BATCH;
1146
1147                 crd->crd_alg = priv->crypto_alg;
1148 #if 0
1149                 crd->crd_key = (caddr_t)priv->crypto_key;
1150                 crd->crd_klen = priv->crypto_klen;
1151 #endif
1152
1153                 crd->crd_skip = 0;
1154                 crd->crd_len = DEV_BSIZE /* XXX */;
1155                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1156                 crd->crd_next = NULL;
1157
1158                 crd->crd_flags |= CRD_F_ENCRYPT;
1159
1160                 /*
1161                  * Note: last argument is used to generate salt(?) and is
1162                  *       a 64 bit value, but the original code passed an
1163                  *       int.  Changing it now will break pre-existing
1164                  *       crypt volumes.
1165                  */
1166
1167                 KTR_LOG(dmcrypt_crypto_write_start, crp, bio->bio_buf,
1168                     i, sectors);
1169
1170                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1171                                     isector + i, crp);
1172         }
1173 }
1174
1175 /*
1176  * STRATEGY WRITE PATH PART 2/3
1177  */
1178 static int
1179 dmtc_crypto_cb_write_done(struct cryptop *crp)
1180 {
1181         struct dmtc_helper *dmtc;
1182         dm_target_crypt_config_t *priv;
1183         struct bio *bio, *obio;
1184         int n;
1185
1186         if (crp->crp_etype == EAGAIN)
1187                 return crypto_dispatch(crp);
1188
1189         bio = (struct bio *)crp->crp_opaque;
1190         KKASSERT(bio != NULL);
1191
1192         /*
1193          * Cumulative error
1194          */
1195         if (crp->crp_etype != 0) {
1196                 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1197                         "crp_etype = %d\n",
1198                 crp->crp_etype);
1199                 bio->bio_buf->b_error = crp->crp_etype;
1200         }
1201
1202         /*
1203          * On the last chunk of the encryption we issue the write
1204          */
1205         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1206 #if 0
1207         kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1208 #endif
1209
1210         KTR_LOG(dmcrypt_crypto_cb_write_done, crp, bio->bio_buf, n);
1211
1212         if (n == 1) {
1213                 dmtc = bio->bio_caller_info2.ptr;
1214                 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1215
1216                 if (bio->bio_buf->b_error) {
1217                         bio->bio_buf->b_flags |= B_ERROR;
1218                         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1219                         obio = pop_bio(bio);
1220                         biodone(obio);
1221                 } else {
1222                         dmtc->orig_buf = bio->bio_buf->b_data;
1223                         bio->bio_buf->b_data = dmtc->data_buf;
1224                         bio->bio_done = dmtc_bio_write_done;
1225                         vn_strategy(priv->pdev->pdev_vnode, bio);
1226                 }
1227         }
1228         return 0;
1229 }
1230
1231 /*
1232  * STRATEGY WRITE PATH PART 3/3
1233  */
1234 static void
1235 dmtc_bio_write_done(struct bio *bio)
1236 {
1237         struct dmtc_helper *dmtc;
1238         struct bio *obio;
1239
1240         dmtc = bio->bio_caller_info2.ptr;
1241         bio->bio_buf->b_data = dmtc->orig_buf;
1242         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1243
1244         KTR_LOG(dmcrypt_bio_write_done, bio->bio_buf);
1245
1246         obio = pop_bio(bio);
1247         biodone(obio);
1248 }
1249 /* END OF STRATEGY WRITE SECTION */
1250
1251
1252
1253 /* DUMPING MAGIC */
1254
1255 extern int tsleep_crypto_dump;
1256
1257 static int
1258 dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1259 {
1260         static struct dmtc_dump_helper dump_helper;
1261         dm_target_crypt_config_t *priv;
1262         int id;
1263         static int first_call = 1;
1264
1265         priv = table_en->target_config;
1266
1267         if (first_call) {
1268                 first_call = 0;
1269                 dump_reactivate_cpus();
1270         }
1271
1272         /* Magically enable tsleep */
1273         tsleep_crypto_dump = 1;
1274         id = 0;
1275
1276         /*
1277          * 0 length means flush buffers and return
1278          */
1279         if (length == 0) {
1280                 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1281                         tsleep_crypto_dump = 0;
1282                         return ENXIO;
1283                 }
1284                 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1285                     data, 0, offset, 0);
1286                 tsleep_crypto_dump = 0;
1287                 return 0;
1288         }
1289
1290         bzero(&dump_helper, sizeof(dump_helper));
1291         dump_helper.priv = priv;
1292         dump_helper.data = data;
1293         dump_helper.length = length;
1294         dump_helper.offset = offset +
1295             priv->block_offset * DEV_BSIZE;
1296         dump_helper.ident = &id;
1297         dmtc_crypto_dump_start(priv, &dump_helper);
1298
1299         /*
1300          * Hackery to make stuff appear synchronous. The crypto callback will
1301          * set id to 1 and call wakeup on it. If the request completed
1302          * synchronously, id will be 1 and we won't bother to sleep. If not,
1303          * the crypto request will complete asynchronously and we sleep until
1304          * it's done.
1305          */
1306         if (id == 0)
1307                 tsleep(&dump_helper, 0, "cryptdump", 0);
1308
1309         dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1310             dump_helper.offset);
1311
1312         dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1313             dump_helper.space, 0, dump_helper.offset,
1314             dump_helper.length);
1315
1316         tsleep_crypto_dump = 0;
1317         return 0;
1318 }
1319
1320 static void
1321 dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1322 {
1323         struct cryptodesc *crd;
1324         struct cryptop *crp;
1325         int i, bytes, sectors;
1326         off_t isector;
1327
1328         bytes = dump_helper->length;
1329
1330         isector = dump_helper->offset / DEV_BSIZE;      /* ivgen salt base? */
1331         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1332         dump_helper->sectors = sectors;
1333 #if 0
1334         kprintf("Dump, bytes = %d, "
1335                 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1336 #endif
1337         KKASSERT(dump_helper->length <= 65536);
1338
1339         memcpy(dump_helper->space, dump_helper->data, bytes);
1340
1341         cpu_sfence();
1342
1343         for (i = 0; i < sectors; i++) {
1344                 crp = &dump_helper->crp[i];
1345                 crd = &dump_helper->crd[i];
1346
1347                 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1348
1349                 crp->crp_sid = priv->crypto_sid;
1350                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1351
1352                 crp->crp_opaque = (void *)dump_helper;
1353
1354                 crp->crp_callback = dmtc_crypto_cb_dump_done;
1355                 crp->crp_desc = crd;
1356                 crp->crp_etype = 0;
1357                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1358                                  CRYPTO_F_BATCH;
1359
1360                 crd->crd_alg = priv->crypto_alg;
1361
1362                 crd->crd_skip = 0;
1363                 crd->crd_len = DEV_BSIZE /* XXX */;
1364                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1365                 crd->crd_next = NULL;
1366
1367                 crd->crd_flags |= CRD_F_ENCRYPT;
1368
1369                 /*
1370                  * Note: last argument is used to generate salt(?) and is
1371                  *       a 64 bit value, but the original code passed an
1372                  *       int.  Changing it now will break pre-existing
1373                  *       crypt volumes.
1374                  */
1375                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1376                                     isector + i, crp);
1377         }
1378 }
1379
1380 static int
1381 dmtc_crypto_cb_dump_done(struct cryptop *crp)
1382 {
1383         struct dmtc_dump_helper *dump_helper;
1384         int n;
1385
1386         if (crp->crp_etype == EAGAIN)
1387                 return crypto_dispatch(crp);
1388
1389         dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1390         KKASSERT(dump_helper != NULL);
1391
1392         if (crp->crp_etype != 0) {
1393                 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1394                         "crp_etype = %d\n",
1395                 crp->crp_etype);
1396                 return crp->crp_etype;
1397         }
1398
1399         /*
1400          * On the last chunk of the encryption we return control
1401          */
1402         n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1403
1404         if (n == 1) {
1405                 atomic_add_int(dump_helper->ident, 1);
1406                 wakeup(dump_helper);
1407         }
1408
1409         return 0;
1410 }
1411
1412 static int
1413 dmtc_mod_handler(module_t mod, int type, void *unused)
1414 {
1415         dm_target_t *dmt = NULL;
1416         int err = 0;
1417
1418         switch (type) {
1419         case MOD_LOAD:
1420                 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1421                         dm_target_unbusy(dmt);
1422                         return EEXIST;
1423                 }
1424                 dmt = dm_target_alloc("crypt");
1425                 dmt->version[0] = 1;
1426                 dmt->version[1] = 6;
1427                 dmt->version[2] = 0;
1428                 dmt->init = &dm_target_crypt_init;
1429                 dmt->destroy = &dm_target_crypt_destroy;
1430                 dmt->strategy = &dm_target_crypt_strategy;
1431                 dmt->table = &dm_target_crypt_table;
1432                 dmt->dump = &dm_target_crypt_dump;
1433
1434                 err = dm_target_insert(dmt);
1435                 if (!err)
1436                         kprintf("dm_target_crypt: Successfully initialized\n");
1437                 break;
1438
1439         case MOD_UNLOAD:
1440                 err = dm_target_remove("crypt");
1441                 if (err == 0) {
1442                         kprintf("dm_target_crypt: unloaded\n");
1443                 }
1444                 break;
1445         }
1446
1447         return err;
1448 }
1449
1450 DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);