sys/dev/disk/dm/targets/crypt/dm_target_crypt.c

   1 /*
   2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Alex Hornung <ahornung@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * This file implements initial version of device-mapper crypt target.
  37  */
  38 #include <sys/types.h>
  39 #include <sys/param.h>
  40 #include <sys/endian.h>
  41
  42 #include <sys/bio.h>
  43 #include <sys/buf.h>
  44 #include <sys/globaldata.h>
  45 #include <sys/kerneldump.h>
  46 #include <sys/malloc.h>
  47 #include <sys/mpipe.h>
  48 #include <sys/md5.h>
  49 #include <sys/mutex2.h>
  50 #include <sys/vnode.h>
  51 #include <crypto/sha1.h>
  52 #include <crypto/sha2/sha2.h>
  53 #include <opencrypto/cryptodev.h>
  54 #include <opencrypto/rmd160.h>
  55 #include <machine/cpufunc.h>
  56
  57 #include <dev/disk/dm/dm.h>
  58 MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
  59
  60 struct target_crypt_config;
  61
  62 typedef void dispatch_t(void *);
  63 typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
  64     void *);
  65
  66 typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
  67 typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
  68
  69 struct iv_generator {
  70         const char      *name;
  71         ivgen_ctor_t    *ctor;
  72         ivgen_dtor_t    *dtor;
  73         ivgen_t         *gen_iv;
  74 };
  75
  76 struct essiv_ivgen_priv {
  77         struct cryptoini        crypto_session;
  78         struct objcache *crp_crd_cache;
  79         u_int64_t       crypto_sid;
  80         size_t          keyhash_len;
  81         u_int8_t        crypto_keyhash[SHA512_DIGEST_LENGTH];
  82 };
  83
  84 typedef struct target_crypt_config {
  85         size_t  params_len;
  86         dm_pdev_t *pdev;
  87         char    *status_str;
  88         int     crypto_alg;
  89         int     crypto_klen;
  90         u_int8_t        crypto_key[512>>3];
  91
  92         u_int64_t       crypto_sid;
  93         u_int64_t       block_offset;
  94         int64_t         iv_offset;
  95         SHA512_CTX      essivsha512_ctx;
  96
  97         struct cryptoini        crypto_session;
  98
  99         struct iv_generator     *ivgen;
 100         void    *ivgen_priv;
 101 } dm_target_crypt_config_t;
 102
 103 struct dmtc_helper {
 104         caddr_t free_addr;
 105         caddr_t orig_buf;
 106         caddr_t data_buf;
 107 };
 108
 109 struct dmtc_dump_helper {
 110         dm_target_crypt_config_t *priv;
 111         void *data;
 112         size_t length;
 113         off_t offset;
 114
 115         int sectors;
 116         int *ident;
 117
 118         struct cryptodesc crd[128];
 119         struct cryptop crp[128];
 120         u_char space[65536];
 121 };
 122
 123 #define DMTC_BUF_SIZE_WRITE \
 124     MAXPHYS + sizeof(struct dmtc_helper) + \
 125     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 126 #define DMTC_BUF_SIZE_READ \
 127     sizeof(struct dmtc_helper) + \
 128     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 129
 130 static void dmtc_crypto_dispatch(void *arg);
 131 static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
 132                                 struct dmtc_dump_helper *dump_helper);
 133 static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
 134                                 struct bio *bio);
 135 static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
 136                                 struct bio *bio);
 137 static void dmtc_bio_read_done(struct bio *bio);
 138 static void dmtc_bio_write_done(struct bio *bio);
 139 static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
 140 static int dmtc_crypto_cb_read_done(struct cryptop *crp);
 141 static int dmtc_crypto_cb_write_done(struct cryptop *crp);
 142
 143 static ivgen_ctor_t     essiv_ivgen_ctor;
 144 static ivgen_dtor_t     essiv_ivgen_dtor;
 145 static ivgen_t          essiv_ivgen;
 146 static ivgen_t          plain_ivgen;
 147 static ivgen_t          plain64_ivgen;
 148
 149 static struct iv_generator ivgens[] = {
 150         { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
 151             .gen_iv = essiv_ivgen },
 152         { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
 153         { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
 154         { NULL, NULL, NULL, NULL }
 155 };
 156
 157 struct objcache_malloc_args essiv_ivgen_malloc_args = {
 158                 2*sizeof(void *) + (sizeof(struct cryptodesc) +
 159                 sizeof(struct cryptop)), M_DMCRYPT };
 160
 161 static struct malloc_pipe dmtc_read_mpipe;
 162 static struct malloc_pipe dmtc_write_mpipe;
 163
 164 static void
 165 dmtc_init_mpipe(void)
 166 {
 167         int nmax;
 168
 169         nmax = (physmem*5/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
 170
 171         if (nmax < 2)
 172                 nmax = 2;
 173
 174         kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
 175
 176         mpipe_init(&dmtc_write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
 177                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 178         mpipe_init(&dmtc_read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
 179                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 180 }
 181
 182 static void
 183 dmtc_destroy_mpipe(void)
 184 {
 185         mpipe_done(&dmtc_write_mpipe);
 186         mpipe_done(&dmtc_read_mpipe);
 187 }
 188
 189 /*
 190  * Overwrite private information (in buf) to avoid leaking it
 191  */
 192 static void
 193 dmtc_crypto_clear(void *buf, size_t len)
 194 {
 195         memset(buf, 0xFF, len);
 196         bzero(buf, len);
 197 }
 198
 199 /*
 200  * ESSIV IV Generator Routines
 201  */
 202 static int
 203 essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
 204 {
 205         struct essiv_ivgen_priv *ivpriv;
 206         u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
 207         unsigned int klen, hashlen;
 208         int error;
 209
 210         klen = (priv->crypto_klen >> 3);
 211
 212         if (iv_hash == NULL)
 213                 return EINVAL;
 214
 215         if (!strcmp(iv_hash, "sha1")) {
 216                 SHA1_CTX ctx;
 217
 218                 hashlen = SHA1_RESULTLEN;
 219                 SHA1Init(&ctx);
 220                 SHA1Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 221                 SHA1Final(crypto_keyhash, &ctx);
 222         } else if (!strcmp(iv_hash, "sha256")) {
 223                 SHA256_CTX ctx;
 224
 225                 hashlen = SHA256_DIGEST_LENGTH;
 226                 SHA256_Init(&ctx);
 227                 SHA256_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 228                 SHA256_Final(crypto_keyhash, &ctx);
 229         } else if (!strcmp(iv_hash, "sha384")) {
 230                 SHA384_CTX ctx;
 231
 232                 hashlen = SHA384_DIGEST_LENGTH;
 233                 SHA384_Init(&ctx);
 234                 SHA384_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 235                 SHA384_Final(crypto_keyhash, &ctx);
 236         } else if (!strcmp(iv_hash, "sha512")) {
 237                 SHA512_CTX ctx;
 238
 239                 hashlen = SHA512_DIGEST_LENGTH;
 240                 SHA512_Init(&ctx);
 241                 SHA512_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 242                 SHA512_Final(crypto_keyhash, &ctx);
 243         } else if (!strcmp(iv_hash, "md5")) {
 244                 MD5_CTX ctx;
 245
 246                 hashlen = MD5_DIGEST_LENGTH;
 247                 MD5Init(&ctx);
 248                 MD5Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 249                 MD5Final(crypto_keyhash, &ctx);
 250         } else if (!strcmp(iv_hash, "rmd160") ||
 251                    !strcmp(iv_hash, "ripemd160")) {
 252                 RMD160_CTX ctx;
 253
 254                 hashlen = 160/8;
 255                 RMD160Init(&ctx);
 256                 RMD160Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 257                 RMD160Final(crypto_keyhash, &ctx);
 258         } else {
 259                 return EINVAL;
 260         }
 261
 262         /* Convert hashlen to bits */
 263         hashlen <<= 3;
 264
 265         ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
 266             M_WAITOK | M_ZERO);
 267         memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
 268         ivpriv->keyhash_len = sizeof(crypto_keyhash);
 269         dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
 270
 271         ivpriv->crypto_session.cri_alg = priv->crypto_alg;
 272         ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
 273         ivpriv->crypto_session.cri_klen = hashlen;
 274         ivpriv->crypto_session.cri_mlen = 0;
 275         ivpriv->crypto_session.cri_next = NULL;
 276
 277         /*
 278          * XXX: in principle we also need to check if the block size of the
 279          *      cipher is a valid iv size for the block cipher.
 280          */
 281
 282         error = crypto_newsession(&ivpriv->crypto_sid,
 283                                   &ivpriv->crypto_session,
 284                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 285         if (error) {
 286                 kprintf("dm_target_crypt: Error during crypto_newsession "
 287                         "for essiv_ivgen, error = %d\n",
 288                         error);
 289                 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 290                 kfree(ivpriv, M_DMCRYPT);
 291                 return ENOTSUP;
 292         }
 293
 294         ivpriv->crp_crd_cache = objcache_create(
 295             "dmcrypt-essiv-cache", 0, 0,
 296             NULL, NULL, NULL,
 297             objcache_malloc_alloc,
 298             objcache_malloc_free,
 299             &essiv_ivgen_malloc_args );
 300
 301         *p_ivpriv = ivpriv;
 302         return 0;
 303 }
 304
 305 static int
 306 essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
 307 {
 308         struct essiv_ivgen_priv *ivpriv;
 309
 310         ivpriv = (struct essiv_ivgen_priv *)arg;
 311         KKASSERT(ivpriv != NULL);
 312
 313         crypto_freesession(ivpriv->crypto_sid);
 314
 315         objcache_destroy(ivpriv->crp_crd_cache);
 316
 317         dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 318         kfree(ivpriv, M_DMCRYPT);
 319
 320         return 0;
 321 }
 322
 323 static int
 324 essiv_ivgen_done(struct cryptop *crp)
 325 {
 326         struct essiv_ivgen_priv *ivpriv;
 327         void *free_addr;
 328         void *opaque;
 329
 330
 331         if (crp->crp_etype == EAGAIN)
 332                 return crypto_dispatch(crp);
 333
 334         if (crp->crp_etype != 0) {
 335                 kprintf("dm_target_crypt: essiv_ivgen_done, "
 336                         "crp->crp_etype = %d\n", crp->crp_etype);
 337         }
 338
 339         free_addr = crp->crp_opaque;
 340         /*
 341          * In-memory structure is:
 342          * |  ivpriv  |  opaque  |     crp     |      crd      |
 343          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 344          */
 345         ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
 346         crp->crp_opaque += sizeof(void *);
 347         opaque = *((void **)crp->crp_opaque);
 348
 349         objcache_put(ivpriv->crp_crd_cache, free_addr);
 350         dmtc_crypto_dispatch(opaque);
 351         return 0;
 352 }
 353
 354 static void
 355 essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 356             size_t iv_len, off_t sector, void *opaque)
 357 {
 358         struct essiv_ivgen_priv *ivpriv;
 359         struct cryptodesc *crd;
 360         struct cryptop *crp;
 361         caddr_t space, alloc_addr;
 362         int error;
 363
 364         ivpriv = priv->ivgen_priv;
 365         KKASSERT(ivpriv != NULL);
 366
 367         /*
 368          * In-memory structure is:
 369          * |  ivpriv  |  opaque  |     crp     |      crd      |
 370          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 371          */
 372         alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
 373         *((struct essiv_ivgen_priv **)space) = ivpriv;
 374         space += sizeof(void *);
 375         *((void **)space) = opaque;
 376         space += sizeof(void *);
 377         crp = (struct cryptop *)space;
 378         space += sizeof(struct cryptop);
 379         crd = (struct cryptodesc *)space;
 380
 381         bzero(iv, iv_len);
 382         bzero(crd, sizeof(struct cryptodesc));
 383         bzero(crp, sizeof(struct cryptop));
 384         *((off_t *)iv) = htole64(sector + priv->iv_offset);
 385         crp->crp_buf = (caddr_t)iv;
 386
 387         crp->crp_sid = ivpriv->crypto_sid;
 388         crp->crp_ilen = crp->crp_olen = iv_len;
 389
 390         crp->crp_opaque = alloc_addr;
 391
 392         crp->crp_callback = essiv_ivgen_done;
 393
 394         crp->crp_desc = crd;
 395         crp->crp_etype = 0;
 396         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
 397
 398         crd->crd_alg = priv->crypto_alg;
 399 #if 0
 400         crd->crd_key = (caddr_t)priv->crypto_keyhash;
 401         crd->crd_klen = priv->crypto_klen;
 402 #endif
 403
 404         bzero(crd->crd_iv, sizeof(crd->crd_iv));
 405
 406         crd->crd_skip = 0;
 407         crd->crd_len = iv_len;
 408         crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 409         crd->crd_flags |= CRD_F_ENCRYPT;
 410         crd->crd_next = NULL;
 411
 412         error = crypto_dispatch(crp);
 413         if (error)
 414                 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
 415 }
 416
 417
 418 static void
 419 plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 420             size_t iv_len, off_t sector, void *opaque)
 421 {
 422         bzero(iv, iv_len);
 423         *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
 424         dmtc_crypto_dispatch(opaque);
 425 }
 426
 427 static void
 428 plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 429     size_t iv_len, off_t sector, void *opaque)
 430 {
 431         bzero(iv, iv_len);
 432         *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
 433         dmtc_crypto_dispatch(opaque);
 434 }
 435
 436 #if 0
 437 static void
 438 geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 439            size_t iv_len, off_t sector, void *opaque)
 440 {
 441
 442         SHA512_CTX      ctx512;
 443         u_int8_t        md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
 444
 445         memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
 446         SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
 447         SHA512_Final(md, &ctx512);
 448
 449         memcpy(iv, md, iv_len);
 450         dmtc_crypto_dispatch(opaque);
 451 }
 452 #endif
 453
 454 /*
 455  * Init function called from dm_table_load_ioctl.
 456  * cryptsetup actually passes us this:
 457  * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
 458  */
 459 static int
 460 hex2key(char *hex, size_t key_len, u_int8_t *key)
 461 {
 462         char hex_buf[3];
 463         size_t key_idx;
 464
 465         hex_buf[2] = 0;
 466         for (key_idx = 0; key_idx < key_len; ++key_idx) {
 467                 hex_buf[0] = *hex++;
 468                 hex_buf[1] = *hex++;
 469                 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
 470         }
 471         hex_buf[0] = 0;
 472         hex_buf[1] = 0;
 473
 474         return 0;
 475 }
 476
 477 static int
 478 dm_target_crypt_init(dm_dev_t * dmv, void **target_config, char *params)
 479 {
 480         dm_target_crypt_config_t *priv;
 481         size_t len;
 482         char **ap, *args[5];
 483         char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
 484         char *status_str;
 485         int i, argc, klen, error;
 486         uint64_t iv_offset, block_offset;
 487
 488         if (params == NULL)
 489                 return EINVAL;
 490
 491         len = strlen(params) + 1;
 492         argc = 0;
 493
 494         status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
 495         /*
 496          * Parse a string, containing tokens delimited by white space,
 497          * into an argument vector
 498          */
 499         for (ap = args; ap < &args[5] &&
 500             (*ap = strsep(&params, " \t")) != NULL;) {
 501                 if (**ap != '\0') {
 502                         argc++;
 503                         ap++;
 504                 }
 505         }
 506
 507         if (argc != 5) {
 508                 kprintf("dm_target_crypt: not enough arguments, "
 509                         "need exactly 5\n");
 510                 kfree(status_str, M_DMCRYPT);
 511                 return ENOMEM; /* XXX */
 512         }
 513
 514         crypto_alg = strsep(&args[0], "-");
 515         crypto_mode = strsep(&args[0], "-");
 516         iv_opt = strsep(&args[0], "-");
 517         iv_mode = strsep(&iv_opt, ":");
 518         key = args[1];
 519         iv_offset = strtouq(args[2], NULL, 0);
 520         dev = args[3];
 521         block_offset = strtouq(args[4], NULL, 0);
 522         /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
 523         klen = strlen(key) << 2;
 524
 525 #if 0
 526         kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
 527                 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
 528                 "block_offset=%ju\n",
 529                 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
 530                 block_offset);
 531 #endif
 532
 533         priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
 534         if (priv == NULL) {
 535                 kprintf("dm_target_crypt: could not allocate memory\n");
 536                 kfree(status_str, M_DMCRYPT);
 537                 return ENOMEM;
 538         }
 539
 540         /* Insert dmp to global pdev list */
 541         if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
 542                 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
 543                 kfree(status_str, M_DMCRYPT);
 544                 return ENOENT;
 545         }
 546
 547         /*
 548          * This code checks for valid combinations of algorithm and mode.
 549          * Currently supported options are:
 550          *
 551          * *-cbc
 552          * aes-xts
 553          * twofish-xts
 554          * serpent-xts
 555          */
 556         if ((strcmp(crypto_mode, "cbc") != 0) &&
 557             !((strcmp(crypto_mode, "xts") == 0) &&
 558             ((strcmp(crypto_alg, "aes") == 0) ||
 559             (strcmp(crypto_alg, "twofish") == 0) ||
 560             (strcmp(crypto_alg, "serpent") == 0))))
 561         {
 562                 kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
 563                     " aes-xts, twofish-xts and serpent-xts, "
 564                     "invalid mode '%s-%s'\n",
 565                     crypto_alg, crypto_mode);
 566                 goto notsup;
 567         }
 568
 569         if (!strcmp(crypto_alg, "aes")) {
 570                 if (!strcmp(crypto_mode, "xts")) {
 571                         priv->crypto_alg = CRYPTO_AES_XTS;
 572                         if (klen != 256 && klen != 512)
 573                                 goto notsup;
 574                 } else if (!strcmp(crypto_mode, "cbc")) {
 575                         priv->crypto_alg = CRYPTO_AES_CBC;
 576                         if (klen != 128 && klen != 192 && klen != 256)
 577                                 goto notsup;
 578                 } else {
 579                         goto notsup;
 580                 }
 581                 priv->crypto_klen = klen;
 582         } else if (!strcmp(crypto_alg, "twofish")) {
 583                 if (!strcmp(crypto_mode, "xts")) {
 584                         priv->crypto_alg = CRYPTO_TWOFISH_XTS;
 585                         if (klen != 256 && klen != 512)
 586                                 goto notsup;
 587                 } else if (!strcmp(crypto_mode, "cbc")) {
 588                         priv->crypto_alg = CRYPTO_TWOFISH_CBC;
 589                         if (klen != 128 && klen != 192 && klen != 256)
 590                                 goto notsup;
 591                 } else {
 592                         goto notsup;
 593                 }
 594                 priv->crypto_klen = klen;
 595         } else if (!strcmp(crypto_alg, "serpent")) {
 596                 if (!strcmp(crypto_mode, "xts")) {
 597                         priv->crypto_alg = CRYPTO_SERPENT_XTS;
 598                         if (klen != 256 && klen != 512)
 599                                 goto notsup;
 600                 } else if (!strcmp(crypto_mode, "cbc")) {
 601                         priv->crypto_alg = CRYPTO_SERPENT_CBC;
 602                         if (klen != 128 && klen != 192 && klen != 256)
 603                                 goto notsup;
 604                 } else {
 605                         goto notsup;
 606                 }
 607                 priv->crypto_klen = klen;
 608         } else if (!strcmp(crypto_alg, "blowfish")) {
 609                 priv->crypto_alg = CRYPTO_BLF_CBC;
 610                 if (klen < 128 || klen > 448 || (klen % 8) != 0)
 611                         goto notsup;
 612                 priv->crypto_klen = klen;
 613         } else if (!strcmp(crypto_alg, "3des") ||
 614                    !strncmp(crypto_alg, "des3", 4)) {
 615                 priv->crypto_alg = CRYPTO_3DES_CBC;
 616                 if (klen != 168)
 617                         goto notsup;
 618                 priv->crypto_klen = 168;
 619         } else if (!strcmp(crypto_alg, "camellia")) {
 620                 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
 621                 if (klen != 128 && klen != 192 && klen != 256)
 622                         goto notsup;
 623                 priv->crypto_klen = klen;
 624         } else if (!strcmp(crypto_alg, "skipjack")) {
 625                 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
 626                 if (klen != 80)
 627                         goto notsup;
 628                 priv->crypto_klen = 80;
 629         } else if (!strcmp(crypto_alg, "cast5")) {
 630                 priv->crypto_alg = CRYPTO_CAST_CBC;
 631                 if (klen != 128)
 632                         goto notsup;
 633                 priv->crypto_klen = 128;
 634         } else if (!strcmp(crypto_alg, "null")) {
 635                 priv->crypto_alg = CRYPTO_NULL_CBC;
 636                 if (klen != 128)
 637                         goto notsup;
 638                 priv->crypto_klen = 128;
 639         } else {
 640                 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
 641                         crypto_alg);
 642                 goto notsup;
 643         }
 644
 645         /* Save length of param string */
 646         priv->params_len = len;
 647         priv->block_offset = block_offset;
 648         priv->iv_offset = iv_offset - block_offset;
 649
 650         *target_config = priv;
 651
 652         dmv->dev_type = DM_CRYPTO_DEV;
 653
 654         error = hex2key(key, priv->crypto_klen >> 3,
 655                         (u_int8_t *)priv->crypto_key);
 656
 657         if (error) {
 658                 kprintf("dm_target_crypt: hex2key failed, "
 659                         "invalid key format\n");
 660                 goto notsup;
 661         }
 662
 663         /* Handle cmd */
 664         for(i = 0; ivgens[i].name != NULL; i++) {
 665                 if (!strcmp(iv_mode, ivgens[i].name))
 666                         break;
 667         }
 668
 669         if (ivgens[i].name == NULL) {
 670                 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
 671                         iv_mode);
 672                 goto notsup;
 673         }
 674
 675         /* Call our ivgen constructor */
 676         if (ivgens[i].ctor != NULL) {
 677                 error = ivgens[i].ctor(priv, iv_opt,
 678                     &priv->ivgen_priv);
 679                 if (error) {
 680                         kprintf("dm_target_crypt: ctor for '%s' failed\n",
 681                             ivgens[i].name);
 682                         goto notsup;
 683                 }
 684         }
 685
 686         priv->ivgen = &ivgens[i];
 687
 688         priv->crypto_session.cri_alg = priv->crypto_alg;
 689         priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
 690         priv->crypto_session.cri_klen = priv->crypto_klen;
 691         priv->crypto_session.cri_mlen = 0;
 692         priv->crypto_session.cri_next = NULL;
 693
 694         error = crypto_newsession(&priv->crypto_sid,
 695                                   &priv->crypto_session,
 696                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 697         if (error) {
 698                 kprintf("dm_target_crypt: Error during crypto_newsession, "
 699                         "error = %d\n",
 700                         error);
 701                 goto notsup;
 702         }
 703
 704         memset(key, '0', strlen(key));
 705         if (iv_opt) {
 706                 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
 707                     crypto_alg, crypto_mode, iv_mode, iv_opt,
 708                     key, iv_offset, dev, block_offset);
 709         } else {
 710                 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
 711                     crypto_alg, crypto_mode, iv_mode,
 712                     key, iv_offset, dev, block_offset);
 713         }
 714         priv->status_str = status_str;
 715
 716         return 0;
 717
 718 notsup:
 719         kprintf("dm_target_crypt: ENOTSUP\n");
 720         kfree(status_str, M_DMCRYPT);
 721         return ENOTSUP;
 722 }
 723
 724 /* Status routine called to get params string. */
 725 static char *
 726 dm_target_crypt_status(void *target_config)
 727 {
 728         dm_target_crypt_config_t *priv;
 729         char *params;
 730
 731         priv = target_config;
 732
 733         /* caller expects use of M_DM */
 734         params = kmalloc(DM_MAX_PARAMS_SIZE, M_DM, M_WAITOK);
 735
 736         ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
 737             priv->status_str);
 738
 739         return params;
 740 }
 741
 742 static int
 743 dm_target_crypt_destroy(dm_table_entry_t * table_en)
 744 {
 745         dm_target_crypt_config_t *priv;
 746
 747         /*
 748          * Disconnect the crypt config before unbusying the target.
 749          */
 750         priv = table_en->target_config;
 751         if (priv == NULL)
 752                 return 0;
 753         table_en->target_config = NULL;
 754         dm_pdev_decr(priv->pdev);
 755
 756         dm_target_unbusy(table_en->target);
 757
 758         /*
 759          * Clean up the crypt config
 760          *
 761          * Overwrite the private information before freeing memory to
 762          * avoid leaking it.
 763          */
 764         if (priv->status_str) {
 765                 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
 766                 kfree(priv->status_str, M_DMCRYPT);
 767                 crypto_freesession(priv->crypto_sid);
 768         }
 769
 770         if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
 771                 priv->ivgen->dtor(priv, priv->ivgen_priv);
 772         }
 773
 774         dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
 775         kfree(priv, M_DMCRYPT);
 776
 777         return 0;
 778 }
 779
 780 static int
 781 dm_target_crypt_deps(dm_table_entry_t * table_en, prop_array_t prop_array)
 782 {
 783         dm_target_crypt_config_t *priv;
 784         struct vattr va;
 785
 786         int error;
 787
 788         if (table_en->target_config == NULL)
 789                 return ENOENT;
 790
 791         priv = table_en->target_config;
 792
 793         if ((error = VOP_GETATTR(priv->pdev->pdev_vnode, &va)) != 0)
 794                 return error;
 795
 796         prop_array_add_uint64(prop_array,
 797                               (uint64_t)makeudev(va.va_rmajor, va.va_rminor));
 798
 799         return 0;
 800 }
 801
 802 /* Unsupported for this target. */
 803 static int
 804 dm_target_crypt_upcall(dm_table_entry_t * table_en, struct buf * bp)
 805 {
 806         return 0;
 807 }
 808
 809 /************************************************************************
 810  *                      STRATEGY SUPPORT FUNCTIONS                      *
 811  ************************************************************************
 812  *
 813  * READ PATH:   doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
 814  * WRITE PATH:  crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
 815  */
 816
 817 /*
 818  * Wrapper around crypto_dispatch() to match dispatch_t type
 819  */
 820 static void
 821 dmtc_crypto_dispatch(void *arg)
 822 {
 823         struct cryptop *crp;
 824
 825         crp = (struct cryptop *)arg;
 826         KKASSERT(crp != NULL);
 827         crypto_dispatch(crp);
 828 }
 829
 830 /*
 831  * Start IO operation, called from dmstrategy routine.
 832  */
 833 static int
 834 dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
 835 {
 836         struct bio *bio;
 837
 838         dm_target_crypt_config_t *priv;
 839         priv = table_en->target_config;
 840
 841         /* Get rid of stuff we can't really handle */
 842         if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
 843                 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
 844                         kprintf("dm_target_crypt_strategy: can't really "
 845                                 "handle bp->b_bcount = %d\n",
 846                                 bp->b_bcount);
 847                         bp->b_error = EINVAL;
 848                         bp->b_flags |= B_ERROR | B_INVAL;
 849                         biodone(&bp->b_bio1);
 850                         return 0;
 851                 }
 852         }
 853
 854         switch (bp->b_cmd) {
 855         case BUF_CMD_READ:
 856                 bio = push_bio(&bp->b_bio1);
 857                 bio->bio_offset = bp->b_bio1.bio_offset +
 858                                   priv->block_offset * DEV_BSIZE;
 859                 bio->bio_caller_info1.ptr = priv;
 860                 bio->bio_done = dmtc_bio_read_done;
 861                 vn_strategy(priv->pdev->pdev_vnode, bio);
 862                 break;
 863         case BUF_CMD_WRITE:
 864                 bio = push_bio(&bp->b_bio1);
 865                 bio->bio_offset = bp->b_bio1.bio_offset +
 866                                   priv->block_offset * DEV_BSIZE;
 867                 bio->bio_caller_info1.ptr = priv;
 868                 dmtc_crypto_write_start(priv, bio);
 869                 break;
 870         default:
 871                 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
 872                 break;
 873         }
 874         return 0;
 875 }
 876
 877 /*
 878  * STRATEGY READ PATH PART 1/3 (after read BIO completes)
 879  */
 880 static void
 881 dmtc_bio_read_done(struct bio *bio)
 882 {
 883         struct bio *obio;
 884
 885         dm_target_crypt_config_t *priv;
 886
 887         /*
 888          * If a read error occurs we shortcut the operation, otherwise
 889          * go on to stage 2.
 890          */
 891         if (bio->bio_buf->b_flags & B_ERROR) {
 892                 obio = pop_bio(bio);
 893                 biodone(obio);
 894         } else {
 895                 priv = bio->bio_caller_info1.ptr;
 896                 dmtc_crypto_read_start(priv, bio);
 897         }
 898 }
 899
 900 /*
 901  * STRATEGY READ PATH PART 2/3
 902  */
 903 static void
 904 dmtc_crypto_read_retry(void *arg1, void *arg2)
 905 {
 906         dm_target_crypt_config_t *priv = arg1;
 907         struct bio *bio = arg2;
 908
 909         dmtc_crypto_read_start(priv, bio);
 910 }
 911
 912 static void
 913 dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
 914 {
 915         struct dmtc_helper *dmtc;
 916         struct cryptodesc *crd;
 917         struct cryptop *crp;
 918         struct cryptoini *cri;
 919         int i, bytes, sectors, sz;
 920         off_t isector;
 921         u_char *ptr, *space;
 922
 923         cri = &priv->crypto_session;
 924
 925         /*
 926          * Note: b_resid no good after read I/O, it will be 0, use
 927          *       b_bcount.
 928          */
 929         bytes = bio->bio_buf->b_bcount;
 930         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
 931         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
 932         sz = sectors * (sizeof(*crp) + sizeof(*crd));
 933
 934         /*
 935          * For reads with bogus page we can't decrypt in place as stuff
 936          * can get ripped out from under us.
 937          *
 938          * XXX actually it looks like we can, and in any case the initial
 939          * read already completed and threw crypted data into the buffer
 940          * cache buffer.  Disable for now.
 941          */
 942         space = mpipe_alloc_callback(&dmtc_read_mpipe,
 943                                      dmtc_crypto_read_retry, priv, bio);
 944         if (space == NULL)
 945                 return;
 946
 947         dmtc = (struct dmtc_helper *)space;
 948         dmtc->free_addr = space;
 949         space += sizeof(struct dmtc_helper);
 950         dmtc->orig_buf = NULL;
 951         dmtc->data_buf = bio->bio_buf->b_data;
 952         bio->bio_caller_info2.ptr = dmtc;
 953         bio->bio_buf->b_error = 0;
 954
 955         /*
 956          * Load crypto descriptors (crp/crd loop)
 957          */
 958         bzero(space, sz);
 959         ptr = space;
 960         bio->bio_caller_info3.value = sectors;
 961         cpu_sfence();
 962 #if 0
 963         kprintf("Read, bytes = %d (b_bcount), "
 964                 "sectors = %d (bio = %p, b_cmd = %d)\n",
 965                 bytes, sectors, bio, bio->bio_buf->b_cmd);
 966 #endif
 967         for (i = 0; i < sectors; i++) {
 968                 crp = (struct cryptop *)ptr;
 969                 ptr += sizeof(*crp);
 970                 crd = (struct cryptodesc *)ptr;
 971                 ptr += sizeof (*crd);
 972
 973                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
 974
 975                 crp->crp_sid = priv->crypto_sid;
 976                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
 977
 978                 crp->crp_opaque = (void *)bio;
 979
 980                 crp->crp_callback = dmtc_crypto_cb_read_done;
 981                 crp->crp_desc = crd;
 982                 crp->crp_etype = 0;
 983                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
 984                                  CRYPTO_F_BATCH;
 985
 986                 crd->crd_alg = priv->crypto_alg;
 987 #if 0
 988                 crd->crd_key = (caddr_t)priv->crypto_key;
 989                 crd->crd_klen = priv->crypto_klen;
 990 #endif
 991
 992                 crd->crd_skip = 0;
 993                 crd->crd_len = DEV_BSIZE /* XXX */;
 994                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 995                 crd->crd_next = NULL;
 996
 997                 crd->crd_flags &= ~CRD_F_ENCRYPT;
 998
 999                 /*
1000                  * Note: last argument is used to generate salt(?) and is
1001                  *       a 64 bit value, but the original code passed an
1002                  *       int.  Changing it now will break pre-existing
1003                  *       crypt volumes.
1004                  */
1005                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1006                                     isector + i, crp);
1007         }
1008 }
1009
1010 /*
1011  * STRATEGY READ PATH PART 3/3
1012  */
1013 static int
1014 dmtc_crypto_cb_read_done(struct cryptop *crp)
1015 {
1016         struct dmtc_helper *dmtc;
1017         struct bio *bio, *obio;
1018         int n;
1019
1020         if (crp->crp_etype == EAGAIN)
1021                 return crypto_dispatch(crp);
1022
1023         bio = (struct bio *)crp->crp_opaque;
1024         KKASSERT(bio != NULL);
1025
1026         /*
1027          * Cumulative error
1028          */
1029         if (crp->crp_etype) {
1030                 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1031                         "crp_etype = %d\n",
1032                         crp->crp_etype);
1033                 bio->bio_buf->b_error = crp->crp_etype;
1034         }
1035
1036         /*
1037          * On the last chunk of the decryption we do any required copybacks
1038          * and complete the I/O.
1039          */
1040         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1041 #if 0
1042         kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1043 #endif
1044
1045         if (n == 1) {
1046                 /*
1047                  * For the B_HASBOGUS case we didn't decrypt in place,
1048                  * so we need to copy stuff back into the buf.
1049                  *
1050                  * (disabled for now).
1051                  */
1052                 dmtc = bio->bio_caller_info2.ptr;
1053                 if (bio->bio_buf->b_error) {
1054                         bio->bio_buf->b_flags |= B_ERROR;
1055                 }
1056 #if 0
1057                 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
1058                         memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1059                                bio->bio_buf->b_bcount);
1060                 }
1061 #endif
1062                 mpipe_free(&dmtc_read_mpipe, dmtc->free_addr);
1063                 obio = pop_bio(bio);
1064                 biodone(obio);
1065         }
1066         return 0;
1067 }
1068 /* END OF STRATEGY READ SECTION */
1069
1070 /*
1071  * STRATEGY WRITE PATH PART 1/3
1072  */
1073
1074 static void
1075 dmtc_crypto_write_retry(void *arg1, void *arg2)
1076 {
1077         dm_target_crypt_config_t *priv = arg1;
1078         struct bio *bio = arg2;
1079
1080         dmtc_crypto_write_start(priv, bio);
1081 }
1082
1083 static void
1084 dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1085 {
1086         struct dmtc_helper *dmtc;
1087         struct cryptodesc *crd;
1088         struct cryptop *crp;
1089         struct cryptoini *cri;
1090         int i, bytes, sectors, sz;
1091         off_t isector;
1092         u_char *ptr, *space;
1093
1094         cri = &priv->crypto_session;
1095
1096         /*
1097          * Use b_bcount for consistency
1098          */
1099         bytes = bio->bio_buf->b_bcount;
1100
1101         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
1102         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1103         sz = sectors * (sizeof(*crp) + sizeof(*crd));
1104
1105         /*
1106          * For writes and reads with bogus page don't decrypt in place.
1107          */
1108         space = mpipe_alloc_callback(&dmtc_write_mpipe,
1109                                      dmtc_crypto_write_retry, priv, bio);
1110         if (space == NULL)
1111                 return;
1112
1113         dmtc = (struct dmtc_helper *)space;
1114         dmtc->free_addr = space;
1115         space += sizeof(struct dmtc_helper);
1116         memcpy(space + sz, bio->bio_buf->b_data, bytes);
1117
1118         bio->bio_caller_info2.ptr = dmtc;
1119         bio->bio_buf->b_error = 0;
1120
1121         dmtc->orig_buf = bio->bio_buf->b_data;
1122         dmtc->data_buf = space + sz;
1123
1124         /*
1125          * Load crypto descriptors (crp/crd loop)
1126          */
1127         bzero(space, sz);
1128         ptr = space;
1129         bio->bio_caller_info3.value = sectors;
1130         cpu_sfence();
1131 #if 0
1132         kprintf("Write, bytes = %d (b_bcount), "
1133                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1134                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1135 #endif
1136         for (i = 0; i < sectors; i++) {
1137                 crp = (struct cryptop *)ptr;
1138                 ptr += sizeof(*crp);
1139                 crd = (struct cryptodesc *)ptr;
1140                 ptr += sizeof (*crd);
1141
1142                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1143
1144                 crp->crp_sid = priv->crypto_sid;
1145                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1146
1147                 crp->crp_opaque = (void *)bio;
1148
1149                 crp->crp_callback = dmtc_crypto_cb_write_done;
1150                 crp->crp_desc = crd;
1151                 crp->crp_etype = 0;
1152                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1153                                  CRYPTO_F_BATCH;
1154
1155                 crd->crd_alg = priv->crypto_alg;
1156 #if 0
1157                 crd->crd_key = (caddr_t)priv->crypto_key;
1158                 crd->crd_klen = priv->crypto_klen;
1159 #endif
1160
1161                 crd->crd_skip = 0;
1162                 crd->crd_len = DEV_BSIZE /* XXX */;
1163                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1164                 crd->crd_next = NULL;
1165
1166                 crd->crd_flags |= CRD_F_ENCRYPT;
1167
1168                 /*
1169                  * Note: last argument is used to generate salt(?) and is
1170                  *       a 64 bit value, but the original code passed an
1171                  *       int.  Changing it now will break pre-existing
1172                  *       crypt volumes.
1173                  */
1174                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1175                                     isector + i, crp);
1176         }
1177 }
1178
1179 /*
1180  * STRATEGY WRITE PATH PART 2/3
1181  */
1182 static int
1183 dmtc_crypto_cb_write_done(struct cryptop *crp)
1184 {
1185         struct dmtc_helper *dmtc;
1186         dm_target_crypt_config_t *priv;
1187         struct bio *bio, *obio;
1188         int n;
1189
1190         if (crp->crp_etype == EAGAIN)
1191                 return crypto_dispatch(crp);
1192
1193         bio = (struct bio *)crp->crp_opaque;
1194         KKASSERT(bio != NULL);
1195
1196         /*
1197          * Cumulative error
1198          */
1199         if (crp->crp_etype != 0) {
1200                 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1201                         "crp_etype = %d\n",
1202                 crp->crp_etype);
1203                 bio->bio_buf->b_error = crp->crp_etype;
1204         }
1205
1206         /*
1207          * On the last chunk of the encryption we issue the write
1208          */
1209         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1210 #if 0
1211         kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1212 #endif
1213
1214         if (n == 1) {
1215                 dmtc = bio->bio_caller_info2.ptr;
1216                 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1217
1218                 if (bio->bio_buf->b_error) {
1219                         bio->bio_buf->b_flags |= B_ERROR;
1220                         mpipe_free(&dmtc_write_mpipe, dmtc->free_addr);
1221                         obio = pop_bio(bio);
1222                         biodone(obio);
1223                 } else {
1224                         dmtc->orig_buf = bio->bio_buf->b_data;
1225                         bio->bio_buf->b_data = dmtc->data_buf;
1226                         bio->bio_done = dmtc_bio_write_done;
1227                         vn_strategy(priv->pdev->pdev_vnode, bio);
1228                 }
1229         }
1230         return 0;
1231 }
1232
1233 /*
1234  * STRATEGY WRITE PATH PART 3/3
1235  */
1236 static void
1237 dmtc_bio_write_done(struct bio *bio)
1238 {
1239         struct dmtc_helper *dmtc;
1240         struct bio *obio;
1241
1242         dmtc = bio->bio_caller_info2.ptr;
1243         bio->bio_buf->b_data = dmtc->orig_buf;
1244         mpipe_free(&dmtc_write_mpipe, dmtc->free_addr);
1245         obio = pop_bio(bio);
1246         biodone(obio);
1247 }
1248 /* END OF STRATEGY WRITE SECTION */
1249
1250
1251
1252 /* DUMPING MAGIC */
1253
1254 extern int tsleep_crypto_dump;
1255
1256 static int
1257 dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1258 {
1259         static struct dmtc_dump_helper dump_helper;
1260         dm_target_crypt_config_t *priv;
1261         int id;
1262         static int first_call = 1;
1263
1264         priv = table_en->target_config;
1265
1266         if (first_call) {
1267                 first_call = 0;
1268                 dump_reactivate_cpus();
1269         }
1270
1271         /* Magically enable tsleep */
1272         tsleep_crypto_dump = 1;
1273         id = 0;
1274
1275         /*
1276          * 0 length means flush buffers and return
1277          */
1278         if (length == 0) {
1279                 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1280                         tsleep_crypto_dump = 0;
1281                         return ENXIO;
1282                 }
1283                 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1284                     data, 0, offset, 0);
1285                 tsleep_crypto_dump = 0;
1286                 return 0;
1287         }
1288
1289         bzero(&dump_helper, sizeof(dump_helper));
1290         dump_helper.priv = priv;
1291         dump_helper.data = data;
1292         dump_helper.length = length;
1293         dump_helper.offset = offset +
1294             priv->block_offset * DEV_BSIZE;
1295         dump_helper.ident = &id;
1296         dmtc_crypto_dump_start(priv, &dump_helper);
1297
1298         /*
1299          * Hackery to make stuff appear synchronous. The crypto callback will
1300          * set id to 1 and call wakeup on it. If the request completed
1301          * synchronously, id will be 1 and we won't bother to sleep. If not,
1302          * the crypto request will complete asynchronously and we sleep until
1303          * it's done.
1304          */
1305         if (id == 0)
1306                 tsleep(&dump_helper, 0, "cryptdump", 0);
1307
1308         dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1309             dump_helper.offset);
1310
1311         dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1312             dump_helper.space, 0, dump_helper.offset,
1313             dump_helper.length);
1314
1315         tsleep_crypto_dump = 0;
1316         return 0;
1317 }
1318
1319 static void
1320 dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1321 {
1322         struct cryptodesc *crd;
1323         struct cryptop *crp;
1324         struct cryptoini *cri;
1325         int i, bytes, sectors;
1326         off_t isector;
1327
1328         cri = &priv->crypto_session;
1329
1330         bytes = dump_helper->length;
1331
1332         isector = dump_helper->offset / DEV_BSIZE;      /* ivgen salt base? */
1333         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1334         dump_helper->sectors = sectors;
1335 #if 0
1336         kprintf("Dump, bytes = %d, "
1337                 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1338 #endif
1339         KKASSERT(dump_helper->length <= 65536);
1340
1341         memcpy(dump_helper->space, dump_helper->data, bytes);
1342
1343         cpu_sfence();
1344
1345         for (i = 0; i < sectors; i++) {
1346                 crp = &dump_helper->crp[i];
1347                 crd = &dump_helper->crd[i];
1348
1349                 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1350
1351                 crp->crp_sid = priv->crypto_sid;
1352                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1353
1354                 crp->crp_opaque = (void *)dump_helper;
1355
1356                 crp->crp_callback = dmtc_crypto_cb_dump_done;
1357                 crp->crp_desc = crd;
1358                 crp->crp_etype = 0;
1359                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1360                                  CRYPTO_F_BATCH;
1361
1362                 crd->crd_alg = priv->crypto_alg;
1363
1364                 crd->crd_skip = 0;
1365                 crd->crd_len = DEV_BSIZE /* XXX */;
1366                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1367                 crd->crd_next = NULL;
1368
1369                 crd->crd_flags |= CRD_F_ENCRYPT;
1370
1371                 /*
1372                  * Note: last argument is used to generate salt(?) and is
1373                  *       a 64 bit value, but the original code passed an
1374                  *       int.  Changing it now will break pre-existing
1375                  *       crypt volumes.
1376                  */
1377                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1378                                     isector + i, crp);
1379         }
1380 }
1381
1382 static int
1383 dmtc_crypto_cb_dump_done(struct cryptop *crp)
1384 {
1385         struct dmtc_dump_helper *dump_helper;
1386         dm_target_crypt_config_t *priv;
1387         int n;
1388
1389         if (crp->crp_etype == EAGAIN)
1390                 return crypto_dispatch(crp);
1391
1392         dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1393         KKASSERT(dump_helper != NULL);
1394
1395         if (crp->crp_etype != 0) {
1396                 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1397                         "crp_etype = %d\n",
1398                 crp->crp_etype);
1399                 return crp->crp_etype;
1400         }
1401
1402         /*
1403          * On the last chunk of the encryption we return control
1404          */
1405         n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1406
1407         if (n == 1) {
1408                 priv = (dm_target_crypt_config_t *)dump_helper->priv;
1409                 atomic_add_int(dump_helper->ident, 1);
1410                 wakeup(dump_helper);
1411         }
1412
1413         return 0;
1414 }
1415
1416 static int
1417 dmtc_mod_handler(module_t mod, int type, void *unused)
1418 {
1419         dm_target_t *dmt = NULL;
1420         int err = 0;
1421
1422         switch (type) {
1423         case MOD_LOAD:
1424                 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1425                         dm_target_unbusy(dmt);
1426                         return EEXIST;
1427                 }
1428                 dmt = dm_target_alloc("crypt");
1429                 dmt->version[0] = 1;
1430                 dmt->version[1] = 6;
1431                 dmt->version[2] = 0;
1432                 strlcpy(dmt->name, "crypt", DM_MAX_TYPE_NAME);
1433                 dmt->init = &dm_target_crypt_init;
1434                 dmt->status = &dm_target_crypt_status;
1435                 dmt->strategy = &dm_target_crypt_strategy;
1436                 dmt->deps = &dm_target_crypt_deps;
1437                 dmt->destroy = &dm_target_crypt_destroy;
1438                 dmt->upcall = &dm_target_crypt_upcall;
1439                 dmt->dump = &dm_target_crypt_dump;
1440
1441                 dmtc_init_mpipe();
1442
1443                 err = dm_target_insert(dmt);
1444                 if (err)
1445                         dmtc_destroy_mpipe();
1446                 else
1447                         kprintf("dm_target_crypt: Successfully initialized\n");
1448                 break;
1449
1450         case MOD_UNLOAD:
1451                 err = dm_target_rem("crypt");
1452                 if (err == 0) {
1453                         dmtc_destroy_mpipe();
1454                         kprintf("dm_target_crypt: unloaded\n");
1455                 }
1456                 break;
1457
1458         default:
1459                 break;
1460         }
1461
1462         return err;
1463 }
1464
1465 DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);