sys/dev/disk/dm/targets/crypt/dm_target_crypt.c

   1 /*
   2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Alex Hornung <ahornung@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * This file implements initial version of device-mapper crypt target.
  37  */
  38 #include <sys/endian.h>
  39
  40 #include <sys/bio.h>
  41 #include <sys/globaldata.h>
  42 #include <sys/kerneldump.h>
  43 #include <sys/malloc.h>
  44 #include <sys/mpipe.h>
  45 #include <sys/md5.h>
  46 #include <sys/mutex2.h>
  47 #include <sys/vnode.h>
  48 #include <crypto/sha1.h>
  49 #include <crypto/sha2/sha2.h>
  50 #include <opencrypto/cryptodev.h>
  51 #include <opencrypto/rmd160.h>
  52 #include <machine/cpufunc.h>
  53 #include <cpu/atomic.h>
  54
  55 #include <sys/ktr.h>
  56
  57 #include <dev/disk/dm/dm.h>
  58 MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
  59
  60 KTR_INFO_MASTER(dmcrypt);
  61
  62 #if !defined(KTR_DMCRYPT)
  63 #define KTR_DMCRYPT     KTR_ALL
  64 #endif
  65
  66 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_dispatch, 0,
  67     "crypto_dispatch(%p)", struct cryptop *crp);
  68 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypt_strategy, 0,
  69     "crypt_strategy(b_cmd = %d, bp = %p)", int cmd, struct buf *bp);
  70 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_start, 1,
  71     "crypto_write_start(crp = %p, bp = %p, sector = %d/%d)",
  72     struct cryptop *crp, struct buf *bp, int i, int sectors);
  73 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_write_done, 1,
  74     "crypto_cb_write_done(crp = %p, bp = %p, n = %d)",
  75     struct cryptop *crp, struct buf *bp, int n);
  76 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_write_done, 1,
  77     "bio_write_done(bp = %p)", struct buf *bp);
  78 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_retry, 1,
  79     "crypto_write_retry(crp = %p)", struct buf *bp);
  80 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_read_done, 2,
  81     "bio_read_done(bp = %p)", struct buf *bp);
  82 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_read_start, 2,
  83     "crypto_read_start(crp = %p, bp = %p, sector = %d/%d)",
  84     struct cryptop *crp, struct buf *bp, int i, int sectors);
  85 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_read_done, 2,
  86     "crypto_cb_read_done(crp = %p, bp = %p, n = %d)",
  87     struct cryptop *crp, struct buf *bp, int n);
  88
  89 struct target_crypt_config;
  90
  91 typedef void dispatch_t(void *);
  92 typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
  93     void *);
  94
  95 typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
  96 typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
  97
  98 struct iv_generator {
  99         const char      *name;
 100         ivgen_ctor_t    *ctor;
 101         ivgen_dtor_t    *dtor;
 102         ivgen_t         *gen_iv;
 103 };
 104
 105 struct essiv_ivgen_priv {
 106         struct cryptoini        crypto_session;
 107         struct objcache *crp_crd_cache;
 108         u_int64_t       crypto_sid;
 109         size_t          keyhash_len;
 110         u_int8_t        crypto_keyhash[SHA512_DIGEST_LENGTH];
 111 };
 112
 113 typedef struct target_crypt_config {
 114         size_t  params_len;
 115         dm_pdev_t *pdev;
 116         char    *status_str;
 117         int     crypto_alg;
 118         int     crypto_klen;
 119         u_int8_t        crypto_key[512>>3];
 120
 121         u_int64_t       crypto_sid;
 122         u_int64_t       block_offset;
 123         int64_t         iv_offset;
 124         SHA512_CTX      essivsha512_ctx;
 125
 126         struct cryptoini        crypto_session;
 127
 128         struct iv_generator     *ivgen;
 129         void    *ivgen_priv;
 130
 131         struct malloc_pipe      read_mpipe;
 132         struct malloc_pipe      write_mpipe;
 133 } dm_target_crypt_config_t;
 134
 135 struct dmtc_helper {
 136         dm_target_crypt_config_t *priv;
 137         caddr_t free_addr;
 138         caddr_t orig_buf;
 139         caddr_t data_buf;
 140 };
 141
 142 struct dmtc_dump_helper {
 143         dm_target_crypt_config_t *priv;
 144         void *data;
 145         size_t length;
 146         off_t offset;
 147
 148         int sectors;
 149         int *ident;
 150
 151         struct cryptodesc crd[128];
 152         struct cryptop crp[128];
 153         u_char space[65536];
 154 };
 155
 156 #define DMTC_BUF_SIZE_WRITE \
 157     MAXPHYS + sizeof(struct dmtc_helper) + \
 158     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 159 #define DMTC_BUF_SIZE_READ \
 160     sizeof(struct dmtc_helper) + \
 161     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 162
 163 static void dmtc_crypto_dispatch(void *arg);
 164 static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
 165                                 struct dmtc_dump_helper *dump_helper);
 166 static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
 167                                 struct bio *bio);
 168 static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
 169                                 struct bio *bio);
 170 static void dmtc_bio_read_done(struct bio *bio);
 171 static void dmtc_bio_write_done(struct bio *bio);
 172 static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
 173 static int dmtc_crypto_cb_read_done(struct cryptop *crp);
 174 static int dmtc_crypto_cb_write_done(struct cryptop *crp);
 175
 176 static ivgen_ctor_t     essiv_ivgen_ctor;
 177 static ivgen_dtor_t     essiv_ivgen_dtor;
 178 static ivgen_t          essiv_ivgen;
 179 static ivgen_t          plain_ivgen;
 180 static ivgen_t          plain64_ivgen;
 181
 182 static struct iv_generator ivgens[] = {
 183         { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
 184             .gen_iv = essiv_ivgen },
 185         { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
 186         { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
 187         { NULL, NULL, NULL, NULL }
 188 };
 189
 190 struct objcache_malloc_args essiv_ivgen_malloc_args = {
 191                 2*sizeof(void *) + (sizeof(struct cryptodesc) +
 192                 sizeof(struct cryptop)), M_DMCRYPT };
 193
 194 static void
 195 dmtc_init_mpipe(struct target_crypt_config *priv)
 196 {
 197         int nmax;
 198
 199         nmax = (physmem*2/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
 200
 201         if (nmax < 2)
 202                 nmax = 2;
 203
 204         kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
 205
 206         mpipe_init(&priv->write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
 207                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 208         mpipe_init(&priv->read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
 209                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 210 }
 211
 212 static void
 213 dmtc_destroy_mpipe(struct target_crypt_config *priv)
 214 {
 215         mpipe_done(&priv->write_mpipe);
 216         mpipe_done(&priv->read_mpipe);
 217 }
 218
 219 /*
 220  * Overwrite private information (in buf) to avoid leaking it
 221  */
 222 static void
 223 dmtc_crypto_clear(void *buf, size_t len)
 224 {
 225         memset(buf, 0xFF, len);
 226         bzero(buf, len);
 227 }
 228
 229 /*
 230  * ESSIV IV Generator Routines
 231  */
 232 static int
 233 essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
 234 {
 235         struct essiv_ivgen_priv *ivpriv;
 236         u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
 237         unsigned int klen, hashlen;
 238         int error;
 239
 240         klen = (priv->crypto_klen >> 3);
 241
 242         if (iv_hash == NULL)
 243                 return EINVAL;
 244
 245         if (!strcmp(iv_hash, "sha1")) {
 246                 SHA1_CTX ctx;
 247
 248                 hashlen = SHA1_RESULTLEN;
 249                 SHA1Init(&ctx);
 250                 SHA1Update(&ctx, priv->crypto_key, klen);
 251                 SHA1Final(crypto_keyhash, &ctx);
 252         } else if (!strcmp(iv_hash, "sha256")) {
 253                 SHA256_CTX ctx;
 254
 255                 hashlen = SHA256_DIGEST_LENGTH;
 256                 SHA256_Init(&ctx);
 257                 SHA256_Update(&ctx, priv->crypto_key, klen);
 258                 SHA256_Final(crypto_keyhash, &ctx);
 259         } else if (!strcmp(iv_hash, "sha384")) {
 260                 SHA384_CTX ctx;
 261
 262                 hashlen = SHA384_DIGEST_LENGTH;
 263                 SHA384_Init(&ctx);
 264                 SHA384_Update(&ctx, priv->crypto_key, klen);
 265                 SHA384_Final(crypto_keyhash, &ctx);
 266         } else if (!strcmp(iv_hash, "sha512")) {
 267                 SHA512_CTX ctx;
 268
 269                 hashlen = SHA512_DIGEST_LENGTH;
 270                 SHA512_Init(&ctx);
 271                 SHA512_Update(&ctx, priv->crypto_key, klen);
 272                 SHA512_Final(crypto_keyhash, &ctx);
 273         } else if (!strcmp(iv_hash, "md5")) {
 274                 MD5_CTX ctx;
 275
 276                 hashlen = MD5_DIGEST_LENGTH;
 277                 MD5Init(&ctx);
 278                 MD5Update(&ctx, priv->crypto_key, klen);
 279                 MD5Final(crypto_keyhash, &ctx);
 280         } else if (!strcmp(iv_hash, "rmd160") ||
 281                    !strcmp(iv_hash, "ripemd160")) {
 282                 RMD160_CTX ctx;
 283
 284                 hashlen = 160/8;
 285                 RMD160Init(&ctx);
 286                 RMD160Update(&ctx, priv->crypto_key, klen);
 287                 RMD160Final(crypto_keyhash, &ctx);
 288         } else {
 289                 return EINVAL;
 290         }
 291
 292         /* Convert hashlen to bits */
 293         hashlen <<= 3;
 294
 295         ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
 296             M_WAITOK | M_ZERO);
 297         memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
 298         ivpriv->keyhash_len = sizeof(crypto_keyhash);
 299         dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
 300
 301         ivpriv->crypto_session.cri_alg = priv->crypto_alg;
 302         ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
 303         ivpriv->crypto_session.cri_klen = hashlen;
 304         ivpriv->crypto_session.cri_mlen = 0;
 305         ivpriv->crypto_session.cri_next = NULL;
 306
 307         /*
 308          * XXX: in principle we also need to check if the block size of the
 309          *      cipher is a valid iv size for the block cipher.
 310          */
 311
 312         error = crypto_newsession(&ivpriv->crypto_sid,
 313                                   &ivpriv->crypto_session,
 314                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 315         if (error) {
 316                 kprintf("dm_target_crypt: Error during crypto_newsession "
 317                         "for essiv_ivgen, error = %d\n",
 318                         error);
 319                 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 320                 kfree(ivpriv, M_DMCRYPT);
 321                 return ENOTSUP;
 322         }
 323
 324         ivpriv->crp_crd_cache = objcache_create(
 325             "dmcrypt-essiv-cache", 0, 0,
 326             NULL, NULL, NULL,
 327             objcache_malloc_alloc,
 328             objcache_malloc_free,
 329             &essiv_ivgen_malloc_args );
 330
 331         *p_ivpriv = ivpriv;
 332         return 0;
 333 }
 334
 335 static int
 336 essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
 337 {
 338         struct essiv_ivgen_priv *ivpriv;
 339
 340         ivpriv = (struct essiv_ivgen_priv *)arg;
 341         KKASSERT(ivpriv != NULL);
 342
 343         crypto_freesession(ivpriv->crypto_sid);
 344
 345         objcache_destroy(ivpriv->crp_crd_cache);
 346
 347         dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 348         kfree(ivpriv, M_DMCRYPT);
 349
 350         return 0;
 351 }
 352
 353 static int
 354 essiv_ivgen_done(struct cryptop *crp)
 355 {
 356         struct essiv_ivgen_priv *ivpriv;
 357         void *free_addr;
 358         void *opaque;
 359
 360
 361         if (crp->crp_etype == EAGAIN)
 362                 return crypto_dispatch(crp);
 363
 364         if (crp->crp_etype != 0) {
 365                 kprintf("dm_target_crypt: essiv_ivgen_done, "
 366                         "crp->crp_etype = %d\n", crp->crp_etype);
 367         }
 368
 369         free_addr = crp->crp_opaque;
 370         /*
 371          * In-memory structure is:
 372          * |  ivpriv  |  opaque  |     crp     |      crd      |
 373          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 374          */
 375         ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
 376         crp->crp_opaque += sizeof(void *);
 377         opaque = *((void **)crp->crp_opaque);
 378
 379         objcache_put(ivpriv->crp_crd_cache, free_addr);
 380         dmtc_crypto_dispatch(opaque);
 381         return 0;
 382 }
 383
 384 static void
 385 essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 386             size_t iv_len, off_t sector, void *opaque)
 387 {
 388         struct essiv_ivgen_priv *ivpriv;
 389         struct cryptodesc *crd;
 390         struct cryptop *crp;
 391         caddr_t space, alloc_addr;
 392         int error;
 393
 394         ivpriv = priv->ivgen_priv;
 395         KKASSERT(ivpriv != NULL);
 396
 397         /*
 398          * In-memory structure is:
 399          * |  ivpriv  |  opaque  |     crp     |      crd      |
 400          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 401          */
 402         alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
 403         *((struct essiv_ivgen_priv **)space) = ivpriv;
 404         space += sizeof(void *);
 405         *((void **)space) = opaque;
 406         space += sizeof(void *);
 407         crp = (struct cryptop *)space;
 408         space += sizeof(struct cryptop);
 409         crd = (struct cryptodesc *)space;
 410
 411         bzero(iv, iv_len);
 412         bzero(crd, sizeof(struct cryptodesc));
 413         bzero(crp, sizeof(struct cryptop));
 414         *((off_t *)iv) = htole64(sector + priv->iv_offset);
 415         crp->crp_buf = (caddr_t)iv;
 416
 417         crp->crp_sid = ivpriv->crypto_sid;
 418         crp->crp_ilen = crp->crp_olen = iv_len;
 419
 420         crp->crp_opaque = alloc_addr;
 421
 422         crp->crp_callback = essiv_ivgen_done;
 423
 424         crp->crp_desc = crd;
 425         crp->crp_etype = 0;
 426         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
 427
 428         crd->crd_alg = priv->crypto_alg;
 429 #if 0
 430         crd->crd_key = (caddr_t)priv->crypto_keyhash;
 431         crd->crd_klen = priv->crypto_klen;
 432 #endif
 433
 434         bzero(crd->crd_iv, sizeof(crd->crd_iv));
 435
 436         crd->crd_skip = 0;
 437         crd->crd_len = iv_len;
 438         crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 439         crd->crd_flags |= CRD_F_ENCRYPT;
 440         crd->crd_next = NULL;
 441
 442         error = crypto_dispatch(crp);
 443         if (error)
 444                 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
 445 }
 446
 447
 448 static void
 449 plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 450             size_t iv_len, off_t sector, void *opaque)
 451 {
 452         bzero(iv, iv_len);
 453         *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
 454         dmtc_crypto_dispatch(opaque);
 455 }
 456
 457 static void
 458 plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 459     size_t iv_len, off_t sector, void *opaque)
 460 {
 461         bzero(iv, iv_len);
 462         *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
 463         dmtc_crypto_dispatch(opaque);
 464 }
 465
 466 #if 0
 467 static void
 468 geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 469            size_t iv_len, off_t sector, void *opaque)
 470 {
 471
 472         SHA512_CTX      ctx512;
 473         u_int8_t        md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
 474
 475         memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
 476         SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
 477         SHA512_Final(md, &ctx512);
 478
 479         memcpy(iv, md, iv_len);
 480         dmtc_crypto_dispatch(opaque);
 481 }
 482 #endif
 483
 484 /*
 485  * Init function called from dm_table_load_ioctl.
 486  * cryptsetup actually passes us this:
 487  * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
 488  */
 489 static int
 490 hex2key(char *hex, size_t key_len, u_int8_t *key)
 491 {
 492         char hex_buf[3];
 493         size_t key_idx;
 494
 495         hex_buf[2] = 0;
 496         for (key_idx = 0; key_idx < key_len; ++key_idx) {
 497                 hex_buf[0] = *hex++;
 498                 hex_buf[1] = *hex++;
 499                 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
 500         }
 501         hex_buf[0] = 0;
 502         hex_buf[1] = 0;
 503
 504         return 0;
 505 }
 506
 507 static int
 508 dm_target_crypt_init(dm_table_entry_t *table_en, int argc, char **argv)
 509 {
 510         dm_target_crypt_config_t *priv;
 511         size_t len;
 512         char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
 513         char *status_str;
 514         int i, klen, error;
 515         uint64_t iv_offset, block_offset;
 516
 517         if (argc != 5) {
 518                 kprintf("dm_target_crypt: not enough arguments, "
 519                         "need exactly 5\n");
 520                 return EINVAL;
 521         }
 522
 523         len = 0;
 524         for (i = 0; i < argc; i++) {
 525                 len += strlen(argv[i]);
 526                 len++;
 527         }
 528         /* len is strlen() of input string +1 */
 529         status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
 530
 531         crypto_alg = strsep(&argv[0], "-");
 532         crypto_mode = strsep(&argv[0], "-");
 533         iv_opt = strsep(&argv[0], "-");
 534         iv_mode = strsep(&iv_opt, ":");
 535         key = argv[1];
 536         iv_offset = strtouq(argv[2], NULL, 0);
 537         dev = argv[3];
 538         block_offset = strtouq(argv[4], NULL, 0);
 539         /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
 540         klen = strlen(key) << 2;
 541
 542 #if 0
 543         kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
 544                 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
 545                 "block_offset=%ju\n",
 546                 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
 547                 block_offset);
 548 #endif
 549
 550         priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
 551
 552         /* Insert dmp to global pdev list */
 553         if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
 554                 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
 555                 kfree(status_str, M_DMCRYPT);
 556                 return ENOENT;
 557         }
 558
 559         /*
 560          * This code checks for valid combinations of algorithm and mode.
 561          * Currently supported options are:
 562          *
 563          * *-cbc
 564          * aes-xts
 565          * twofish-xts
 566          * serpent-xts
 567          */
 568         if ((strcmp(crypto_mode, "cbc") != 0) &&
 569             !((strcmp(crypto_mode, "xts") == 0) &&
 570             ((strcmp(crypto_alg, "aes") == 0) ||
 571             (strcmp(crypto_alg, "twofish") == 0) ||
 572             (strcmp(crypto_alg, "serpent") == 0))))
 573         {
 574                 kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
 575                     " aes-xts, twofish-xts and serpent-xts, "
 576                     "invalid mode '%s-%s'\n",
 577                     crypto_alg, crypto_mode);
 578                 goto notsup;
 579         }
 580
 581         if (!strcmp(crypto_alg, "aes")) {
 582                 if (!strcmp(crypto_mode, "xts")) {
 583                         priv->crypto_alg = CRYPTO_AES_XTS;
 584                         if (klen != 256 && klen != 512)
 585                                 goto notsup;
 586                 } else if (!strcmp(crypto_mode, "cbc")) {
 587                         priv->crypto_alg = CRYPTO_AES_CBC;
 588                         if (klen != 128 && klen != 192 && klen != 256)
 589                                 goto notsup;
 590                 } else {
 591                         goto notsup;
 592                 }
 593                 priv->crypto_klen = klen;
 594         } else if (!strcmp(crypto_alg, "twofish")) {
 595                 if (!strcmp(crypto_mode, "xts")) {
 596                         priv->crypto_alg = CRYPTO_TWOFISH_XTS;
 597                         if (klen != 256 && klen != 512)
 598                                 goto notsup;
 599                 } else if (!strcmp(crypto_mode, "cbc")) {
 600                         priv->crypto_alg = CRYPTO_TWOFISH_CBC;
 601                         if (klen != 128 && klen != 192 && klen != 256)
 602                                 goto notsup;
 603                 } else {
 604                         goto notsup;
 605                 }
 606                 priv->crypto_klen = klen;
 607         } else if (!strcmp(crypto_alg, "serpent")) {
 608                 if (!strcmp(crypto_mode, "xts")) {
 609                         priv->crypto_alg = CRYPTO_SERPENT_XTS;
 610                         if (klen != 256 && klen != 512)
 611                                 goto notsup;
 612                 } else if (!strcmp(crypto_mode, "cbc")) {
 613                         priv->crypto_alg = CRYPTO_SERPENT_CBC;
 614                         if (klen != 128 && klen != 192 && klen != 256)
 615                                 goto notsup;
 616                 } else {
 617                         goto notsup;
 618                 }
 619                 priv->crypto_klen = klen;
 620         } else if (!strcmp(crypto_alg, "blowfish")) {
 621                 priv->crypto_alg = CRYPTO_BLF_CBC;
 622                 if (klen < 128 || klen > 448 || (klen % 8) != 0)
 623                         goto notsup;
 624                 priv->crypto_klen = klen;
 625         } else if (!strcmp(crypto_alg, "3des") ||
 626                    !strncmp(crypto_alg, "des3", 4)) {
 627                 priv->crypto_alg = CRYPTO_3DES_CBC;
 628                 if (klen != 168)
 629                         goto notsup;
 630                 priv->crypto_klen = 168;
 631         } else if (!strcmp(crypto_alg, "camellia")) {
 632                 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
 633                 if (klen != 128 && klen != 192 && klen != 256)
 634                         goto notsup;
 635                 priv->crypto_klen = klen;
 636         } else if (!strcmp(crypto_alg, "skipjack")) {
 637                 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
 638                 if (klen != 80)
 639                         goto notsup;
 640                 priv->crypto_klen = 80;
 641         } else if (!strcmp(crypto_alg, "cast5")) {
 642                 priv->crypto_alg = CRYPTO_CAST_CBC;
 643                 if (klen != 128)
 644                         goto notsup;
 645                 priv->crypto_klen = 128;
 646         } else if (!strcmp(crypto_alg, "null")) {
 647                 priv->crypto_alg = CRYPTO_NULL_CBC;
 648                 if (klen != 128)
 649                         goto notsup;
 650                 priv->crypto_klen = 128;
 651         } else {
 652                 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
 653                         crypto_alg);
 654                 goto notsup;
 655         }
 656
 657         /* Save length of param string */
 658         priv->params_len = len;
 659         priv->block_offset = block_offset;
 660         priv->iv_offset = iv_offset - block_offset;
 661
 662         dm_table_add_deps(table_en, priv->pdev);
 663
 664         dm_table_init_target(table_en, DM_CRYPTO_DEV, priv);
 665
 666         error = hex2key(key, priv->crypto_klen >> 3,
 667                         (u_int8_t *)priv->crypto_key);
 668
 669         if (error) {
 670                 kprintf("dm_target_crypt: hex2key failed, "
 671                         "invalid key format\n");
 672                 goto notsup;
 673         }
 674
 675         /* Handle cmd */
 676         for(i = 0; ivgens[i].name != NULL; i++) {
 677                 if (!strcmp(iv_mode, ivgens[i].name))
 678                         break;
 679         }
 680
 681         if (ivgens[i].name == NULL) {
 682                 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
 683                         iv_mode);
 684                 goto notsup;
 685         }
 686
 687         /* Call our ivgen constructor */
 688         if (ivgens[i].ctor != NULL) {
 689                 error = ivgens[i].ctor(priv, iv_opt,
 690                     &priv->ivgen_priv);
 691                 if (error) {
 692                         kprintf("dm_target_crypt: ctor for '%s' failed\n",
 693                             ivgens[i].name);
 694                         goto notsup;
 695                 }
 696         }
 697
 698         priv->ivgen = &ivgens[i];
 699
 700         priv->crypto_session.cri_alg = priv->crypto_alg;
 701         priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
 702         priv->crypto_session.cri_klen = priv->crypto_klen;
 703         priv->crypto_session.cri_mlen = 0;
 704         priv->crypto_session.cri_next = NULL;
 705
 706         error = crypto_newsession(&priv->crypto_sid,
 707                                   &priv->crypto_session,
 708                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 709         if (error) {
 710                 kprintf("dm_target_crypt: Error during crypto_newsession, "
 711                         "error = %d\n",
 712                         error);
 713                 goto notsup;
 714         }
 715
 716         memset(key, '0', strlen(key));
 717         if (iv_opt) {
 718                 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
 719                     crypto_alg, crypto_mode, iv_mode, iv_opt,
 720                     key, iv_offset, dev, block_offset);
 721         } else {
 722                 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
 723                     crypto_alg, crypto_mode, iv_mode,
 724                     key, iv_offset, dev, block_offset);
 725         }
 726         priv->status_str = status_str;
 727
 728         /* Initialize mpipes */
 729         dmtc_init_mpipe(priv);
 730
 731         return 0;
 732
 733 notsup:
 734         kprintf("dm_target_crypt: ENOTSUP\n");
 735         kfree(status_str, M_DMCRYPT);
 736         return ENOTSUP;
 737 }
 738
 739 /* Table routine called to get params string. */
 740 static char *
 741 dm_target_crypt_table(void *target_config)
 742 {
 743         dm_target_crypt_config_t *priv;
 744         char *params;
 745
 746         priv = target_config;
 747
 748         params = dm_alloc_string(DM_MAX_PARAMS_SIZE);
 749
 750         ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
 751             priv->status_str);
 752
 753         return params;
 754 }
 755
 756 static int
 757 dm_target_crypt_destroy(dm_table_entry_t *table_en)
 758 {
 759         dm_target_crypt_config_t *priv;
 760
 761         /*
 762          * Disconnect the crypt config before unbusying the target.
 763          */
 764         priv = table_en->target_config;
 765         if (priv == NULL)
 766                 return 0;
 767         dm_pdev_decr(priv->pdev);
 768
 769         /*
 770          * Clean up the crypt config
 771          *
 772          * Overwrite the private information before freeing memory to
 773          * avoid leaking it.
 774          */
 775         if (priv->status_str) {
 776                 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
 777                 kfree(priv->status_str, M_DMCRYPT);
 778                 crypto_freesession(priv->crypto_sid);
 779         }
 780
 781         if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
 782                 priv->ivgen->dtor(priv, priv->ivgen_priv);
 783         }
 784
 785         /* Destroy mpipes */
 786         dmtc_destroy_mpipe(priv);
 787
 788         dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
 789         kfree(priv, M_DMCRYPT);
 790
 791         return 0;
 792 }
 793
 794 /************************************************************************
 795  *                      STRATEGY SUPPORT FUNCTIONS                      *
 796  ************************************************************************
 797  *
 798  * READ PATH:   doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
 799  * WRITE PATH:  crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
 800  */
 801
 802 /*
 803  * Wrapper around crypto_dispatch() to match dispatch_t type
 804  */
 805 static void
 806 dmtc_crypto_dispatch(void *arg)
 807 {
 808         struct cryptop *crp;
 809
 810         crp = (struct cryptop *)arg;
 811         KKASSERT(crp != NULL);
 812         KTR_LOG(dmcrypt_crypto_dispatch, crp);
 813         crypto_dispatch(crp);
 814 }
 815
 816 /*
 817  * Start IO operation, called from dmstrategy routine.
 818  */
 819 static int
 820 dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
 821 {
 822         struct bio *bio;
 823
 824         dm_target_crypt_config_t *priv;
 825         priv = table_en->target_config;
 826
 827         /* Get rid of stuff we can't really handle */
 828         if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
 829                 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
 830                         kprintf("dm_target_crypt_strategy: can't really "
 831                                 "handle bp->b_bcount = %d\n",
 832                                 bp->b_bcount);
 833                         bp->b_error = EINVAL;
 834                         bp->b_flags |= B_ERROR | B_INVAL;
 835                         biodone(&bp->b_bio1);
 836                         return 0;
 837                 }
 838         }
 839
 840         KTR_LOG(dmcrypt_crypt_strategy, bp->b_cmd, bp);
 841
 842         switch (bp->b_cmd) {
 843         case BUF_CMD_READ:
 844                 bio = push_bio(&bp->b_bio1);
 845                 bio->bio_offset = bp->b_bio1.bio_offset +
 846                                   priv->block_offset * DEV_BSIZE;
 847                 bio->bio_caller_info1.ptr = priv;
 848                 bio->bio_done = dmtc_bio_read_done;
 849                 vn_strategy(priv->pdev->pdev_vnode, bio);
 850                 break;
 851         case BUF_CMD_WRITE:
 852                 bio = push_bio(&bp->b_bio1);
 853                 bio->bio_offset = bp->b_bio1.bio_offset +
 854                                   priv->block_offset * DEV_BSIZE;
 855                 bio->bio_caller_info1.ptr = priv;
 856                 dmtc_crypto_write_start(priv, bio);
 857                 break;
 858         default:
 859                 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
 860                 break;
 861         }
 862         return 0;
 863 }
 864
 865 /*
 866  * STRATEGY READ PATH PART 1/3 (after read BIO completes)
 867  */
 868 static void
 869 dmtc_bio_read_done(struct bio *bio)
 870 {
 871         struct bio *obio;
 872
 873         dm_target_crypt_config_t *priv;
 874
 875         KTR_LOG(dmcrypt_bio_read_done, bio->bio_buf);
 876
 877         /*
 878          * If a read error occurs we shortcut the operation, otherwise
 879          * go on to stage 2.
 880          */
 881         if (bio->bio_buf->b_flags & B_ERROR) {
 882                 obio = pop_bio(bio);
 883                 biodone(obio);
 884         } else {
 885                 priv = bio->bio_caller_info1.ptr;
 886                 dmtc_crypto_read_start(priv, bio);
 887         }
 888 }
 889
 890 /*
 891  * STRATEGY READ PATH PART 2/3
 892  */
 893 static void
 894 dmtc_crypto_read_retry(void *arg1, void *arg2)
 895 {
 896         dm_target_crypt_config_t *priv = arg1;
 897         struct bio *bio = arg2;
 898
 899         dmtc_crypto_read_start(priv, bio);
 900 }
 901
 902 static void
 903 dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
 904 {
 905         struct dmtc_helper *dmtc;
 906         struct cryptodesc *crd;
 907         struct cryptop *crp;
 908         int i, bytes, sectors, sz;
 909         off_t isector;
 910         u_char *ptr, *space;
 911
 912         /*
 913          * Note: b_resid no good after read I/O, it will be 0, use
 914          *       b_bcount.
 915          */
 916         bytes = bio->bio_buf->b_bcount;
 917         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
 918         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
 919         sz = sectors * (sizeof(*crp) + sizeof(*crd));
 920
 921         /*
 922          * For reads with bogus page we can't decrypt in place as stuff
 923          * can get ripped out from under us.
 924          *
 925          * XXX actually it looks like we can, and in any case the initial
 926          * read already completed and threw crypted data into the buffer
 927          * cache buffer.  Disable for now.
 928          */
 929         space = mpipe_alloc_callback(&priv->read_mpipe,
 930                                      dmtc_crypto_read_retry, priv, bio);
 931         if (space == NULL)
 932                 return;
 933
 934         dmtc = (struct dmtc_helper *)space;
 935         dmtc->free_addr = space;
 936         space += sizeof(struct dmtc_helper);
 937         dmtc->orig_buf = NULL;
 938         dmtc->data_buf = bio->bio_buf->b_data;
 939         dmtc->priv = priv;
 940         bio->bio_caller_info2.ptr = dmtc;
 941         bio->bio_buf->b_error = 0;
 942
 943         /*
 944          * Load crypto descriptors (crp/crd loop)
 945          */
 946         bzero(space, sz);
 947         ptr = space;
 948         bio->bio_caller_info3.value = sectors;
 949         cpu_sfence();
 950 #if 0
 951         kprintf("Read, bytes = %d (b_bcount), "
 952                 "sectors = %d (bio = %p, b_cmd = %d)\n",
 953                 bytes, sectors, bio, bio->bio_buf->b_cmd);
 954 #endif
 955         for (i = 0; i < sectors; i++) {
 956                 crp = (struct cryptop *)ptr;
 957                 ptr += sizeof(*crp);
 958                 crd = (struct cryptodesc *)ptr;
 959                 ptr += sizeof (*crd);
 960
 961                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
 962
 963                 crp->crp_sid = priv->crypto_sid;
 964                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
 965
 966                 crp->crp_opaque = (void *)bio;
 967
 968                 crp->crp_callback = dmtc_crypto_cb_read_done;
 969                 crp->crp_desc = crd;
 970                 crp->crp_etype = 0;
 971                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
 972                                  CRYPTO_F_BATCH;
 973
 974                 crd->crd_alg = priv->crypto_alg;
 975 #if 0
 976                 crd->crd_key = (caddr_t)priv->crypto_key;
 977                 crd->crd_klen = priv->crypto_klen;
 978 #endif
 979
 980                 crd->crd_skip = 0;
 981                 crd->crd_len = DEV_BSIZE /* XXX */;
 982                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 983                 crd->crd_next = NULL;
 984
 985                 crd->crd_flags &= ~CRD_F_ENCRYPT;
 986
 987                 KTR_LOG(dmcrypt_crypto_read_start, crp, bio->bio_buf, i,
 988                     sectors);
 989
 990                 /*
 991                  * Note: last argument is used to generate salt(?) and is
 992                  *       a 64 bit value, but the original code passed an
 993                  *       int.  Changing it now will break pre-existing
 994                  *       crypt volumes.
 995                  */
 996                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
 997                                     isector + i, crp);
 998         }
 999 }
1000
1001 /*
1002  * STRATEGY READ PATH PART 3/3
1003  */
1004 static int
1005 dmtc_crypto_cb_read_done(struct cryptop *crp)
1006 {
1007         struct dmtc_helper *dmtc;
1008         struct bio *bio, *obio;
1009         int n;
1010
1011         if (crp->crp_etype == EAGAIN)
1012                 return crypto_dispatch(crp);
1013
1014         bio = (struct bio *)crp->crp_opaque;
1015         KKASSERT(bio != NULL);
1016
1017         /*
1018          * Cumulative error
1019          */
1020         if (crp->crp_etype) {
1021                 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1022                         "crp_etype = %d\n",
1023                         crp->crp_etype);
1024                 bio->bio_buf->b_error = crp->crp_etype;
1025         }
1026
1027         /*
1028          * On the last chunk of the decryption we do any required copybacks
1029          * and complete the I/O.
1030          */
1031         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1032 #if 0
1033         kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1034 #endif
1035
1036         KTR_LOG(dmcrypt_crypto_cb_read_done, crp, bio->bio_buf, n);
1037
1038         if (n == 1) {
1039                 /*
1040                  * For the B_HASBOGUS case we didn't decrypt in place,
1041                  * so we need to copy stuff back into the buf.
1042                  *
1043                  * (disabled for now).
1044                  */
1045                 dmtc = bio->bio_caller_info2.ptr;
1046                 if (bio->bio_buf->b_error) {
1047                         bio->bio_buf->b_flags |= B_ERROR;
1048                 }
1049 #if 0
1050                 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
1051                         memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1052                                bio->bio_buf->b_bcount);
1053                 }
1054 #endif
1055                 mpipe_free(&dmtc->priv->read_mpipe, dmtc->free_addr);
1056                 obio = pop_bio(bio);
1057                 biodone(obio);
1058         }
1059         return 0;
1060 }
1061 /* END OF STRATEGY READ SECTION */
1062
1063 /*
1064  * STRATEGY WRITE PATH PART 1/3
1065  */
1066
1067 static void
1068 dmtc_crypto_write_retry(void *arg1, void *arg2)
1069 {
1070         dm_target_crypt_config_t *priv = arg1;
1071         struct bio *bio = arg2;
1072
1073         KTR_LOG(dmcrypt_crypto_write_retry, bio->bio_buf);
1074
1075         dmtc_crypto_write_start(priv, bio);
1076 }
1077
1078 static void
1079 dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1080 {
1081         struct dmtc_helper *dmtc;
1082         struct cryptodesc *crd;
1083         struct cryptop *crp;
1084         int i, bytes, sectors, sz;
1085         off_t isector;
1086         u_char *ptr, *space;
1087
1088         /*
1089          * Use b_bcount for consistency
1090          */
1091         bytes = bio->bio_buf->b_bcount;
1092
1093         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
1094         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1095         sz = sectors * (sizeof(*crp) + sizeof(*crd));
1096
1097         /*
1098          * For writes and reads with bogus page don't decrypt in place.
1099          */
1100         space = mpipe_alloc_callback(&priv->write_mpipe,
1101                                      dmtc_crypto_write_retry, priv, bio);
1102         if (space == NULL)
1103                 return;
1104
1105         dmtc = (struct dmtc_helper *)space;
1106         dmtc->free_addr = space;
1107         space += sizeof(struct dmtc_helper);
1108         memcpy(space + sz, bio->bio_buf->b_data, bytes);
1109
1110         bio->bio_caller_info2.ptr = dmtc;
1111         bio->bio_buf->b_error = 0;
1112
1113         dmtc->orig_buf = bio->bio_buf->b_data;
1114         dmtc->data_buf = space + sz;
1115         dmtc->priv = priv;
1116
1117         /*
1118          * Load crypto descriptors (crp/crd loop)
1119          */
1120         bzero(space, sz);
1121         ptr = space;
1122         bio->bio_caller_info3.value = sectors;
1123         cpu_sfence();
1124 #if 0
1125         kprintf("Write, bytes = %d (b_bcount), "
1126                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1127                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1128 #endif
1129         for (i = 0; i < sectors; i++) {
1130                 crp = (struct cryptop *)ptr;
1131                 ptr += sizeof(*crp);
1132                 crd = (struct cryptodesc *)ptr;
1133                 ptr += sizeof (*crd);
1134
1135                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1136
1137                 crp->crp_sid = priv->crypto_sid;
1138                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1139
1140                 crp->crp_opaque = (void *)bio;
1141
1142                 crp->crp_callback = dmtc_crypto_cb_write_done;
1143                 crp->crp_desc = crd;
1144                 crp->crp_etype = 0;
1145                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1146                                  CRYPTO_F_BATCH;
1147
1148                 crd->crd_alg = priv->crypto_alg;
1149 #if 0
1150                 crd->crd_key = (caddr_t)priv->crypto_key;
1151                 crd->crd_klen = priv->crypto_klen;
1152 #endif
1153
1154                 crd->crd_skip = 0;
1155                 crd->crd_len = DEV_BSIZE /* XXX */;
1156                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1157                 crd->crd_next = NULL;
1158
1159                 crd->crd_flags |= CRD_F_ENCRYPT;
1160
1161                 /*
1162                  * Note: last argument is used to generate salt(?) and is
1163                  *       a 64 bit value, but the original code passed an
1164                  *       int.  Changing it now will break pre-existing
1165                  *       crypt volumes.
1166                  */
1167
1168                 KTR_LOG(dmcrypt_crypto_write_start, crp, bio->bio_buf,
1169                     i, sectors);
1170
1171                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1172                                     isector + i, crp);
1173         }
1174 }
1175
1176 /*
1177  * STRATEGY WRITE PATH PART 2/3
1178  */
1179 static int
1180 dmtc_crypto_cb_write_done(struct cryptop *crp)
1181 {
1182         struct dmtc_helper *dmtc;
1183         dm_target_crypt_config_t *priv;
1184         struct bio *bio, *obio;
1185         int n;
1186
1187         if (crp->crp_etype == EAGAIN)
1188                 return crypto_dispatch(crp);
1189
1190         bio = (struct bio *)crp->crp_opaque;
1191         KKASSERT(bio != NULL);
1192
1193         /*
1194          * Cumulative error
1195          */
1196         if (crp->crp_etype != 0) {
1197                 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1198                         "crp_etype = %d\n",
1199                 crp->crp_etype);
1200                 bio->bio_buf->b_error = crp->crp_etype;
1201         }
1202
1203         /*
1204          * On the last chunk of the encryption we issue the write
1205          */
1206         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1207 #if 0
1208         kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1209 #endif
1210
1211         KTR_LOG(dmcrypt_crypto_cb_write_done, crp, bio->bio_buf, n);
1212
1213         if (n == 1) {
1214                 dmtc = bio->bio_caller_info2.ptr;
1215                 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1216
1217                 if (bio->bio_buf->b_error) {
1218                         bio->bio_buf->b_flags |= B_ERROR;
1219                         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1220                         obio = pop_bio(bio);
1221                         biodone(obio);
1222                 } else {
1223                         dmtc->orig_buf = bio->bio_buf->b_data;
1224                         bio->bio_buf->b_data = dmtc->data_buf;
1225                         bio->bio_done = dmtc_bio_write_done;
1226                         vn_strategy(priv->pdev->pdev_vnode, bio);
1227                 }
1228         }
1229         return 0;
1230 }
1231
1232 /*
1233  * STRATEGY WRITE PATH PART 3/3
1234  */
1235 static void
1236 dmtc_bio_write_done(struct bio *bio)
1237 {
1238         struct dmtc_helper *dmtc;
1239         struct bio *obio;
1240
1241         dmtc = bio->bio_caller_info2.ptr;
1242         bio->bio_buf->b_data = dmtc->orig_buf;
1243         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1244
1245         KTR_LOG(dmcrypt_bio_write_done, bio->bio_buf);
1246
1247         obio = pop_bio(bio);
1248         biodone(obio);
1249 }
1250 /* END OF STRATEGY WRITE SECTION */
1251
1252
1253
1254 /* DUMPING MAGIC */
1255
1256 extern int tsleep_crypto_dump;
1257
1258 static int
1259 dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1260 {
1261         static struct dmtc_dump_helper dump_helper;
1262         dm_target_crypt_config_t *priv;
1263         int id;
1264         static int first_call = 1;
1265
1266         priv = table_en->target_config;
1267
1268         if (first_call) {
1269                 first_call = 0;
1270                 dump_reactivate_cpus();
1271         }
1272
1273         /* Magically enable tsleep */
1274         tsleep_crypto_dump = 1;
1275         id = 0;
1276
1277         /*
1278          * 0 length means flush buffers and return
1279          */
1280         if (length == 0) {
1281                 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1282                         tsleep_crypto_dump = 0;
1283                         return ENXIO;
1284                 }
1285                 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1286                     data, 0, offset, 0);
1287                 tsleep_crypto_dump = 0;
1288                 return 0;
1289         }
1290
1291         bzero(&dump_helper, sizeof(dump_helper));
1292         dump_helper.priv = priv;
1293         dump_helper.data = data;
1294         dump_helper.length = length;
1295         dump_helper.offset = offset +
1296             priv->block_offset * DEV_BSIZE;
1297         dump_helper.ident = &id;
1298         dmtc_crypto_dump_start(priv, &dump_helper);
1299
1300         /*
1301          * Hackery to make stuff appear synchronous. The crypto callback will
1302          * set id to 1 and call wakeup on it. If the request completed
1303          * synchronously, id will be 1 and we won't bother to sleep. If not,
1304          * the crypto request will complete asynchronously and we sleep until
1305          * it's done.
1306          */
1307         if (id == 0)
1308                 tsleep(&dump_helper, 0, "cryptdump", 0);
1309
1310         dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1311             dump_helper.offset);
1312
1313         dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1314             dump_helper.space, 0, dump_helper.offset,
1315             dump_helper.length);
1316
1317         tsleep_crypto_dump = 0;
1318         return 0;
1319 }
1320
1321 static void
1322 dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1323 {
1324         struct cryptodesc *crd;
1325         struct cryptop *crp;
1326         int i, bytes, sectors;
1327         off_t isector;
1328
1329         bytes = dump_helper->length;
1330
1331         isector = dump_helper->offset / DEV_BSIZE;      /* ivgen salt base? */
1332         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1333         dump_helper->sectors = sectors;
1334 #if 0
1335         kprintf("Dump, bytes = %d, "
1336                 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1337 #endif
1338         KKASSERT(dump_helper->length <= 65536);
1339
1340         memcpy(dump_helper->space, dump_helper->data, bytes);
1341
1342         cpu_sfence();
1343
1344         for (i = 0; i < sectors; i++) {
1345                 crp = &dump_helper->crp[i];
1346                 crd = &dump_helper->crd[i];
1347
1348                 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1349
1350                 crp->crp_sid = priv->crypto_sid;
1351                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1352
1353                 crp->crp_opaque = (void *)dump_helper;
1354
1355                 crp->crp_callback = dmtc_crypto_cb_dump_done;
1356                 crp->crp_desc = crd;
1357                 crp->crp_etype = 0;
1358                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1359                                  CRYPTO_F_BATCH;
1360
1361                 crd->crd_alg = priv->crypto_alg;
1362
1363                 crd->crd_skip = 0;
1364                 crd->crd_len = DEV_BSIZE /* XXX */;
1365                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1366                 crd->crd_next = NULL;
1367
1368                 crd->crd_flags |= CRD_F_ENCRYPT;
1369
1370                 /*
1371                  * Note: last argument is used to generate salt(?) and is
1372                  *       a 64 bit value, but the original code passed an
1373                  *       int.  Changing it now will break pre-existing
1374                  *       crypt volumes.
1375                  */
1376                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1377                                     isector + i, crp);
1378         }
1379 }
1380
1381 static int
1382 dmtc_crypto_cb_dump_done(struct cryptop *crp)
1383 {
1384         struct dmtc_dump_helper *dump_helper;
1385         int n;
1386
1387         if (crp->crp_etype == EAGAIN)
1388                 return crypto_dispatch(crp);
1389
1390         dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1391         KKASSERT(dump_helper != NULL);
1392
1393         if (crp->crp_etype != 0) {
1394                 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1395                         "crp_etype = %d\n",
1396                 crp->crp_etype);
1397                 return crp->crp_etype;
1398         }
1399
1400         /*
1401          * On the last chunk of the encryption we return control
1402          */
1403         n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1404
1405         if (n == 1) {
1406                 atomic_add_int(dump_helper->ident, 1);
1407                 wakeup(dump_helper);
1408         }
1409
1410         return 0;
1411 }
1412
1413 static int
1414 dmtc_mod_handler(module_t mod, int type, void *unused)
1415 {
1416         dm_target_t *dmt = NULL;
1417         int err = 0;
1418
1419         switch (type) {
1420         case MOD_LOAD:
1421                 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1422                         dm_target_unbusy(dmt);
1423                         return EEXIST;
1424                 }
1425                 dmt = dm_target_alloc("crypt");
1426                 dmt->version[0] = 1;
1427                 dmt->version[1] = 6;
1428                 dmt->version[2] = 0;
1429                 dmt->init = &dm_target_crypt_init;
1430                 dmt->destroy = &dm_target_crypt_destroy;
1431                 dmt->strategy = &dm_target_crypt_strategy;
1432                 dmt->table = &dm_target_crypt_table;
1433                 dmt->dump = &dm_target_crypt_dump;
1434
1435                 err = dm_target_insert(dmt);
1436                 if (!err)
1437                         kprintf("dm_target_crypt: Successfully initialized\n");
1438                 break;
1439
1440         case MOD_UNLOAD:
1441                 err = dm_target_remove("crypt");
1442                 if (err == 0) {
1443                         kprintf("dm_target_crypt: unloaded\n");
1444                 }
1445                 break;
1446
1447         default:
1448                 break;
1449         }
1450
1451         return err;
1452 }
1453
1454 DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);