sys/dev/disk/dm/targets/crypt/dm_target_crypt.c

   1 /*
   2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Alex Hornung <ahornung@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * This file implements initial version of device-mapper crypt target.
  37  */
  38 #include <sys/types.h>
  39 #include <sys/endian.h>
  40
  41 #include <sys/bio.h>
  42 #include <sys/buf.h>
  43 #include <sys/globaldata.h>
  44 #include <sys/kerneldump.h>
  45 #include <sys/malloc.h>
  46 #include <sys/mpipe.h>
  47 #include <sys/md5.h>
  48 #include <sys/mutex2.h>
  49 #include <sys/vnode.h>
  50 #include <crypto/sha1.h>
  51 #include <crypto/sha2/sha2.h>
  52 #include <opencrypto/cryptodev.h>
  53 #include <opencrypto/rmd160.h>
  54 #include <machine/cpufunc.h>
  55
  56 #include <sys/ktr.h>
  57
  58 #include <dev/disk/dm/dm.h>
  59 MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
  60
  61 KTR_INFO_MASTER(dmcrypt);
  62
  63 #if !defined(KTR_DMCRYPT)
  64 #define KTR_DMCRYPT     KTR_ALL
  65 #endif
  66
  67 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_dispatch, 0,
  68     "crypto_dispatch(%p)", struct cryptop *crp);
  69 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypt_strategy, 0,
  70     "crypt_strategy(b_cmd = %d, bp = %p)", int cmd, struct buf *bp);
  71 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_start, 1,
  72     "crypto_write_start(crp = %p, bp = %p, sector = %d/%d)",
  73     struct cryptop *crp, struct buf *bp, int i, int sectors);
  74 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_write_done, 1,
  75     "crypto_cb_write_done(crp = %p, bp = %p, n = %d)",
  76     struct cryptop *crp, struct buf *bp, int n);
  77 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_write_done, 1,
  78     "bio_write_done(bp = %p)", struct buf *bp);
  79 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_retry, 1,
  80     "crypto_write_retry(crp = %p)", struct buf *bp);
  81 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_read_done, 2,
  82     "bio_read_done(bp = %p)", struct buf *bp);
  83 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_read_start, 2,
  84     "crypto_read_start(crp = %p, bp = %p, sector = %d/%d)",
  85     struct cryptop *crp, struct buf *bp, int i, int sectors);
  86 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_read_done, 2,
  87     "crypto_cb_read_done(crp = %p, bp = %p, n = %d)",
  88     struct cryptop *crp, struct buf *bp, int n);
  89
  90 struct target_crypt_config;
  91
  92 typedef void dispatch_t(void *);
  93 typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
  94     void *);
  95
  96 typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
  97 typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
  98
  99 struct iv_generator {
 100         const char      *name;
 101         ivgen_ctor_t    *ctor;
 102         ivgen_dtor_t    *dtor;
 103         ivgen_t         *gen_iv;
 104 };
 105
 106 struct essiv_ivgen_priv {
 107         struct cryptoini        crypto_session;
 108         struct objcache *crp_crd_cache;
 109         u_int64_t       crypto_sid;
 110         size_t          keyhash_len;
 111         u_int8_t        crypto_keyhash[SHA512_DIGEST_LENGTH];
 112 };
 113
 114 typedef struct target_crypt_config {
 115         size_t  params_len;
 116         dm_pdev_t *pdev;
 117         char    *status_str;
 118         int     crypto_alg;
 119         int     crypto_klen;
 120         u_int8_t        crypto_key[512>>3];
 121
 122         u_int64_t       crypto_sid;
 123         u_int64_t       block_offset;
 124         int64_t         iv_offset;
 125         SHA512_CTX      essivsha512_ctx;
 126
 127         struct cryptoini        crypto_session;
 128
 129         struct iv_generator     *ivgen;
 130         void    *ivgen_priv;
 131
 132         struct malloc_pipe      read_mpipe;
 133         struct malloc_pipe      write_mpipe;
 134 } dm_target_crypt_config_t;
 135
 136 struct dmtc_helper {
 137         dm_target_crypt_config_t *priv;
 138         caddr_t free_addr;
 139         caddr_t orig_buf;
 140         caddr_t data_buf;
 141 };
 142
 143 struct dmtc_dump_helper {
 144         dm_target_crypt_config_t *priv;
 145         void *data;
 146         size_t length;
 147         off_t offset;
 148
 149         int sectors;
 150         int *ident;
 151
 152         struct cryptodesc crd[128];
 153         struct cryptop crp[128];
 154         u_char space[65536];
 155 };
 156
 157 #define DMTC_BUF_SIZE_WRITE \
 158     MAXPHYS + sizeof(struct dmtc_helper) + \
 159     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 160 #define DMTC_BUF_SIZE_READ \
 161     sizeof(struct dmtc_helper) + \
 162     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 163
 164 static void dmtc_crypto_dispatch(void *arg);
 165 static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
 166                                 struct dmtc_dump_helper *dump_helper);
 167 static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
 168                                 struct bio *bio);
 169 static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
 170                                 struct bio *bio);
 171 static void dmtc_bio_read_done(struct bio *bio);
 172 static void dmtc_bio_write_done(struct bio *bio);
 173 static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
 174 static int dmtc_crypto_cb_read_done(struct cryptop *crp);
 175 static int dmtc_crypto_cb_write_done(struct cryptop *crp);
 176
 177 static ivgen_ctor_t     essiv_ivgen_ctor;
 178 static ivgen_dtor_t     essiv_ivgen_dtor;
 179 static ivgen_t          essiv_ivgen;
 180 static ivgen_t          plain_ivgen;
 181 static ivgen_t          plain64_ivgen;
 182
 183 static struct iv_generator ivgens[] = {
 184         { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
 185             .gen_iv = essiv_ivgen },
 186         { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
 187         { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
 188         { NULL, NULL, NULL, NULL }
 189 };
 190
 191 struct objcache_malloc_args essiv_ivgen_malloc_args = {
 192                 2*sizeof(void *) + (sizeof(struct cryptodesc) +
 193                 sizeof(struct cryptop)), M_DMCRYPT };
 194
 195 static void
 196 dmtc_init_mpipe(struct target_crypt_config *priv)
 197 {
 198         int nmax;
 199
 200         nmax = (physmem*2/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
 201
 202         if (nmax < 2)
 203                 nmax = 2;
 204
 205         kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
 206
 207         mpipe_init(&priv->write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
 208                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 209         mpipe_init(&priv->read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
 210                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 211 }
 212
 213 static void
 214 dmtc_destroy_mpipe(struct target_crypt_config *priv)
 215 {
 216         mpipe_done(&priv->write_mpipe);
 217         mpipe_done(&priv->read_mpipe);
 218 }
 219
 220 /*
 221  * Overwrite private information (in buf) to avoid leaking it
 222  */
 223 static void
 224 dmtc_crypto_clear(void *buf, size_t len)
 225 {
 226         memset(buf, 0xFF, len);
 227         bzero(buf, len);
 228 }
 229
 230 /*
 231  * ESSIV IV Generator Routines
 232  */
 233 static int
 234 essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
 235 {
 236         struct essiv_ivgen_priv *ivpriv;
 237         u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
 238         unsigned int klen, hashlen;
 239         int error;
 240
 241         klen = (priv->crypto_klen >> 3);
 242
 243         if (iv_hash == NULL)
 244                 return EINVAL;
 245
 246         if (!strcmp(iv_hash, "sha1")) {
 247                 SHA1_CTX ctx;
 248
 249                 hashlen = SHA1_RESULTLEN;
 250                 SHA1Init(&ctx);
 251                 SHA1Update(&ctx, priv->crypto_key, klen);
 252                 SHA1Final(crypto_keyhash, &ctx);
 253         } else if (!strcmp(iv_hash, "sha256")) {
 254                 SHA256_CTX ctx;
 255
 256                 hashlen = SHA256_DIGEST_LENGTH;
 257                 SHA256_Init(&ctx);
 258                 SHA256_Update(&ctx, priv->crypto_key, klen);
 259                 SHA256_Final(crypto_keyhash, &ctx);
 260         } else if (!strcmp(iv_hash, "sha384")) {
 261                 SHA384_CTX ctx;
 262
 263                 hashlen = SHA384_DIGEST_LENGTH;
 264                 SHA384_Init(&ctx);
 265                 SHA384_Update(&ctx, priv->crypto_key, klen);
 266                 SHA384_Final(crypto_keyhash, &ctx);
 267         } else if (!strcmp(iv_hash, "sha512")) {
 268                 SHA512_CTX ctx;
 269
 270                 hashlen = SHA512_DIGEST_LENGTH;
 271                 SHA512_Init(&ctx);
 272                 SHA512_Update(&ctx, priv->crypto_key, klen);
 273                 SHA512_Final(crypto_keyhash, &ctx);
 274         } else if (!strcmp(iv_hash, "md5")) {
 275                 MD5_CTX ctx;
 276
 277                 hashlen = MD5_DIGEST_LENGTH;
 278                 MD5Init(&ctx);
 279                 MD5Update(&ctx, priv->crypto_key, klen);
 280                 MD5Final(crypto_keyhash, &ctx);
 281         } else if (!strcmp(iv_hash, "rmd160") ||
 282                    !strcmp(iv_hash, "ripemd160")) {
 283                 RMD160_CTX ctx;
 284
 285                 hashlen = 160/8;
 286                 RMD160Init(&ctx);
 287                 RMD160Update(&ctx, priv->crypto_key, klen);
 288                 RMD160Final(crypto_keyhash, &ctx);
 289         } else {
 290                 return EINVAL;
 291         }
 292
 293         /* Convert hashlen to bits */
 294         hashlen <<= 3;
 295
 296         ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
 297             M_WAITOK | M_ZERO);
 298         memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
 299         ivpriv->keyhash_len = sizeof(crypto_keyhash);
 300         dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
 301
 302         ivpriv->crypto_session.cri_alg = priv->crypto_alg;
 303         ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
 304         ivpriv->crypto_session.cri_klen = hashlen;
 305         ivpriv->crypto_session.cri_mlen = 0;
 306         ivpriv->crypto_session.cri_next = NULL;
 307
 308         /*
 309          * XXX: in principle we also need to check if the block size of the
 310          *      cipher is a valid iv size for the block cipher.
 311          */
 312
 313         error = crypto_newsession(&ivpriv->crypto_sid,
 314                                   &ivpriv->crypto_session,
 315                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 316         if (error) {
 317                 kprintf("dm_target_crypt: Error during crypto_newsession "
 318                         "for essiv_ivgen, error = %d\n",
 319                         error);
 320                 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 321                 kfree(ivpriv, M_DMCRYPT);
 322                 return ENOTSUP;
 323         }
 324
 325         ivpriv->crp_crd_cache = objcache_create(
 326             "dmcrypt-essiv-cache", 0, 0,
 327             NULL, NULL, NULL,
 328             objcache_malloc_alloc,
 329             objcache_malloc_free,
 330             &essiv_ivgen_malloc_args );
 331
 332         *p_ivpriv = ivpriv;
 333         return 0;
 334 }
 335
 336 static int
 337 essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
 338 {
 339         struct essiv_ivgen_priv *ivpriv;
 340
 341         ivpriv = (struct essiv_ivgen_priv *)arg;
 342         KKASSERT(ivpriv != NULL);
 343
 344         crypto_freesession(ivpriv->crypto_sid);
 345
 346         objcache_destroy(ivpriv->crp_crd_cache);
 347
 348         dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 349         kfree(ivpriv, M_DMCRYPT);
 350
 351         return 0;
 352 }
 353
 354 static int
 355 essiv_ivgen_done(struct cryptop *crp)
 356 {
 357         struct essiv_ivgen_priv *ivpriv;
 358         void *free_addr;
 359         void *opaque;
 360
 361
 362         if (crp->crp_etype == EAGAIN)
 363                 return crypto_dispatch(crp);
 364
 365         if (crp->crp_etype != 0) {
 366                 kprintf("dm_target_crypt: essiv_ivgen_done, "
 367                         "crp->crp_etype = %d\n", crp->crp_etype);
 368         }
 369
 370         free_addr = crp->crp_opaque;
 371         /*
 372          * In-memory structure is:
 373          * |  ivpriv  |  opaque  |     crp     |      crd      |
 374          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 375          */
 376         ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
 377         crp->crp_opaque += sizeof(void *);
 378         opaque = *((void **)crp->crp_opaque);
 379
 380         objcache_put(ivpriv->crp_crd_cache, free_addr);
 381         dmtc_crypto_dispatch(opaque);
 382         return 0;
 383 }
 384
 385 static void
 386 essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 387             size_t iv_len, off_t sector, void *opaque)
 388 {
 389         struct essiv_ivgen_priv *ivpriv;
 390         struct cryptodesc *crd;
 391         struct cryptop *crp;
 392         caddr_t space, alloc_addr;
 393         int error;
 394
 395         ivpriv = priv->ivgen_priv;
 396         KKASSERT(ivpriv != NULL);
 397
 398         /*
 399          * In-memory structure is:
 400          * |  ivpriv  |  opaque  |     crp     |      crd      |
 401          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 402          */
 403         alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
 404         *((struct essiv_ivgen_priv **)space) = ivpriv;
 405         space += sizeof(void *);
 406         *((void **)space) = opaque;
 407         space += sizeof(void *);
 408         crp = (struct cryptop *)space;
 409         space += sizeof(struct cryptop);
 410         crd = (struct cryptodesc *)space;
 411
 412         bzero(iv, iv_len);
 413         bzero(crd, sizeof(struct cryptodesc));
 414         bzero(crp, sizeof(struct cryptop));
 415         *((off_t *)iv) = htole64(sector + priv->iv_offset);
 416         crp->crp_buf = (caddr_t)iv;
 417
 418         crp->crp_sid = ivpriv->crypto_sid;
 419         crp->crp_ilen = crp->crp_olen = iv_len;
 420
 421         crp->crp_opaque = alloc_addr;
 422
 423         crp->crp_callback = essiv_ivgen_done;
 424
 425         crp->crp_desc = crd;
 426         crp->crp_etype = 0;
 427         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
 428
 429         crd->crd_alg = priv->crypto_alg;
 430 #if 0
 431         crd->crd_key = (caddr_t)priv->crypto_keyhash;
 432         crd->crd_klen = priv->crypto_klen;
 433 #endif
 434
 435         bzero(crd->crd_iv, sizeof(crd->crd_iv));
 436
 437         crd->crd_skip = 0;
 438         crd->crd_len = iv_len;
 439         crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 440         crd->crd_flags |= CRD_F_ENCRYPT;
 441         crd->crd_next = NULL;
 442
 443         error = crypto_dispatch(crp);
 444         if (error)
 445                 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
 446 }
 447
 448
 449 static void
 450 plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 451             size_t iv_len, off_t sector, void *opaque)
 452 {
 453         bzero(iv, iv_len);
 454         *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
 455         dmtc_crypto_dispatch(opaque);
 456 }
 457
 458 static void
 459 plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 460     size_t iv_len, off_t sector, void *opaque)
 461 {
 462         bzero(iv, iv_len);
 463         *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
 464         dmtc_crypto_dispatch(opaque);
 465 }
 466
 467 #if 0
 468 static void
 469 geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 470            size_t iv_len, off_t sector, void *opaque)
 471 {
 472
 473         SHA512_CTX      ctx512;
 474         u_int8_t        md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
 475
 476         memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
 477         SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
 478         SHA512_Final(md, &ctx512);
 479
 480         memcpy(iv, md, iv_len);
 481         dmtc_crypto_dispatch(opaque);
 482 }
 483 #endif
 484
 485 /*
 486  * Init function called from dm_table_load_ioctl.
 487  * cryptsetup actually passes us this:
 488  * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
 489  */
 490 static int
 491 hex2key(char *hex, size_t key_len, u_int8_t *key)
 492 {
 493         char hex_buf[3];
 494         size_t key_idx;
 495
 496         hex_buf[2] = 0;
 497         for (key_idx = 0; key_idx < key_len; ++key_idx) {
 498                 hex_buf[0] = *hex++;
 499                 hex_buf[1] = *hex++;
 500                 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
 501         }
 502         hex_buf[0] = 0;
 503         hex_buf[1] = 0;
 504
 505         return 0;
 506 }
 507
 508 static int
 509 dm_target_crypt_init(dm_table_entry_t *table_en, char *params)
 510 {
 511         dm_target_crypt_config_t *priv;
 512         size_t len;
 513         char **ap, *args[5];
 514         char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
 515         char *status_str;
 516         int i, argc, klen, error;
 517         uint64_t iv_offset, block_offset;
 518
 519         if (params == NULL)
 520                 return EINVAL;
 521
 522         len = strlen(params) + 1;
 523         argc = 0;
 524
 525         status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
 526         /*
 527          * Parse a string, containing tokens delimited by white space,
 528          * into an argument vector
 529          */
 530         for (ap = args; ap < &args[5] &&
 531             (*ap = strsep(&params, " \t")) != NULL;) {
 532                 if (**ap != '\0') {
 533                         argc++;
 534                         ap++;
 535                 }
 536         }
 537
 538         if (argc != 5) {
 539                 kprintf("dm_target_crypt: not enough arguments, "
 540                         "need exactly 5\n");
 541                 kfree(status_str, M_DMCRYPT);
 542                 return ENOMEM; /* XXX */
 543         }
 544
 545         crypto_alg = strsep(&args[0], "-");
 546         crypto_mode = strsep(&args[0], "-");
 547         iv_opt = strsep(&args[0], "-");
 548         iv_mode = strsep(&iv_opt, ":");
 549         key = args[1];
 550         iv_offset = strtouq(args[2], NULL, 0);
 551         dev = args[3];
 552         block_offset = strtouq(args[4], NULL, 0);
 553         /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
 554         klen = strlen(key) << 2;
 555
 556 #if 0
 557         kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
 558                 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
 559                 "block_offset=%ju\n",
 560                 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
 561                 block_offset);
 562 #endif
 563
 564         priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
 565
 566         /* Insert dmp to global pdev list */
 567         if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
 568                 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
 569                 kfree(status_str, M_DMCRYPT);
 570                 return ENOENT;
 571         }
 572
 573         /*
 574          * This code checks for valid combinations of algorithm and mode.
 575          * Currently supported options are:
 576          *
 577          * *-cbc
 578          * aes-xts
 579          * twofish-xts
 580          * serpent-xts
 581          */
 582         if ((strcmp(crypto_mode, "cbc") != 0) &&
 583             !((strcmp(crypto_mode, "xts") == 0) &&
 584             ((strcmp(crypto_alg, "aes") == 0) ||
 585             (strcmp(crypto_alg, "twofish") == 0) ||
 586             (strcmp(crypto_alg, "serpent") == 0))))
 587         {
 588                 kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
 589                     " aes-xts, twofish-xts and serpent-xts, "
 590                     "invalid mode '%s-%s'\n",
 591                     crypto_alg, crypto_mode);
 592                 goto notsup;
 593         }
 594
 595         if (!strcmp(crypto_alg, "aes")) {
 596                 if (!strcmp(crypto_mode, "xts")) {
 597                         priv->crypto_alg = CRYPTO_AES_XTS;
 598                         if (klen != 256 && klen != 512)
 599                                 goto notsup;
 600                 } else if (!strcmp(crypto_mode, "cbc")) {
 601                         priv->crypto_alg = CRYPTO_AES_CBC;
 602                         if (klen != 128 && klen != 192 && klen != 256)
 603                                 goto notsup;
 604                 } else {
 605                         goto notsup;
 606                 }
 607                 priv->crypto_klen = klen;
 608         } else if (!strcmp(crypto_alg, "twofish")) {
 609                 if (!strcmp(crypto_mode, "xts")) {
 610                         priv->crypto_alg = CRYPTO_TWOFISH_XTS;
 611                         if (klen != 256 && klen != 512)
 612                                 goto notsup;
 613                 } else if (!strcmp(crypto_mode, "cbc")) {
 614                         priv->crypto_alg = CRYPTO_TWOFISH_CBC;
 615                         if (klen != 128 && klen != 192 && klen != 256)
 616                                 goto notsup;
 617                 } else {
 618                         goto notsup;
 619                 }
 620                 priv->crypto_klen = klen;
 621         } else if (!strcmp(crypto_alg, "serpent")) {
 622                 if (!strcmp(crypto_mode, "xts")) {
 623                         priv->crypto_alg = CRYPTO_SERPENT_XTS;
 624                         if (klen != 256 && klen != 512)
 625                                 goto notsup;
 626                 } else if (!strcmp(crypto_mode, "cbc")) {
 627                         priv->crypto_alg = CRYPTO_SERPENT_CBC;
 628                         if (klen != 128 && klen != 192 && klen != 256)
 629                                 goto notsup;
 630                 } else {
 631                         goto notsup;
 632                 }
 633                 priv->crypto_klen = klen;
 634         } else if (!strcmp(crypto_alg, "blowfish")) {
 635                 priv->crypto_alg = CRYPTO_BLF_CBC;
 636                 if (klen < 128 || klen > 448 || (klen % 8) != 0)
 637                         goto notsup;
 638                 priv->crypto_klen = klen;
 639         } else if (!strcmp(crypto_alg, "3des") ||
 640                    !strncmp(crypto_alg, "des3", 4)) {
 641                 priv->crypto_alg = CRYPTO_3DES_CBC;
 642                 if (klen != 168)
 643                         goto notsup;
 644                 priv->crypto_klen = 168;
 645         } else if (!strcmp(crypto_alg, "camellia")) {
 646                 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
 647                 if (klen != 128 && klen != 192 && klen != 256)
 648                         goto notsup;
 649                 priv->crypto_klen = klen;
 650         } else if (!strcmp(crypto_alg, "skipjack")) {
 651                 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
 652                 if (klen != 80)
 653                         goto notsup;
 654                 priv->crypto_klen = 80;
 655         } else if (!strcmp(crypto_alg, "cast5")) {
 656                 priv->crypto_alg = CRYPTO_CAST_CBC;
 657                 if (klen != 128)
 658                         goto notsup;
 659                 priv->crypto_klen = 128;
 660         } else if (!strcmp(crypto_alg, "null")) {
 661                 priv->crypto_alg = CRYPTO_NULL_CBC;
 662                 if (klen != 128)
 663                         goto notsup;
 664                 priv->crypto_klen = 128;
 665         } else {
 666                 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
 667                         crypto_alg);
 668                 goto notsup;
 669         }
 670
 671         /* Save length of param string */
 672         priv->params_len = len;
 673         priv->block_offset = block_offset;
 674         priv->iv_offset = iv_offset - block_offset;
 675
 676         dm_table_add_deps(table_en, priv->pdev);
 677
 678         dm_table_init_target(table_en, DM_CRYPTO_DEV, priv);
 679
 680         error = hex2key(key, priv->crypto_klen >> 3,
 681                         (u_int8_t *)priv->crypto_key);
 682
 683         if (error) {
 684                 kprintf("dm_target_crypt: hex2key failed, "
 685                         "invalid key format\n");
 686                 goto notsup;
 687         }
 688
 689         /* Handle cmd */
 690         for(i = 0; ivgens[i].name != NULL; i++) {
 691                 if (!strcmp(iv_mode, ivgens[i].name))
 692                         break;
 693         }
 694
 695         if (ivgens[i].name == NULL) {
 696                 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
 697                         iv_mode);
 698                 goto notsup;
 699         }
 700
 701         /* Call our ivgen constructor */
 702         if (ivgens[i].ctor != NULL) {
 703                 error = ivgens[i].ctor(priv, iv_opt,
 704                     &priv->ivgen_priv);
 705                 if (error) {
 706                         kprintf("dm_target_crypt: ctor for '%s' failed\n",
 707                             ivgens[i].name);
 708                         goto notsup;
 709                 }
 710         }
 711
 712         priv->ivgen = &ivgens[i];
 713
 714         priv->crypto_session.cri_alg = priv->crypto_alg;
 715         priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
 716         priv->crypto_session.cri_klen = priv->crypto_klen;
 717         priv->crypto_session.cri_mlen = 0;
 718         priv->crypto_session.cri_next = NULL;
 719
 720         error = crypto_newsession(&priv->crypto_sid,
 721                                   &priv->crypto_session,
 722                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 723         if (error) {
 724                 kprintf("dm_target_crypt: Error during crypto_newsession, "
 725                         "error = %d\n",
 726                         error);
 727                 goto notsup;
 728         }
 729
 730         memset(key, '0', strlen(key));
 731         if (iv_opt) {
 732                 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
 733                     crypto_alg, crypto_mode, iv_mode, iv_opt,
 734                     key, iv_offset, dev, block_offset);
 735         } else {
 736                 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
 737                     crypto_alg, crypto_mode, iv_mode,
 738                     key, iv_offset, dev, block_offset);
 739         }
 740         priv->status_str = status_str;
 741
 742         /* Initialize mpipes */
 743         dmtc_init_mpipe(priv);
 744
 745         return 0;
 746
 747 notsup:
 748         kprintf("dm_target_crypt: ENOTSUP\n");
 749         kfree(status_str, M_DMCRYPT);
 750         return ENOTSUP;
 751 }
 752
 753 /* Table routine called to get params string. */
 754 static char *
 755 dm_target_crypt_table(void *target_config)
 756 {
 757         dm_target_crypt_config_t *priv;
 758         char *params;
 759
 760         priv = target_config;
 761
 762         /* caller expects use of M_DM */
 763         params = kmalloc(DM_MAX_PARAMS_SIZE, M_DM, M_WAITOK);
 764
 765         ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
 766             priv->status_str);
 767
 768         return params;
 769 }
 770
 771 static int
 772 dm_target_crypt_destroy(dm_table_entry_t *table_en)
 773 {
 774         dm_target_crypt_config_t *priv;
 775
 776         /*
 777          * Disconnect the crypt config before unbusying the target.
 778          */
 779         priv = table_en->target_config;
 780         if (priv == NULL)
 781                 return 0;
 782         table_en->target_config = NULL;
 783         dm_pdev_decr(priv->pdev);
 784
 785         /*
 786          * Clean up the crypt config
 787          *
 788          * Overwrite the private information before freeing memory to
 789          * avoid leaking it.
 790          */
 791         if (priv->status_str) {
 792                 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
 793                 kfree(priv->status_str, M_DMCRYPT);
 794                 crypto_freesession(priv->crypto_sid);
 795         }
 796
 797         if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
 798                 priv->ivgen->dtor(priv, priv->ivgen_priv);
 799         }
 800
 801         /* Destroy mpipes */
 802         dmtc_destroy_mpipe(priv);
 803
 804         dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
 805         kfree(priv, M_DMCRYPT);
 806
 807         return 0;
 808 }
 809
 810 /* Unsupported for this target. */
 811 static int
 812 dm_target_crypt_upcall(dm_table_entry_t *table_en, struct buf *bp)
 813 {
 814         return 0;
 815 }
 816
 817 /************************************************************************
 818  *                      STRATEGY SUPPORT FUNCTIONS                      *
 819  ************************************************************************
 820  *
 821  * READ PATH:   doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
 822  * WRITE PATH:  crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
 823  */
 824
 825 /*
 826  * Wrapper around crypto_dispatch() to match dispatch_t type
 827  */
 828 static void
 829 dmtc_crypto_dispatch(void *arg)
 830 {
 831         struct cryptop *crp;
 832
 833         crp = (struct cryptop *)arg;
 834         KKASSERT(crp != NULL);
 835         KTR_LOG(dmcrypt_crypto_dispatch, crp);
 836         crypto_dispatch(crp);
 837 }
 838
 839 /*
 840  * Start IO operation, called from dmstrategy routine.
 841  */
 842 static int
 843 dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
 844 {
 845         struct bio *bio;
 846
 847         dm_target_crypt_config_t *priv;
 848         priv = table_en->target_config;
 849
 850         /* Get rid of stuff we can't really handle */
 851         if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
 852                 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
 853                         kprintf("dm_target_crypt_strategy: can't really "
 854                                 "handle bp->b_bcount = %d\n",
 855                                 bp->b_bcount);
 856                         bp->b_error = EINVAL;
 857                         bp->b_flags |= B_ERROR | B_INVAL;
 858                         biodone(&bp->b_bio1);
 859                         return 0;
 860                 }
 861         }
 862
 863         KTR_LOG(dmcrypt_crypt_strategy, bp->b_cmd, bp);
 864
 865         switch (bp->b_cmd) {
 866         case BUF_CMD_READ:
 867                 bio = push_bio(&bp->b_bio1);
 868                 bio->bio_offset = bp->b_bio1.bio_offset +
 869                                   priv->block_offset * DEV_BSIZE;
 870                 bio->bio_caller_info1.ptr = priv;
 871                 bio->bio_done = dmtc_bio_read_done;
 872                 vn_strategy(priv->pdev->pdev_vnode, bio);
 873                 break;
 874         case BUF_CMD_WRITE:
 875                 bio = push_bio(&bp->b_bio1);
 876                 bio->bio_offset = bp->b_bio1.bio_offset +
 877                                   priv->block_offset * DEV_BSIZE;
 878                 bio->bio_caller_info1.ptr = priv;
 879                 dmtc_crypto_write_start(priv, bio);
 880                 break;
 881         default:
 882                 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
 883                 break;
 884         }
 885         return 0;
 886 }
 887
 888 /*
 889  * STRATEGY READ PATH PART 1/3 (after read BIO completes)
 890  */
 891 static void
 892 dmtc_bio_read_done(struct bio *bio)
 893 {
 894         struct bio *obio;
 895
 896         dm_target_crypt_config_t *priv;
 897
 898         KTR_LOG(dmcrypt_bio_read_done, bio->bio_buf);
 899
 900         /*
 901          * If a read error occurs we shortcut the operation, otherwise
 902          * go on to stage 2.
 903          */
 904         if (bio->bio_buf->b_flags & B_ERROR) {
 905                 obio = pop_bio(bio);
 906                 biodone(obio);
 907         } else {
 908                 priv = bio->bio_caller_info1.ptr;
 909                 dmtc_crypto_read_start(priv, bio);
 910         }
 911 }
 912
 913 /*
 914  * STRATEGY READ PATH PART 2/3
 915  */
 916 static void
 917 dmtc_crypto_read_retry(void *arg1, void *arg2)
 918 {
 919         dm_target_crypt_config_t *priv = arg1;
 920         struct bio *bio = arg2;
 921
 922         dmtc_crypto_read_start(priv, bio);
 923 }
 924
 925 static void
 926 dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
 927 {
 928         struct dmtc_helper *dmtc;
 929         struct cryptodesc *crd;
 930         struct cryptop *crp;
 931         int i, bytes, sectors, sz;
 932         off_t isector;
 933         u_char *ptr, *space;
 934
 935         /*
 936          * Note: b_resid no good after read I/O, it will be 0, use
 937          *       b_bcount.
 938          */
 939         bytes = bio->bio_buf->b_bcount;
 940         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
 941         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
 942         sz = sectors * (sizeof(*crp) + sizeof(*crd));
 943
 944         /*
 945          * For reads with bogus page we can't decrypt in place as stuff
 946          * can get ripped out from under us.
 947          *
 948          * XXX actually it looks like we can, and in any case the initial
 949          * read already completed and threw crypted data into the buffer
 950          * cache buffer.  Disable for now.
 951          */
 952         space = mpipe_alloc_callback(&priv->read_mpipe,
 953                                      dmtc_crypto_read_retry, priv, bio);
 954         if (space == NULL)
 955                 return;
 956
 957         dmtc = (struct dmtc_helper *)space;
 958         dmtc->free_addr = space;
 959         space += sizeof(struct dmtc_helper);
 960         dmtc->orig_buf = NULL;
 961         dmtc->data_buf = bio->bio_buf->b_data;
 962         dmtc->priv = priv;
 963         bio->bio_caller_info2.ptr = dmtc;
 964         bio->bio_buf->b_error = 0;
 965
 966         /*
 967          * Load crypto descriptors (crp/crd loop)
 968          */
 969         bzero(space, sz);
 970         ptr = space;
 971         bio->bio_caller_info3.value = sectors;
 972         cpu_sfence();
 973 #if 0
 974         kprintf("Read, bytes = %d (b_bcount), "
 975                 "sectors = %d (bio = %p, b_cmd = %d)\n",
 976                 bytes, sectors, bio, bio->bio_buf->b_cmd);
 977 #endif
 978         for (i = 0; i < sectors; i++) {
 979                 crp = (struct cryptop *)ptr;
 980                 ptr += sizeof(*crp);
 981                 crd = (struct cryptodesc *)ptr;
 982                 ptr += sizeof (*crd);
 983
 984                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
 985
 986                 crp->crp_sid = priv->crypto_sid;
 987                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
 988
 989                 crp->crp_opaque = (void *)bio;
 990
 991                 crp->crp_callback = dmtc_crypto_cb_read_done;
 992                 crp->crp_desc = crd;
 993                 crp->crp_etype = 0;
 994                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
 995                                  CRYPTO_F_BATCH;
 996
 997                 crd->crd_alg = priv->crypto_alg;
 998 #if 0
 999                 crd->crd_key = (caddr_t)priv->crypto_key;
1000                 crd->crd_klen = priv->crypto_klen;
1001 #endif
1002
1003                 crd->crd_skip = 0;
1004                 crd->crd_len = DEV_BSIZE /* XXX */;
1005                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1006                 crd->crd_next = NULL;
1007
1008                 crd->crd_flags &= ~CRD_F_ENCRYPT;
1009
1010                 KTR_LOG(dmcrypt_crypto_read_start, crp, bio->bio_buf, i,
1011                     sectors);
1012
1013                 /*
1014                  * Note: last argument is used to generate salt(?) and is
1015                  *       a 64 bit value, but the original code passed an
1016                  *       int.  Changing it now will break pre-existing
1017                  *       crypt volumes.
1018                  */
1019                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1020                                     isector + i, crp);
1021         }
1022 }
1023
1024 /*
1025  * STRATEGY READ PATH PART 3/3
1026  */
1027 static int
1028 dmtc_crypto_cb_read_done(struct cryptop *crp)
1029 {
1030         struct dmtc_helper *dmtc;
1031         struct bio *bio, *obio;
1032         int n;
1033
1034         if (crp->crp_etype == EAGAIN)
1035                 return crypto_dispatch(crp);
1036
1037         bio = (struct bio *)crp->crp_opaque;
1038         KKASSERT(bio != NULL);
1039
1040         /*
1041          * Cumulative error
1042          */
1043         if (crp->crp_etype) {
1044                 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1045                         "crp_etype = %d\n",
1046                         crp->crp_etype);
1047                 bio->bio_buf->b_error = crp->crp_etype;
1048         }
1049
1050         /*
1051          * On the last chunk of the decryption we do any required copybacks
1052          * and complete the I/O.
1053          */
1054         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1055 #if 0
1056         kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1057 #endif
1058
1059         KTR_LOG(dmcrypt_crypto_cb_read_done, crp, bio->bio_buf, n);
1060
1061         if (n == 1) {
1062                 /*
1063                  * For the B_HASBOGUS case we didn't decrypt in place,
1064                  * so we need to copy stuff back into the buf.
1065                  *
1066                  * (disabled for now).
1067                  */
1068                 dmtc = bio->bio_caller_info2.ptr;
1069                 if (bio->bio_buf->b_error) {
1070                         bio->bio_buf->b_flags |= B_ERROR;
1071                 }
1072 #if 0
1073                 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
1074                         memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1075                                bio->bio_buf->b_bcount);
1076                 }
1077 #endif
1078                 mpipe_free(&dmtc->priv->read_mpipe, dmtc->free_addr);
1079                 obio = pop_bio(bio);
1080                 biodone(obio);
1081         }
1082         return 0;
1083 }
1084 /* END OF STRATEGY READ SECTION */
1085
1086 /*
1087  * STRATEGY WRITE PATH PART 1/3
1088  */
1089
1090 static void
1091 dmtc_crypto_write_retry(void *arg1, void *arg2)
1092 {
1093         dm_target_crypt_config_t *priv = arg1;
1094         struct bio *bio = arg2;
1095
1096         KTR_LOG(dmcrypt_crypto_write_retry, bio->bio_buf);
1097
1098         dmtc_crypto_write_start(priv, bio);
1099 }
1100
1101 static void
1102 dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1103 {
1104         struct dmtc_helper *dmtc;
1105         struct cryptodesc *crd;
1106         struct cryptop *crp;
1107         int i, bytes, sectors, sz;
1108         off_t isector;
1109         u_char *ptr, *space;
1110
1111         /*
1112          * Use b_bcount for consistency
1113          */
1114         bytes = bio->bio_buf->b_bcount;
1115
1116         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
1117         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1118         sz = sectors * (sizeof(*crp) + sizeof(*crd));
1119
1120         /*
1121          * For writes and reads with bogus page don't decrypt in place.
1122          */
1123         space = mpipe_alloc_callback(&priv->write_mpipe,
1124                                      dmtc_crypto_write_retry, priv, bio);
1125         if (space == NULL)
1126                 return;
1127
1128         dmtc = (struct dmtc_helper *)space;
1129         dmtc->free_addr = space;
1130         space += sizeof(struct dmtc_helper);
1131         memcpy(space + sz, bio->bio_buf->b_data, bytes);
1132
1133         bio->bio_caller_info2.ptr = dmtc;
1134         bio->bio_buf->b_error = 0;
1135
1136         dmtc->orig_buf = bio->bio_buf->b_data;
1137         dmtc->data_buf = space + sz;
1138         dmtc->priv = priv;
1139
1140         /*
1141          * Load crypto descriptors (crp/crd loop)
1142          */
1143         bzero(space, sz);
1144         ptr = space;
1145         bio->bio_caller_info3.value = sectors;
1146         cpu_sfence();
1147 #if 0
1148         kprintf("Write, bytes = %d (b_bcount), "
1149                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1150                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1151 #endif
1152         for (i = 0; i < sectors; i++) {
1153                 crp = (struct cryptop *)ptr;
1154                 ptr += sizeof(*crp);
1155                 crd = (struct cryptodesc *)ptr;
1156                 ptr += sizeof (*crd);
1157
1158                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1159
1160                 crp->crp_sid = priv->crypto_sid;
1161                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1162
1163                 crp->crp_opaque = (void *)bio;
1164
1165                 crp->crp_callback = dmtc_crypto_cb_write_done;
1166                 crp->crp_desc = crd;
1167                 crp->crp_etype = 0;
1168                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1169                                  CRYPTO_F_BATCH;
1170
1171                 crd->crd_alg = priv->crypto_alg;
1172 #if 0
1173                 crd->crd_key = (caddr_t)priv->crypto_key;
1174                 crd->crd_klen = priv->crypto_klen;
1175 #endif
1176
1177                 crd->crd_skip = 0;
1178                 crd->crd_len = DEV_BSIZE /* XXX */;
1179                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1180                 crd->crd_next = NULL;
1181
1182                 crd->crd_flags |= CRD_F_ENCRYPT;
1183
1184                 /*
1185                  * Note: last argument is used to generate salt(?) and is
1186                  *       a 64 bit value, but the original code passed an
1187                  *       int.  Changing it now will break pre-existing
1188                  *       crypt volumes.
1189                  */
1190
1191                 KTR_LOG(dmcrypt_crypto_write_start, crp, bio->bio_buf,
1192                     i, sectors);
1193
1194                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1195                                     isector + i, crp);
1196         }
1197 }
1198
1199 /*
1200  * STRATEGY WRITE PATH PART 2/3
1201  */
1202 static int
1203 dmtc_crypto_cb_write_done(struct cryptop *crp)
1204 {
1205         struct dmtc_helper *dmtc;
1206         dm_target_crypt_config_t *priv;
1207         struct bio *bio, *obio;
1208         int n;
1209
1210         if (crp->crp_etype == EAGAIN)
1211                 return crypto_dispatch(crp);
1212
1213         bio = (struct bio *)crp->crp_opaque;
1214         KKASSERT(bio != NULL);
1215
1216         /*
1217          * Cumulative error
1218          */
1219         if (crp->crp_etype != 0) {
1220                 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1221                         "crp_etype = %d\n",
1222                 crp->crp_etype);
1223                 bio->bio_buf->b_error = crp->crp_etype;
1224         }
1225
1226         /*
1227          * On the last chunk of the encryption we issue the write
1228          */
1229         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1230 #if 0
1231         kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1232 #endif
1233
1234         KTR_LOG(dmcrypt_crypto_cb_write_done, crp, bio->bio_buf, n);
1235
1236         if (n == 1) {
1237                 dmtc = bio->bio_caller_info2.ptr;
1238                 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1239
1240                 if (bio->bio_buf->b_error) {
1241                         bio->bio_buf->b_flags |= B_ERROR;
1242                         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1243                         obio = pop_bio(bio);
1244                         biodone(obio);
1245                 } else {
1246                         dmtc->orig_buf = bio->bio_buf->b_data;
1247                         bio->bio_buf->b_data = dmtc->data_buf;
1248                         bio->bio_done = dmtc_bio_write_done;
1249                         vn_strategy(priv->pdev->pdev_vnode, bio);
1250                 }
1251         }
1252         return 0;
1253 }
1254
1255 /*
1256  * STRATEGY WRITE PATH PART 3/3
1257  */
1258 static void
1259 dmtc_bio_write_done(struct bio *bio)
1260 {
1261         struct dmtc_helper *dmtc;
1262         struct bio *obio;
1263
1264         dmtc = bio->bio_caller_info2.ptr;
1265         bio->bio_buf->b_data = dmtc->orig_buf;
1266         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1267
1268         KTR_LOG(dmcrypt_bio_write_done, bio->bio_buf);
1269
1270         obio = pop_bio(bio);
1271         biodone(obio);
1272 }
1273 /* END OF STRATEGY WRITE SECTION */
1274
1275
1276
1277 /* DUMPING MAGIC */
1278
1279 extern int tsleep_crypto_dump;
1280
1281 static int
1282 dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1283 {
1284         static struct dmtc_dump_helper dump_helper;
1285         dm_target_crypt_config_t *priv;
1286         int id;
1287         static int first_call = 1;
1288
1289         priv = table_en->target_config;
1290
1291         if (first_call) {
1292                 first_call = 0;
1293                 dump_reactivate_cpus();
1294         }
1295
1296         /* Magically enable tsleep */
1297         tsleep_crypto_dump = 1;
1298         id = 0;
1299
1300         /*
1301          * 0 length means flush buffers and return
1302          */
1303         if (length == 0) {
1304                 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1305                         tsleep_crypto_dump = 0;
1306                         return ENXIO;
1307                 }
1308                 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1309                     data, 0, offset, 0);
1310                 tsleep_crypto_dump = 0;
1311                 return 0;
1312         }
1313
1314         bzero(&dump_helper, sizeof(dump_helper));
1315         dump_helper.priv = priv;
1316         dump_helper.data = data;
1317         dump_helper.length = length;
1318         dump_helper.offset = offset +
1319             priv->block_offset * DEV_BSIZE;
1320         dump_helper.ident = &id;
1321         dmtc_crypto_dump_start(priv, &dump_helper);
1322
1323         /*
1324          * Hackery to make stuff appear synchronous. The crypto callback will
1325          * set id to 1 and call wakeup on it. If the request completed
1326          * synchronously, id will be 1 and we won't bother to sleep. If not,
1327          * the crypto request will complete asynchronously and we sleep until
1328          * it's done.
1329          */
1330         if (id == 0)
1331                 tsleep(&dump_helper, 0, "cryptdump", 0);
1332
1333         dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1334             dump_helper.offset);
1335
1336         dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1337             dump_helper.space, 0, dump_helper.offset,
1338             dump_helper.length);
1339
1340         tsleep_crypto_dump = 0;
1341         return 0;
1342 }
1343
1344 static void
1345 dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1346 {
1347         struct cryptodesc *crd;
1348         struct cryptop *crp;
1349         int i, bytes, sectors;
1350         off_t isector;
1351
1352         bytes = dump_helper->length;
1353
1354         isector = dump_helper->offset / DEV_BSIZE;      /* ivgen salt base? */
1355         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1356         dump_helper->sectors = sectors;
1357 #if 0
1358         kprintf("Dump, bytes = %d, "
1359                 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1360 #endif
1361         KKASSERT(dump_helper->length <= 65536);
1362
1363         memcpy(dump_helper->space, dump_helper->data, bytes);
1364
1365         cpu_sfence();
1366
1367         for (i = 0; i < sectors; i++) {
1368                 crp = &dump_helper->crp[i];
1369                 crd = &dump_helper->crd[i];
1370
1371                 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1372
1373                 crp->crp_sid = priv->crypto_sid;
1374                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1375
1376                 crp->crp_opaque = (void *)dump_helper;
1377
1378                 crp->crp_callback = dmtc_crypto_cb_dump_done;
1379                 crp->crp_desc = crd;
1380                 crp->crp_etype = 0;
1381                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1382                                  CRYPTO_F_BATCH;
1383
1384                 crd->crd_alg = priv->crypto_alg;
1385
1386                 crd->crd_skip = 0;
1387                 crd->crd_len = DEV_BSIZE /* XXX */;
1388                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1389                 crd->crd_next = NULL;
1390
1391                 crd->crd_flags |= CRD_F_ENCRYPT;
1392
1393                 /*
1394                  * Note: last argument is used to generate salt(?) and is
1395                  *       a 64 bit value, but the original code passed an
1396                  *       int.  Changing it now will break pre-existing
1397                  *       crypt volumes.
1398                  */
1399                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1400                                     isector + i, crp);
1401         }
1402 }
1403
1404 static int
1405 dmtc_crypto_cb_dump_done(struct cryptop *crp)
1406 {
1407         struct dmtc_dump_helper *dump_helper;
1408         int n;
1409
1410         if (crp->crp_etype == EAGAIN)
1411                 return crypto_dispatch(crp);
1412
1413         dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1414         KKASSERT(dump_helper != NULL);
1415
1416         if (crp->crp_etype != 0) {
1417                 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1418                         "crp_etype = %d\n",
1419                 crp->crp_etype);
1420                 return crp->crp_etype;
1421         }
1422
1423         /*
1424          * On the last chunk of the encryption we return control
1425          */
1426         n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1427
1428         if (n == 1) {
1429                 atomic_add_int(dump_helper->ident, 1);
1430                 wakeup(dump_helper);
1431         }
1432
1433         return 0;
1434 }
1435
1436 static int
1437 dmtc_mod_handler(module_t mod, int type, void *unused)
1438 {
1439         dm_target_t *dmt = NULL;
1440         int err = 0;
1441
1442         switch (type) {
1443         case MOD_LOAD:
1444                 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1445                         dm_target_unbusy(dmt);
1446                         return EEXIST;
1447                 }
1448                 dmt = dm_target_alloc("crypt");
1449                 dmt->version[0] = 1;
1450                 dmt->version[1] = 6;
1451                 dmt->version[2] = 0;
1452                 strlcpy(dmt->name, "crypt", DM_MAX_TYPE_NAME);
1453                 dmt->init = &dm_target_crypt_init;
1454                 dmt->table = &dm_target_crypt_table;
1455                 dmt->strategy = &dm_target_crypt_strategy;
1456                 dmt->destroy = &dm_target_crypt_destroy;
1457                 dmt->upcall = &dm_target_crypt_upcall;
1458                 dmt->dump = &dm_target_crypt_dump;
1459
1460                 err = dm_target_insert(dmt);
1461                 if (!err)
1462                         kprintf("dm_target_crypt: Successfully initialized\n");
1463                 break;
1464
1465         case MOD_UNLOAD:
1466                 err = dm_target_rem("crypt");
1467                 if (err == 0) {
1468                         kprintf("dm_target_crypt: unloaded\n");
1469                 }
1470                 break;
1471
1472         default:
1473                 break;
1474         }
1475
1476         return err;
1477 }
1478
1479 DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);