sys/dev/disk/dm/targets/crypt/dm_target_crypt.c

   1 /*
   2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Alex Hornung <ahornung@gmail.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34
  35 /*
  36  * This file implements initial version of device-mapper crypt target.
  37  */
  38 #include <sys/types.h>
  39 #include <sys/param.h>
  40 #include <sys/endian.h>
  41
  42 #include <sys/bio.h>
  43 #include <sys/buf.h>
  44 #include <sys/globaldata.h>
  45 #include <sys/kerneldump.h>
  46 #include <sys/malloc.h>
  47 #include <sys/mpipe.h>
  48 #include <sys/md5.h>
  49 #include <sys/mutex2.h>
  50 #include <sys/vnode.h>
  51 #include <crypto/sha1.h>
  52 #include <crypto/sha2/sha2.h>
  53 #include <opencrypto/cryptodev.h>
  54 #include <opencrypto/rmd160.h>
  55 #include <machine/cpufunc.h>
  56
  57 #include <sys/ktr.h>
  58
  59 #include <dev/disk/dm/dm.h>
  60 MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
  61
  62 KTR_INFO_MASTER(dmcrypt);
  63
  64 #if !defined(KTR_DMCRYPT)
  65 #define KTR_DMCRYPT     KTR_ALL
  66 #endif
  67
  68 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_dispatch, 0,
  69     "crypto_dispatch(%p)", sizeof(void *));
  70 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypt_strategy, 0,
  71     "crypt_strategy(b_cmd = %d, bp = %p)", sizeof(int) + sizeof(void *));
  72 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_start, 1,
  73     "crypto_write_start(crp = %p, bp = %p, sector = %d/%d)",
  74     sizeof(void *) + sizeof(void *) + sizeof(int) + sizeof(int));
  75 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_write_done, 1,
  76     "crypto_cb_write_done(crp = %p, bp = %p, n = %d)",
  77     sizeof(void *) + sizeof(void *) + sizeof(int));
  78 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_write_done, 1,
  79     "bio_write_done(bp = %p)", sizeof(void *));
  80 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_retry, 1,
  81     "crypto_write_retry(crp = %p)", sizeof(void *));
  82 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_read_done, 2,
  83     "bio_read_done(bp = %p)", sizeof(void *));
  84 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_read_start, 2,
  85     "crypto_read_start(crp = %p, bp = %p, sector = %d/%d)",
  86     sizeof(void *) + sizeof(void *) + sizeof(int) + sizeof(int));
  87 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_read_done, 2,
  88     "crypto_cb_read_done(crp = %p, bp = %p, n = %d)",
  89     sizeof(void *) + sizeof(void *) + sizeof(int));
  90
  91 struct target_crypt_config;
  92
  93 typedef void dispatch_t(void *);
  94 typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
  95     void *);
  96
  97 typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
  98 typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
  99
 100 struct iv_generator {
 101         const char      *name;
 102         ivgen_ctor_t    *ctor;
 103         ivgen_dtor_t    *dtor;
 104         ivgen_t         *gen_iv;
 105 };
 106
 107 struct essiv_ivgen_priv {
 108         struct cryptoini        crypto_session;
 109         struct objcache *crp_crd_cache;
 110         u_int64_t       crypto_sid;
 111         size_t          keyhash_len;
 112         u_int8_t        crypto_keyhash[SHA512_DIGEST_LENGTH];
 113 };
 114
 115 typedef struct target_crypt_config {
 116         size_t  params_len;
 117         dm_pdev_t *pdev;
 118         char    *status_str;
 119         int     crypto_alg;
 120         int     crypto_klen;
 121         u_int8_t        crypto_key[512>>3];
 122
 123         u_int64_t       crypto_sid;
 124         u_int64_t       block_offset;
 125         int64_t         iv_offset;
 126         SHA512_CTX      essivsha512_ctx;
 127
 128         struct cryptoini        crypto_session;
 129
 130         struct iv_generator     *ivgen;
 131         void    *ivgen_priv;
 132
 133         struct malloc_pipe      read_mpipe;
 134         struct malloc_pipe      write_mpipe;
 135 } dm_target_crypt_config_t;
 136
 137 struct dmtc_helper {
 138         dm_target_crypt_config_t *priv;
 139         caddr_t free_addr;
 140         caddr_t orig_buf;
 141         caddr_t data_buf;
 142 };
 143
 144 struct dmtc_dump_helper {
 145         dm_target_crypt_config_t *priv;
 146         void *data;
 147         size_t length;
 148         off_t offset;
 149
 150         int sectors;
 151         int *ident;
 152
 153         struct cryptodesc crd[128];
 154         struct cryptop crp[128];
 155         u_char space[65536];
 156 };
 157
 158 #define DMTC_BUF_SIZE_WRITE \
 159     MAXPHYS + sizeof(struct dmtc_helper) + \
 160     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 161 #define DMTC_BUF_SIZE_READ \
 162     sizeof(struct dmtc_helper) + \
 163     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
 164
 165 static void dmtc_crypto_dispatch(void *arg);
 166 static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
 167                                 struct dmtc_dump_helper *dump_helper);
 168 static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
 169                                 struct bio *bio);
 170 static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
 171                                 struct bio *bio);
 172 static void dmtc_bio_read_done(struct bio *bio);
 173 static void dmtc_bio_write_done(struct bio *bio);
 174 static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
 175 static int dmtc_crypto_cb_read_done(struct cryptop *crp);
 176 static int dmtc_crypto_cb_write_done(struct cryptop *crp);
 177
 178 static ivgen_ctor_t     essiv_ivgen_ctor;
 179 static ivgen_dtor_t     essiv_ivgen_dtor;
 180 static ivgen_t          essiv_ivgen;
 181 static ivgen_t          plain_ivgen;
 182 static ivgen_t          plain64_ivgen;
 183
 184 static struct iv_generator ivgens[] = {
 185         { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
 186             .gen_iv = essiv_ivgen },
 187         { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
 188         { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
 189         { NULL, NULL, NULL, NULL }
 190 };
 191
 192 struct objcache_malloc_args essiv_ivgen_malloc_args = {
 193                 2*sizeof(void *) + (sizeof(struct cryptodesc) +
 194                 sizeof(struct cryptop)), M_DMCRYPT };
 195
 196 static void
 197 dmtc_init_mpipe(struct target_crypt_config *priv)
 198 {
 199         int nmax;
 200
 201         nmax = (physmem*2/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
 202
 203         if (nmax < 2)
 204                 nmax = 2;
 205
 206         kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
 207
 208         mpipe_init(&priv->write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
 209                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 210         mpipe_init(&priv->read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
 211                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
 212 }
 213
 214 static void
 215 dmtc_destroy_mpipe(struct target_crypt_config *priv)
 216 {
 217         mpipe_done(&priv->write_mpipe);
 218         mpipe_done(&priv->read_mpipe);
 219 }
 220
 221 /*
 222  * Overwrite private information (in buf) to avoid leaking it
 223  */
 224 static void
 225 dmtc_crypto_clear(void *buf, size_t len)
 226 {
 227         memset(buf, 0xFF, len);
 228         bzero(buf, len);
 229 }
 230
 231 /*
 232  * ESSIV IV Generator Routines
 233  */
 234 static int
 235 essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
 236 {
 237         struct essiv_ivgen_priv *ivpriv;
 238         u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
 239         unsigned int klen, hashlen;
 240         int error;
 241
 242         klen = (priv->crypto_klen >> 3);
 243
 244         if (iv_hash == NULL)
 245                 return EINVAL;
 246
 247         if (!strcmp(iv_hash, "sha1")) {
 248                 SHA1_CTX ctx;
 249
 250                 hashlen = SHA1_RESULTLEN;
 251                 SHA1Init(&ctx);
 252                 SHA1Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 253                 SHA1Final(crypto_keyhash, &ctx);
 254         } else if (!strcmp(iv_hash, "sha256")) {
 255                 SHA256_CTX ctx;
 256
 257                 hashlen = SHA256_DIGEST_LENGTH;
 258                 SHA256_Init(&ctx);
 259                 SHA256_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 260                 SHA256_Final(crypto_keyhash, &ctx);
 261         } else if (!strcmp(iv_hash, "sha384")) {
 262                 SHA384_CTX ctx;
 263
 264                 hashlen = SHA384_DIGEST_LENGTH;
 265                 SHA384_Init(&ctx);
 266                 SHA384_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 267                 SHA384_Final(crypto_keyhash, &ctx);
 268         } else if (!strcmp(iv_hash, "sha512")) {
 269                 SHA512_CTX ctx;
 270
 271                 hashlen = SHA512_DIGEST_LENGTH;
 272                 SHA512_Init(&ctx);
 273                 SHA512_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 274                 SHA512_Final(crypto_keyhash, &ctx);
 275         } else if (!strcmp(iv_hash, "md5")) {
 276                 MD5_CTX ctx;
 277
 278                 hashlen = MD5_DIGEST_LENGTH;
 279                 MD5Init(&ctx);
 280                 MD5Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 281                 MD5Final(crypto_keyhash, &ctx);
 282         } else if (!strcmp(iv_hash, "rmd160") ||
 283                    !strcmp(iv_hash, "ripemd160")) {
 284                 RMD160_CTX ctx;
 285
 286                 hashlen = 160/8;
 287                 RMD160Init(&ctx);
 288                 RMD160Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
 289                 RMD160Final(crypto_keyhash, &ctx);
 290         } else {
 291                 return EINVAL;
 292         }
 293
 294         /* Convert hashlen to bits */
 295         hashlen <<= 3;
 296
 297         ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
 298             M_WAITOK | M_ZERO);
 299         memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
 300         ivpriv->keyhash_len = sizeof(crypto_keyhash);
 301         dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
 302
 303         ivpriv->crypto_session.cri_alg = priv->crypto_alg;
 304         ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
 305         ivpriv->crypto_session.cri_klen = hashlen;
 306         ivpriv->crypto_session.cri_mlen = 0;
 307         ivpriv->crypto_session.cri_next = NULL;
 308
 309         /*
 310          * XXX: in principle we also need to check if the block size of the
 311          *      cipher is a valid iv size for the block cipher.
 312          */
 313
 314         error = crypto_newsession(&ivpriv->crypto_sid,
 315                                   &ivpriv->crypto_session,
 316                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 317         if (error) {
 318                 kprintf("dm_target_crypt: Error during crypto_newsession "
 319                         "for essiv_ivgen, error = %d\n",
 320                         error);
 321                 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 322                 kfree(ivpriv, M_DMCRYPT);
 323                 return ENOTSUP;
 324         }
 325
 326         ivpriv->crp_crd_cache = objcache_create(
 327             "dmcrypt-essiv-cache", 0, 0,
 328             NULL, NULL, NULL,
 329             objcache_malloc_alloc,
 330             objcache_malloc_free,
 331             &essiv_ivgen_malloc_args );
 332
 333         *p_ivpriv = ivpriv;
 334         return 0;
 335 }
 336
 337 static int
 338 essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
 339 {
 340         struct essiv_ivgen_priv *ivpriv;
 341
 342         ivpriv = (struct essiv_ivgen_priv *)arg;
 343         KKASSERT(ivpriv != NULL);
 344
 345         crypto_freesession(ivpriv->crypto_sid);
 346
 347         objcache_destroy(ivpriv->crp_crd_cache);
 348
 349         dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
 350         kfree(ivpriv, M_DMCRYPT);
 351
 352         return 0;
 353 }
 354
 355 static int
 356 essiv_ivgen_done(struct cryptop *crp)
 357 {
 358         struct essiv_ivgen_priv *ivpriv;
 359         void *free_addr;
 360         void *opaque;
 361
 362
 363         if (crp->crp_etype == EAGAIN)
 364                 return crypto_dispatch(crp);
 365
 366         if (crp->crp_etype != 0) {
 367                 kprintf("dm_target_crypt: essiv_ivgen_done, "
 368                         "crp->crp_etype = %d\n", crp->crp_etype);
 369         }
 370
 371         free_addr = crp->crp_opaque;
 372         /*
 373          * In-memory structure is:
 374          * |  ivpriv  |  opaque  |     crp     |      crd      |
 375          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 376          */
 377         ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
 378         crp->crp_opaque += sizeof(void *);
 379         opaque = *((void **)crp->crp_opaque);
 380
 381         objcache_put(ivpriv->crp_crd_cache, free_addr);
 382         dmtc_crypto_dispatch(opaque);
 383         return 0;
 384 }
 385
 386 static void
 387 essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 388             size_t iv_len, off_t sector, void *opaque)
 389 {
 390         struct essiv_ivgen_priv *ivpriv;
 391         struct cryptodesc *crd;
 392         struct cryptop *crp;
 393         caddr_t space, alloc_addr;
 394         int error;
 395
 396         ivpriv = priv->ivgen_priv;
 397         KKASSERT(ivpriv != NULL);
 398
 399         /*
 400          * In-memory structure is:
 401          * |  ivpriv  |  opaque  |     crp     |      crd      |
 402          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
 403          */
 404         alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
 405         *((struct essiv_ivgen_priv **)space) = ivpriv;
 406         space += sizeof(void *);
 407         *((void **)space) = opaque;
 408         space += sizeof(void *);
 409         crp = (struct cryptop *)space;
 410         space += sizeof(struct cryptop);
 411         crd = (struct cryptodesc *)space;
 412
 413         bzero(iv, iv_len);
 414         bzero(crd, sizeof(struct cryptodesc));
 415         bzero(crp, sizeof(struct cryptop));
 416         *((off_t *)iv) = htole64(sector + priv->iv_offset);
 417         crp->crp_buf = (caddr_t)iv;
 418
 419         crp->crp_sid = ivpriv->crypto_sid;
 420         crp->crp_ilen = crp->crp_olen = iv_len;
 421
 422         crp->crp_opaque = alloc_addr;
 423
 424         crp->crp_callback = essiv_ivgen_done;
 425
 426         crp->crp_desc = crd;
 427         crp->crp_etype = 0;
 428         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
 429
 430         crd->crd_alg = priv->crypto_alg;
 431 #if 0
 432         crd->crd_key = (caddr_t)priv->crypto_keyhash;
 433         crd->crd_klen = priv->crypto_klen;
 434 #endif
 435
 436         bzero(crd->crd_iv, sizeof(crd->crd_iv));
 437
 438         crd->crd_skip = 0;
 439         crd->crd_len = iv_len;
 440         crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
 441         crd->crd_flags |= CRD_F_ENCRYPT;
 442         crd->crd_next = NULL;
 443
 444         error = crypto_dispatch(crp);
 445         if (error)
 446                 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
 447 }
 448
 449
 450 static void
 451 plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 452             size_t iv_len, off_t sector, void *opaque)
 453 {
 454         bzero(iv, iv_len);
 455         *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
 456         dmtc_crypto_dispatch(opaque);
 457 }
 458
 459 static void
 460 plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 461     size_t iv_len, off_t sector, void *opaque)
 462 {
 463         bzero(iv, iv_len);
 464         *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
 465         dmtc_crypto_dispatch(opaque);
 466 }
 467
 468 #if 0
 469 static void
 470 geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
 471            size_t iv_len, off_t sector, void *opaque)
 472 {
 473
 474         SHA512_CTX      ctx512;
 475         u_int8_t        md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
 476
 477         memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
 478         SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
 479         SHA512_Final(md, &ctx512);
 480
 481         memcpy(iv, md, iv_len);
 482         dmtc_crypto_dispatch(opaque);
 483 }
 484 #endif
 485
 486 /*
 487  * Init function called from dm_table_load_ioctl.
 488  * cryptsetup actually passes us this:
 489  * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
 490  */
 491 static int
 492 hex2key(char *hex, size_t key_len, u_int8_t *key)
 493 {
 494         char hex_buf[3];
 495         size_t key_idx;
 496
 497         hex_buf[2] = 0;
 498         for (key_idx = 0; key_idx < key_len; ++key_idx) {
 499                 hex_buf[0] = *hex++;
 500                 hex_buf[1] = *hex++;
 501                 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
 502         }
 503         hex_buf[0] = 0;
 504         hex_buf[1] = 0;
 505
 506         return 0;
 507 }
 508
 509 static int
 510 dm_target_crypt_init(dm_dev_t * dmv, void **target_config, char *params)
 511 {
 512         dm_target_crypt_config_t *priv;
 513         size_t len;
 514         char **ap, *args[5];
 515         char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
 516         char *status_str;
 517         int i, argc, klen, error;
 518         uint64_t iv_offset, block_offset;
 519
 520         if (params == NULL)
 521                 return EINVAL;
 522
 523         len = strlen(params) + 1;
 524         argc = 0;
 525
 526         status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
 527         /*
 528          * Parse a string, containing tokens delimited by white space,
 529          * into an argument vector
 530          */
 531         for (ap = args; ap < &args[5] &&
 532             (*ap = strsep(&params, " \t")) != NULL;) {
 533                 if (**ap != '\0') {
 534                         argc++;
 535                         ap++;
 536                 }
 537         }
 538
 539         if (argc != 5) {
 540                 kprintf("dm_target_crypt: not enough arguments, "
 541                         "need exactly 5\n");
 542                 kfree(status_str, M_DMCRYPT);
 543                 return ENOMEM; /* XXX */
 544         }
 545
 546         crypto_alg = strsep(&args[0], "-");
 547         crypto_mode = strsep(&args[0], "-");
 548         iv_opt = strsep(&args[0], "-");
 549         iv_mode = strsep(&iv_opt, ":");
 550         key = args[1];
 551         iv_offset = strtouq(args[2], NULL, 0);
 552         dev = args[3];
 553         block_offset = strtouq(args[4], NULL, 0);
 554         /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
 555         klen = strlen(key) << 2;
 556
 557 #if 0
 558         kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
 559                 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
 560                 "block_offset=%ju\n",
 561                 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
 562                 block_offset);
 563 #endif
 564
 565         priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
 566         if (priv == NULL) {
 567                 kprintf("dm_target_crypt: could not allocate memory\n");
 568                 kfree(status_str, M_DMCRYPT);
 569                 return ENOMEM;
 570         }
 571
 572         /* Insert dmp to global pdev list */
 573         if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
 574                 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
 575                 kfree(status_str, M_DMCRYPT);
 576                 return ENOENT;
 577         }
 578
 579         /*
 580          * This code checks for valid combinations of algorithm and mode.
 581          * Currently supported options are:
 582          *
 583          * *-cbc
 584          * aes-xts
 585          * twofish-xts
 586          * serpent-xts
 587          */
 588         if ((strcmp(crypto_mode, "cbc") != 0) &&
 589             !((strcmp(crypto_mode, "xts") == 0) &&
 590             ((strcmp(crypto_alg, "aes") == 0) ||
 591             (strcmp(crypto_alg, "twofish") == 0) ||
 592             (strcmp(crypto_alg, "serpent") == 0))))
 593         {
 594                 kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
 595                     " aes-xts, twofish-xts and serpent-xts, "
 596                     "invalid mode '%s-%s'\n",
 597                     crypto_alg, crypto_mode);
 598                 goto notsup;
 599         }
 600
 601         if (!strcmp(crypto_alg, "aes")) {
 602                 if (!strcmp(crypto_mode, "xts")) {
 603                         priv->crypto_alg = CRYPTO_AES_XTS;
 604                         if (klen != 256 && klen != 512)
 605                                 goto notsup;
 606                 } else if (!strcmp(crypto_mode, "cbc")) {
 607                         priv->crypto_alg = CRYPTO_AES_CBC;
 608                         if (klen != 128 && klen != 192 && klen != 256)
 609                                 goto notsup;
 610                 } else {
 611                         goto notsup;
 612                 }
 613                 priv->crypto_klen = klen;
 614         } else if (!strcmp(crypto_alg, "twofish")) {
 615                 if (!strcmp(crypto_mode, "xts")) {
 616                         priv->crypto_alg = CRYPTO_TWOFISH_XTS;
 617                         if (klen != 256 && klen != 512)
 618                                 goto notsup;
 619                 } else if (!strcmp(crypto_mode, "cbc")) {
 620                         priv->crypto_alg = CRYPTO_TWOFISH_CBC;
 621                         if (klen != 128 && klen != 192 && klen != 256)
 622                                 goto notsup;
 623                 } else {
 624                         goto notsup;
 625                 }
 626                 priv->crypto_klen = klen;
 627         } else if (!strcmp(crypto_alg, "serpent")) {
 628                 if (!strcmp(crypto_mode, "xts")) {
 629                         priv->crypto_alg = CRYPTO_SERPENT_XTS;
 630                         if (klen != 256 && klen != 512)
 631                                 goto notsup;
 632                 } else if (!strcmp(crypto_mode, "cbc")) {
 633                         priv->crypto_alg = CRYPTO_SERPENT_CBC;
 634                         if (klen != 128 && klen != 192 && klen != 256)
 635                                 goto notsup;
 636                 } else {
 637                         goto notsup;
 638                 }
 639                 priv->crypto_klen = klen;
 640         } else if (!strcmp(crypto_alg, "blowfish")) {
 641                 priv->crypto_alg = CRYPTO_BLF_CBC;
 642                 if (klen < 128 || klen > 448 || (klen % 8) != 0)
 643                         goto notsup;
 644                 priv->crypto_klen = klen;
 645         } else if (!strcmp(crypto_alg, "3des") ||
 646                    !strncmp(crypto_alg, "des3", 4)) {
 647                 priv->crypto_alg = CRYPTO_3DES_CBC;
 648                 if (klen != 168)
 649                         goto notsup;
 650                 priv->crypto_klen = 168;
 651         } else if (!strcmp(crypto_alg, "camellia")) {
 652                 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
 653                 if (klen != 128 && klen != 192 && klen != 256)
 654                         goto notsup;
 655                 priv->crypto_klen = klen;
 656         } else if (!strcmp(crypto_alg, "skipjack")) {
 657                 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
 658                 if (klen != 80)
 659                         goto notsup;
 660                 priv->crypto_klen = 80;
 661         } else if (!strcmp(crypto_alg, "cast5")) {
 662                 priv->crypto_alg = CRYPTO_CAST_CBC;
 663                 if (klen != 128)
 664                         goto notsup;
 665                 priv->crypto_klen = 128;
 666         } else if (!strcmp(crypto_alg, "null")) {
 667                 priv->crypto_alg = CRYPTO_NULL_CBC;
 668                 if (klen != 128)
 669                         goto notsup;
 670                 priv->crypto_klen = 128;
 671         } else {
 672                 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
 673                         crypto_alg);
 674                 goto notsup;
 675         }
 676
 677         /* Save length of param string */
 678         priv->params_len = len;
 679         priv->block_offset = block_offset;
 680         priv->iv_offset = iv_offset - block_offset;
 681
 682         *target_config = priv;
 683
 684         dmv->dev_type = DM_CRYPTO_DEV;
 685
 686         error = hex2key(key, priv->crypto_klen >> 3,
 687                         (u_int8_t *)priv->crypto_key);
 688
 689         if (error) {
 690                 kprintf("dm_target_crypt: hex2key failed, "
 691                         "invalid key format\n");
 692                 goto notsup;
 693         }
 694
 695         /* Handle cmd */
 696         for(i = 0; ivgens[i].name != NULL; i++) {
 697                 if (!strcmp(iv_mode, ivgens[i].name))
 698                         break;
 699         }
 700
 701         if (ivgens[i].name == NULL) {
 702                 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
 703                         iv_mode);
 704                 goto notsup;
 705         }
 706
 707         /* Call our ivgen constructor */
 708         if (ivgens[i].ctor != NULL) {
 709                 error = ivgens[i].ctor(priv, iv_opt,
 710                     &priv->ivgen_priv);
 711                 if (error) {
 712                         kprintf("dm_target_crypt: ctor for '%s' failed\n",
 713                             ivgens[i].name);
 714                         goto notsup;
 715                 }
 716         }
 717
 718         priv->ivgen = &ivgens[i];
 719
 720         priv->crypto_session.cri_alg = priv->crypto_alg;
 721         priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
 722         priv->crypto_session.cri_klen = priv->crypto_klen;
 723         priv->crypto_session.cri_mlen = 0;
 724         priv->crypto_session.cri_next = NULL;
 725
 726         error = crypto_newsession(&priv->crypto_sid,
 727                                   &priv->crypto_session,
 728                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
 729         if (error) {
 730                 kprintf("dm_target_crypt: Error during crypto_newsession, "
 731                         "error = %d\n",
 732                         error);
 733                 goto notsup;
 734         }
 735
 736         memset(key, '0', strlen(key));
 737         if (iv_opt) {
 738                 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
 739                     crypto_alg, crypto_mode, iv_mode, iv_opt,
 740                     key, iv_offset, dev, block_offset);
 741         } else {
 742                 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
 743                     crypto_alg, crypto_mode, iv_mode,
 744                     key, iv_offset, dev, block_offset);
 745         }
 746         priv->status_str = status_str;
 747
 748         /* Initialize mpipes */
 749         dmtc_init_mpipe(priv);
 750
 751         return 0;
 752
 753 notsup:
 754         kprintf("dm_target_crypt: ENOTSUP\n");
 755         kfree(status_str, M_DMCRYPT);
 756         return ENOTSUP;
 757 }
 758
 759 /* Status routine called to get params string. */
 760 static char *
 761 dm_target_crypt_status(void *target_config)
 762 {
 763         dm_target_crypt_config_t *priv;
 764         char *params;
 765
 766         priv = target_config;
 767
 768         /* caller expects use of M_DM */
 769         params = kmalloc(DM_MAX_PARAMS_SIZE, M_DM, M_WAITOK);
 770
 771         ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
 772             priv->status_str);
 773
 774         return params;
 775 }
 776
 777 static int
 778 dm_target_crypt_destroy(dm_table_entry_t * table_en)
 779 {
 780         dm_target_crypt_config_t *priv;
 781
 782         /*
 783          * Disconnect the crypt config before unbusying the target.
 784          */
 785         priv = table_en->target_config;
 786         if (priv == NULL)
 787                 return 0;
 788         table_en->target_config = NULL;
 789         dm_pdev_decr(priv->pdev);
 790
 791         dm_target_unbusy(table_en->target);
 792
 793         /*
 794          * Clean up the crypt config
 795          *
 796          * Overwrite the private information before freeing memory to
 797          * avoid leaking it.
 798          */
 799         if (priv->status_str) {
 800                 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
 801                 kfree(priv->status_str, M_DMCRYPT);
 802                 crypto_freesession(priv->crypto_sid);
 803         }
 804
 805         if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
 806                 priv->ivgen->dtor(priv, priv->ivgen_priv);
 807         }
 808
 809         /* Destroy mpipes */
 810         dmtc_destroy_mpipe(priv);
 811
 812         dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
 813         kfree(priv, M_DMCRYPT);
 814
 815         return 0;
 816 }
 817
 818 static int
 819 dm_target_crypt_deps(dm_table_entry_t * table_en, prop_array_t prop_array)
 820 {
 821         dm_target_crypt_config_t *priv;
 822         struct vattr va;
 823
 824         int error;
 825
 826         if (table_en->target_config == NULL)
 827                 return ENOENT;
 828
 829         priv = table_en->target_config;
 830
 831         if ((error = VOP_GETATTR(priv->pdev->pdev_vnode, &va)) != 0)
 832                 return error;
 833
 834         prop_array_add_uint64(prop_array,
 835                               (uint64_t)makeudev(va.va_rmajor, va.va_rminor));
 836
 837         return 0;
 838 }
 839
 840 /* Unsupported for this target. */
 841 static int
 842 dm_target_crypt_upcall(dm_table_entry_t * table_en, struct buf * bp)
 843 {
 844         return 0;
 845 }
 846
 847 /************************************************************************
 848  *                      STRATEGY SUPPORT FUNCTIONS                      *
 849  ************************************************************************
 850  *
 851  * READ PATH:   doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
 852  * WRITE PATH:  crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
 853  */
 854
 855 /*
 856  * Wrapper around crypto_dispatch() to match dispatch_t type
 857  */
 858 static void
 859 dmtc_crypto_dispatch(void *arg)
 860 {
 861         struct cryptop *crp;
 862
 863         crp = (struct cryptop *)arg;
 864         KKASSERT(crp != NULL);
 865         KTR_LOG(dmcrypt_crypto_dispatch, crp);
 866         crypto_dispatch(crp);
 867 }
 868
 869 /*
 870  * Start IO operation, called from dmstrategy routine.
 871  */
 872 static int
 873 dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
 874 {
 875         struct bio *bio;
 876
 877         dm_target_crypt_config_t *priv;
 878         priv = table_en->target_config;
 879
 880         /* Get rid of stuff we can't really handle */
 881         if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
 882                 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
 883                         kprintf("dm_target_crypt_strategy: can't really "
 884                                 "handle bp->b_bcount = %d\n",
 885                                 bp->b_bcount);
 886                         bp->b_error = EINVAL;
 887                         bp->b_flags |= B_ERROR | B_INVAL;
 888                         biodone(&bp->b_bio1);
 889                         return 0;
 890                 }
 891         }
 892
 893         KTR_LOG(dmcrypt_crypt_strategy, bp->b_cmd, bp);
 894
 895         switch (bp->b_cmd) {
 896         case BUF_CMD_READ:
 897                 bio = push_bio(&bp->b_bio1);
 898                 bio->bio_offset = bp->b_bio1.bio_offset +
 899                                   priv->block_offset * DEV_BSIZE;
 900                 bio->bio_caller_info1.ptr = priv;
 901                 bio->bio_done = dmtc_bio_read_done;
 902                 vn_strategy(priv->pdev->pdev_vnode, bio);
 903                 break;
 904         case BUF_CMD_WRITE:
 905                 bio = push_bio(&bp->b_bio1);
 906                 bio->bio_offset = bp->b_bio1.bio_offset +
 907                                   priv->block_offset * DEV_BSIZE;
 908                 bio->bio_caller_info1.ptr = priv;
 909                 dmtc_crypto_write_start(priv, bio);
 910                 break;
 911         default:
 912                 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
 913                 break;
 914         }
 915         return 0;
 916 }
 917
 918 /*
 919  * STRATEGY READ PATH PART 1/3 (after read BIO completes)
 920  */
 921 static void
 922 dmtc_bio_read_done(struct bio *bio)
 923 {
 924         struct bio *obio;
 925
 926         dm_target_crypt_config_t *priv;
 927
 928         KTR_LOG(dmcrypt_bio_read_done, bio->bio_buf);
 929
 930         /*
 931          * If a read error occurs we shortcut the operation, otherwise
 932          * go on to stage 2.
 933          */
 934         if (bio->bio_buf->b_flags & B_ERROR) {
 935                 obio = pop_bio(bio);
 936                 biodone(obio);
 937         } else {
 938                 priv = bio->bio_caller_info1.ptr;
 939                 dmtc_crypto_read_start(priv, bio);
 940         }
 941 }
 942
 943 /*
 944  * STRATEGY READ PATH PART 2/3
 945  */
 946 static void
 947 dmtc_crypto_read_retry(void *arg1, void *arg2)
 948 {
 949         dm_target_crypt_config_t *priv = arg1;
 950         struct bio *bio = arg2;
 951
 952         dmtc_crypto_read_start(priv, bio);
 953 }
 954
 955 static void
 956 dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
 957 {
 958         struct dmtc_helper *dmtc;
 959         struct cryptodesc *crd;
 960         struct cryptop *crp;
 961         struct cryptoini *cri;
 962         int i, bytes, sectors, sz;
 963         off_t isector;
 964         u_char *ptr, *space;
 965
 966         cri = &priv->crypto_session;
 967
 968         /*
 969          * Note: b_resid no good after read I/O, it will be 0, use
 970          *       b_bcount.
 971          */
 972         bytes = bio->bio_buf->b_bcount;
 973         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
 974         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
 975         sz = sectors * (sizeof(*crp) + sizeof(*crd));
 976
 977         /*
 978          * For reads with bogus page we can't decrypt in place as stuff
 979          * can get ripped out from under us.
 980          *
 981          * XXX actually it looks like we can, and in any case the initial
 982          * read already completed and threw crypted data into the buffer
 983          * cache buffer.  Disable for now.
 984          */
 985         space = mpipe_alloc_callback(&priv->read_mpipe,
 986                                      dmtc_crypto_read_retry, priv, bio);
 987         if (space == NULL)
 988                 return;
 989
 990         dmtc = (struct dmtc_helper *)space;
 991         dmtc->free_addr = space;
 992         space += sizeof(struct dmtc_helper);
 993         dmtc->orig_buf = NULL;
 994         dmtc->data_buf = bio->bio_buf->b_data;
 995         dmtc->priv = priv;
 996         bio->bio_caller_info2.ptr = dmtc;
 997         bio->bio_buf->b_error = 0;
 998
 999         /*
1000          * Load crypto descriptors (crp/crd loop)
1001          */
1002         bzero(space, sz);
1003         ptr = space;
1004         bio->bio_caller_info3.value = sectors;
1005         cpu_sfence();
1006 #if 0
1007         kprintf("Read, bytes = %d (b_bcount), "
1008                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1009                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1010 #endif
1011         for (i = 0; i < sectors; i++) {
1012                 crp = (struct cryptop *)ptr;
1013                 ptr += sizeof(*crp);
1014                 crd = (struct cryptodesc *)ptr;
1015                 ptr += sizeof (*crd);
1016
1017                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1018
1019                 crp->crp_sid = priv->crypto_sid;
1020                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1021
1022                 crp->crp_opaque = (void *)bio;
1023
1024                 crp->crp_callback = dmtc_crypto_cb_read_done;
1025                 crp->crp_desc = crd;
1026                 crp->crp_etype = 0;
1027                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1028                                  CRYPTO_F_BATCH;
1029
1030                 crd->crd_alg = priv->crypto_alg;
1031 #if 0
1032                 crd->crd_key = (caddr_t)priv->crypto_key;
1033                 crd->crd_klen = priv->crypto_klen;
1034 #endif
1035
1036                 crd->crd_skip = 0;
1037                 crd->crd_len = DEV_BSIZE /* XXX */;
1038                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1039                 crd->crd_next = NULL;
1040
1041                 crd->crd_flags &= ~CRD_F_ENCRYPT;
1042
1043                 KTR_LOG(dmcrypt_crypto_read_start, crp, bio->bio_buf, i,
1044                     sectors);
1045
1046                 /*
1047                  * Note: last argument is used to generate salt(?) and is
1048                  *       a 64 bit value, but the original code passed an
1049                  *       int.  Changing it now will break pre-existing
1050                  *       crypt volumes.
1051                  */
1052                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1053                                     isector + i, crp);
1054         }
1055 }
1056
1057 /*
1058  * STRATEGY READ PATH PART 3/3
1059  */
1060 static int
1061 dmtc_crypto_cb_read_done(struct cryptop *crp)
1062 {
1063         struct dmtc_helper *dmtc;
1064         struct bio *bio, *obio;
1065         int n;
1066
1067         if (crp->crp_etype == EAGAIN)
1068                 return crypto_dispatch(crp);
1069
1070         bio = (struct bio *)crp->crp_opaque;
1071         KKASSERT(bio != NULL);
1072
1073         /*
1074          * Cumulative error
1075          */
1076         if (crp->crp_etype) {
1077                 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1078                         "crp_etype = %d\n",
1079                         crp->crp_etype);
1080                 bio->bio_buf->b_error = crp->crp_etype;
1081         }
1082
1083         /*
1084          * On the last chunk of the decryption we do any required copybacks
1085          * and complete the I/O.
1086          */
1087         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1088 #if 0
1089         kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1090 #endif
1091
1092         KTR_LOG(dmcrypt_crypto_cb_read_done, crp, bio->bio_buf, n);
1093
1094         if (n == 1) {
1095                 /*
1096                  * For the B_HASBOGUS case we didn't decrypt in place,
1097                  * so we need to copy stuff back into the buf.
1098                  *
1099                  * (disabled for now).
1100                  */
1101                 dmtc = bio->bio_caller_info2.ptr;
1102                 if (bio->bio_buf->b_error) {
1103                         bio->bio_buf->b_flags |= B_ERROR;
1104                 }
1105 #if 0
1106                 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
1107                         memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1108                                bio->bio_buf->b_bcount);
1109                 }
1110 #endif
1111                 mpipe_free(&dmtc->priv->read_mpipe, dmtc->free_addr);
1112                 obio = pop_bio(bio);
1113                 biodone(obio);
1114         }
1115         return 0;
1116 }
1117 /* END OF STRATEGY READ SECTION */
1118
1119 /*
1120  * STRATEGY WRITE PATH PART 1/3
1121  */
1122
1123 static void
1124 dmtc_crypto_write_retry(void *arg1, void *arg2)
1125 {
1126         dm_target_crypt_config_t *priv = arg1;
1127         struct bio *bio = arg2;
1128
1129         KTR_LOG(dmcrypt_crypto_write_retry, bio->bio_buf);
1130
1131         dmtc_crypto_write_start(priv, bio);
1132 }
1133
1134 static void
1135 dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1136 {
1137         struct dmtc_helper *dmtc;
1138         struct cryptodesc *crd;
1139         struct cryptop *crp;
1140         struct cryptoini *cri;
1141         int i, bytes, sectors, sz;
1142         off_t isector;
1143         u_char *ptr, *space;
1144
1145         cri = &priv->crypto_session;
1146
1147         /*
1148          * Use b_bcount for consistency
1149          */
1150         bytes = bio->bio_buf->b_bcount;
1151
1152         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
1153         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1154         sz = sectors * (sizeof(*crp) + sizeof(*crd));
1155
1156         /*
1157          * For writes and reads with bogus page don't decrypt in place.
1158          */
1159         space = mpipe_alloc_callback(&priv->write_mpipe,
1160                                      dmtc_crypto_write_retry, priv, bio);
1161         if (space == NULL)
1162                 return;
1163
1164         dmtc = (struct dmtc_helper *)space;
1165         dmtc->free_addr = space;
1166         space += sizeof(struct dmtc_helper);
1167         memcpy(space + sz, bio->bio_buf->b_data, bytes);
1168
1169         bio->bio_caller_info2.ptr = dmtc;
1170         bio->bio_buf->b_error = 0;
1171
1172         dmtc->orig_buf = bio->bio_buf->b_data;
1173         dmtc->data_buf = space + sz;
1174         dmtc->priv = priv;
1175
1176         /*
1177          * Load crypto descriptors (crp/crd loop)
1178          */
1179         bzero(space, sz);
1180         ptr = space;
1181         bio->bio_caller_info3.value = sectors;
1182         cpu_sfence();
1183 #if 0
1184         kprintf("Write, bytes = %d (b_bcount), "
1185                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1186                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1187 #endif
1188         for (i = 0; i < sectors; i++) {
1189                 crp = (struct cryptop *)ptr;
1190                 ptr += sizeof(*crp);
1191                 crd = (struct cryptodesc *)ptr;
1192                 ptr += sizeof (*crd);
1193
1194                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1195
1196                 crp->crp_sid = priv->crypto_sid;
1197                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1198
1199                 crp->crp_opaque = (void *)bio;
1200
1201                 crp->crp_callback = dmtc_crypto_cb_write_done;
1202                 crp->crp_desc = crd;
1203                 crp->crp_etype = 0;
1204                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1205                                  CRYPTO_F_BATCH;
1206
1207                 crd->crd_alg = priv->crypto_alg;
1208 #if 0
1209                 crd->crd_key = (caddr_t)priv->crypto_key;
1210                 crd->crd_klen = priv->crypto_klen;
1211 #endif
1212
1213                 crd->crd_skip = 0;
1214                 crd->crd_len = DEV_BSIZE /* XXX */;
1215                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1216                 crd->crd_next = NULL;
1217
1218                 crd->crd_flags |= CRD_F_ENCRYPT;
1219
1220                 /*
1221                  * Note: last argument is used to generate salt(?) and is
1222                  *       a 64 bit value, but the original code passed an
1223                  *       int.  Changing it now will break pre-existing
1224                  *       crypt volumes.
1225                  */
1226
1227                 KTR_LOG(dmcrypt_crypto_write_start, crp, bio->bio_buf,
1228                     i, sectors);
1229
1230                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1231                                     isector + i, crp);
1232         }
1233 }
1234
1235 /*
1236  * STRATEGY WRITE PATH PART 2/3
1237  */
1238 static int
1239 dmtc_crypto_cb_write_done(struct cryptop *crp)
1240 {
1241         struct dmtc_helper *dmtc;
1242         dm_target_crypt_config_t *priv;
1243         struct bio *bio, *obio;
1244         int n;
1245
1246         if (crp->crp_etype == EAGAIN)
1247                 return crypto_dispatch(crp);
1248
1249         bio = (struct bio *)crp->crp_opaque;
1250         KKASSERT(bio != NULL);
1251
1252         /*
1253          * Cumulative error
1254          */
1255         if (crp->crp_etype != 0) {
1256                 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1257                         "crp_etype = %d\n",
1258                 crp->crp_etype);
1259                 bio->bio_buf->b_error = crp->crp_etype;
1260         }
1261
1262         /*
1263          * On the last chunk of the encryption we issue the write
1264          */
1265         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1266 #if 0
1267         kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1268 #endif
1269
1270         KTR_LOG(dmcrypt_crypto_cb_write_done, crp, bio->bio_buf, n);
1271
1272         if (n == 1) {
1273                 dmtc = bio->bio_caller_info2.ptr;
1274                 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1275
1276                 if (bio->bio_buf->b_error) {
1277                         bio->bio_buf->b_flags |= B_ERROR;
1278                         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1279                         obio = pop_bio(bio);
1280                         biodone(obio);
1281                 } else {
1282                         dmtc->orig_buf = bio->bio_buf->b_data;
1283                         bio->bio_buf->b_data = dmtc->data_buf;
1284                         bio->bio_done = dmtc_bio_write_done;
1285                         vn_strategy(priv->pdev->pdev_vnode, bio);
1286                 }
1287         }
1288         return 0;
1289 }
1290
1291 /*
1292  * STRATEGY WRITE PATH PART 3/3
1293  */
1294 static void
1295 dmtc_bio_write_done(struct bio *bio)
1296 {
1297         struct dmtc_helper *dmtc;
1298         struct bio *obio;
1299
1300         dmtc = bio->bio_caller_info2.ptr;
1301         bio->bio_buf->b_data = dmtc->orig_buf;
1302         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1303
1304         KTR_LOG(dmcrypt_bio_write_done, bio->bio_buf);
1305
1306         obio = pop_bio(bio);
1307         biodone(obio);
1308 }
1309 /* END OF STRATEGY WRITE SECTION */
1310
1311
1312
1313 /* DUMPING MAGIC */
1314
1315 extern int tsleep_crypto_dump;
1316
1317 static int
1318 dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1319 {
1320         static struct dmtc_dump_helper dump_helper;
1321         dm_target_crypt_config_t *priv;
1322         int id;
1323         static int first_call = 1;
1324
1325         priv = table_en->target_config;
1326
1327         if (first_call) {
1328                 first_call = 0;
1329                 dump_reactivate_cpus();
1330         }
1331
1332         /* Magically enable tsleep */
1333         tsleep_crypto_dump = 1;
1334         id = 0;
1335
1336         /*
1337          * 0 length means flush buffers and return
1338          */
1339         if (length == 0) {
1340                 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1341                         tsleep_crypto_dump = 0;
1342                         return ENXIO;
1343                 }
1344                 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1345                     data, 0, offset, 0);
1346                 tsleep_crypto_dump = 0;
1347                 return 0;
1348         }
1349
1350         bzero(&dump_helper, sizeof(dump_helper));
1351         dump_helper.priv = priv;
1352         dump_helper.data = data;
1353         dump_helper.length = length;
1354         dump_helper.offset = offset +
1355             priv->block_offset * DEV_BSIZE;
1356         dump_helper.ident = &id;
1357         dmtc_crypto_dump_start(priv, &dump_helper);
1358
1359         /*
1360          * Hackery to make stuff appear synchronous. The crypto callback will
1361          * set id to 1 and call wakeup on it. If the request completed
1362          * synchronously, id will be 1 and we won't bother to sleep. If not,
1363          * the crypto request will complete asynchronously and we sleep until
1364          * it's done.
1365          */
1366         if (id == 0)
1367                 tsleep(&dump_helper, 0, "cryptdump", 0);
1368
1369         dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1370             dump_helper.offset);
1371
1372         dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1373             dump_helper.space, 0, dump_helper.offset,
1374             dump_helper.length);
1375
1376         tsleep_crypto_dump = 0;
1377         return 0;
1378 }
1379
1380 static void
1381 dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1382 {
1383         struct cryptodesc *crd;
1384         struct cryptop *crp;
1385         struct cryptoini *cri;
1386         int i, bytes, sectors;
1387         off_t isector;
1388
1389         cri = &priv->crypto_session;
1390
1391         bytes = dump_helper->length;
1392
1393         isector = dump_helper->offset / DEV_BSIZE;      /* ivgen salt base? */
1394         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1395         dump_helper->sectors = sectors;
1396 #if 0
1397         kprintf("Dump, bytes = %d, "
1398                 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1399 #endif
1400         KKASSERT(dump_helper->length <= 65536);
1401
1402         memcpy(dump_helper->space, dump_helper->data, bytes);
1403
1404         cpu_sfence();
1405
1406         for (i = 0; i < sectors; i++) {
1407                 crp = &dump_helper->crp[i];
1408                 crd = &dump_helper->crd[i];
1409
1410                 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1411
1412                 crp->crp_sid = priv->crypto_sid;
1413                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1414
1415                 crp->crp_opaque = (void *)dump_helper;
1416
1417                 crp->crp_callback = dmtc_crypto_cb_dump_done;
1418                 crp->crp_desc = crd;
1419                 crp->crp_etype = 0;
1420                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1421                                  CRYPTO_F_BATCH;
1422
1423                 crd->crd_alg = priv->crypto_alg;
1424
1425                 crd->crd_skip = 0;
1426                 crd->crd_len = DEV_BSIZE /* XXX */;
1427                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1428                 crd->crd_next = NULL;
1429
1430                 crd->crd_flags |= CRD_F_ENCRYPT;
1431
1432                 /*
1433                  * Note: last argument is used to generate salt(?) and is
1434                  *       a 64 bit value, but the original code passed an
1435                  *       int.  Changing it now will break pre-existing
1436                  *       crypt volumes.
1437                  */
1438                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1439                                     isector + i, crp);
1440         }
1441 }
1442
1443 static int
1444 dmtc_crypto_cb_dump_done(struct cryptop *crp)
1445 {
1446         struct dmtc_dump_helper *dump_helper;
1447         dm_target_crypt_config_t *priv;
1448         int n;
1449
1450         if (crp->crp_etype == EAGAIN)
1451                 return crypto_dispatch(crp);
1452
1453         dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1454         KKASSERT(dump_helper != NULL);
1455
1456         if (crp->crp_etype != 0) {
1457                 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1458                         "crp_etype = %d\n",
1459                 crp->crp_etype);
1460                 return crp->crp_etype;
1461         }
1462
1463         /*
1464          * On the last chunk of the encryption we return control
1465          */
1466         n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1467
1468         if (n == 1) {
1469                 priv = (dm_target_crypt_config_t *)dump_helper->priv;
1470                 atomic_add_int(dump_helper->ident, 1);
1471                 wakeup(dump_helper);
1472         }
1473
1474         return 0;
1475 }
1476
1477 static int
1478 dmtc_mod_handler(module_t mod, int type, void *unused)
1479 {
1480         dm_target_t *dmt = NULL;
1481         int err = 0;
1482
1483         switch (type) {
1484         case MOD_LOAD:
1485                 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1486                         dm_target_unbusy(dmt);
1487                         return EEXIST;
1488                 }
1489                 dmt = dm_target_alloc("crypt");
1490                 dmt->version[0] = 1;
1491                 dmt->version[1] = 6;
1492                 dmt->version[2] = 0;
1493                 strlcpy(dmt->name, "crypt", DM_MAX_TYPE_NAME);
1494                 dmt->init = &dm_target_crypt_init;
1495                 dmt->status = &dm_target_crypt_status;
1496                 dmt->strategy = &dm_target_crypt_strategy;
1497                 dmt->deps = &dm_target_crypt_deps;
1498                 dmt->destroy = &dm_target_crypt_destroy;
1499                 dmt->upcall = &dm_target_crypt_upcall;
1500                 dmt->dump = &dm_target_crypt_dump;
1501
1502                 err = dm_target_insert(dmt);
1503                 if (!err)
1504                         kprintf("dm_target_crypt: Successfully initialized\n");
1505                 break;
1506
1507         case MOD_UNLOAD:
1508                 err = dm_target_rem("crypt");
1509                 if (err == 0) {
1510                         kprintf("dm_target_crypt: unloaded\n");
1511                 }
1512                 break;
1513
1514         default:
1515                 break;
1516         }
1517
1518         return err;
1519 }
1520
1521 DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);