aa489eacf71835a018c4a048d964d28c4e261d42
[dragonfly.git] / sys / dev / disk / dm / targets / crypt / dm_target_crypt.c
1 /*
2  * Copyright (c) 2010 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Alex Hornung <ahornung@gmail.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  */
34
35 /*
36  * This file implements initial version of device-mapper crypt target.
37  */
38 #include <sys/types.h>
39 #include <sys/param.h>
40 #include <sys/endian.h>
41
42 #include <sys/bio.h>
43 #include <sys/buf.h>
44 #include <sys/globaldata.h>
45 #include <sys/kerneldump.h>
46 #include <sys/malloc.h>
47 #include <sys/mpipe.h>
48 #include <sys/md5.h>
49 #include <sys/mutex2.h>
50 #include <sys/vnode.h>
51 #include <crypto/sha1.h>
52 #include <crypto/sha2/sha2.h>
53 #include <opencrypto/cryptodev.h>
54 #include <opencrypto/rmd160.h>
55 #include <machine/cpufunc.h>
56
57 #include <sys/ktr.h>
58
59 #include <dev/disk/dm/dm.h>
60 MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
61
62 KTR_INFO_MASTER(dmcrypt);
63
64 #if !defined(KTR_DMCRYPT)
65 #define KTR_DMCRYPT     KTR_ALL
66 #endif
67
68 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_dispatch, 0,
69     "crypto_dispatch(%p)", sizeof(void *));
70 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypt_strategy, 0,
71     "crypt_strategy(b_cmd = %d, bp = %p)", sizeof(int) + sizeof(void *));
72 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_start, 1,
73     "crypto_write_start(crp = %p, bp = %p, sector = %d/%d)",
74     sizeof(void *) + sizeof(void *) + sizeof(int) + sizeof(int));
75 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_write_done, 1,
76     "crypto_cb_write_done(crp = %p, bp = %p, n = %d)",
77     sizeof(void *) + sizeof(void *) + sizeof(int));
78 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_write_done, 1,
79     "bio_write_done(bp = %p)", sizeof(void *));
80 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_retry, 1,
81     "crypto_write_retry(crp = %p)", sizeof(void *));
82 KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_read_done, 2,
83     "bio_read_done(bp = %p)", sizeof(void *));
84 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_read_start, 2,
85     "crypto_read_start(crp = %p, bp = %p, sector = %d/%d)",
86     sizeof(void *) + sizeof(void *) + sizeof(int) + sizeof(int));
87 KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_read_done, 2,
88     "crypto_cb_read_done(crp = %p, bp = %p, n = %d)",
89     sizeof(void *) + sizeof(void *) + sizeof(int));
90
91 struct target_crypt_config;
92
93 typedef void dispatch_t(void *);
94 typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
95     void *);
96
97 typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
98 typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
99
100 struct iv_generator {
101         const char      *name;
102         ivgen_ctor_t    *ctor;
103         ivgen_dtor_t    *dtor;
104         ivgen_t         *gen_iv;
105 };
106
107 struct essiv_ivgen_priv {
108         struct cryptoini        crypto_session;
109         struct objcache *crp_crd_cache;
110         u_int64_t       crypto_sid;
111         size_t          keyhash_len;
112         u_int8_t        crypto_keyhash[SHA512_DIGEST_LENGTH];
113 };
114
115 typedef struct target_crypt_config {
116         size_t  params_len;
117         dm_pdev_t *pdev;
118         char    *status_str;
119         int     crypto_alg;
120         int     crypto_klen;
121         u_int8_t        crypto_key[512>>3];
122
123         u_int64_t       crypto_sid;
124         u_int64_t       block_offset;
125         int64_t         iv_offset;
126         SHA512_CTX      essivsha512_ctx;
127
128         struct cryptoini        crypto_session;
129
130         struct iv_generator     *ivgen;
131         void    *ivgen_priv;
132
133         struct malloc_pipe      read_mpipe;
134         struct malloc_pipe      write_mpipe;
135 } dm_target_crypt_config_t;
136
137 struct dmtc_helper {
138         dm_target_crypt_config_t *priv;
139         caddr_t free_addr;
140         caddr_t orig_buf;
141         caddr_t data_buf;
142 };
143
144 struct dmtc_dump_helper {
145         dm_target_crypt_config_t *priv;
146         void *data;
147         size_t length;
148         off_t offset;
149
150         int sectors;
151         int *ident;
152
153         struct cryptodesc crd[128];
154         struct cryptop crp[128];
155         u_char space[65536];
156 };
157
158 #define DMTC_BUF_SIZE_WRITE \
159     MAXPHYS + sizeof(struct dmtc_helper) + \
160     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
161 #define DMTC_BUF_SIZE_READ \
162     sizeof(struct dmtc_helper) + \
163     MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
164
165 static void dmtc_crypto_dispatch(void *arg);
166 static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
167                                 struct dmtc_dump_helper *dump_helper);
168 static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
169                                 struct bio *bio);
170 static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
171                                 struct bio *bio);
172 static void dmtc_bio_read_done(struct bio *bio);
173 static void dmtc_bio_write_done(struct bio *bio);
174 static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
175 static int dmtc_crypto_cb_read_done(struct cryptop *crp);
176 static int dmtc_crypto_cb_write_done(struct cryptop *crp);
177
178 static ivgen_ctor_t     essiv_ivgen_ctor;
179 static ivgen_dtor_t     essiv_ivgen_dtor;
180 static ivgen_t          essiv_ivgen;
181 static ivgen_t          plain_ivgen;
182 static ivgen_t          plain64_ivgen;
183
184 static struct iv_generator ivgens[] = {
185         { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
186             .gen_iv = essiv_ivgen },
187         { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
188         { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
189         { NULL, NULL, NULL, NULL }
190 };
191
192 struct objcache_malloc_args essiv_ivgen_malloc_args = {
193                 2*sizeof(void *) + (sizeof(struct cryptodesc) +
194                 sizeof(struct cryptop)), M_DMCRYPT };
195
196 static void
197 dmtc_init_mpipe(struct target_crypt_config *priv)
198 {
199         int nmax;
200
201         nmax = (physmem*2/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
202
203         if (nmax < 2)
204                 nmax = 2;
205
206         kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
207
208         mpipe_init(&priv->write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
209                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
210         mpipe_init(&priv->read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
211                    2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
212 }
213
214 static void
215 dmtc_destroy_mpipe(struct target_crypt_config *priv)
216 {
217         mpipe_done(&priv->write_mpipe);
218         mpipe_done(&priv->read_mpipe);
219 }
220
221 /*
222  * Overwrite private information (in buf) to avoid leaking it
223  */
224 static void
225 dmtc_crypto_clear(void *buf, size_t len)
226 {
227         memset(buf, 0xFF, len);
228         bzero(buf, len);
229 }
230
231 /*
232  * ESSIV IV Generator Routines
233  */
234 static int
235 essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
236 {
237         struct essiv_ivgen_priv *ivpriv;
238         u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
239         unsigned int klen, hashlen;
240         int error;
241
242         klen = (priv->crypto_klen >> 3);
243
244         if (iv_hash == NULL)
245                 return EINVAL;
246
247         if (!strcmp(iv_hash, "sha1")) {
248                 SHA1_CTX ctx;
249
250                 hashlen = SHA1_RESULTLEN;
251                 SHA1Init(&ctx);
252                 SHA1Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
253                 SHA1Final(crypto_keyhash, &ctx);
254         } else if (!strcmp(iv_hash, "sha256")) {
255                 SHA256_CTX ctx;
256
257                 hashlen = SHA256_DIGEST_LENGTH;
258                 SHA256_Init(&ctx);
259                 SHA256_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
260                 SHA256_Final(crypto_keyhash, &ctx);
261         } else if (!strcmp(iv_hash, "sha384")) {
262                 SHA384_CTX ctx;
263
264                 hashlen = SHA384_DIGEST_LENGTH;
265                 SHA384_Init(&ctx);
266                 SHA384_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
267                 SHA384_Final(crypto_keyhash, &ctx);
268         } else if (!strcmp(iv_hash, "sha512")) {
269                 SHA512_CTX ctx;
270
271                 hashlen = SHA512_DIGEST_LENGTH;
272                 SHA512_Init(&ctx);
273                 SHA512_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
274                 SHA512_Final(crypto_keyhash, &ctx);
275         } else if (!strcmp(iv_hash, "md5")) {
276                 MD5_CTX ctx;
277
278                 hashlen = MD5_DIGEST_LENGTH;
279                 MD5Init(&ctx);
280                 MD5Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
281                 MD5Final(crypto_keyhash, &ctx);
282         } else if (!strcmp(iv_hash, "rmd160") ||
283                    !strcmp(iv_hash, "ripemd160")) {
284                 RMD160_CTX ctx;
285
286                 hashlen = 160/8;
287                 RMD160Init(&ctx);
288                 RMD160Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
289                 RMD160Final(crypto_keyhash, &ctx);
290         } else {
291                 return EINVAL;
292         }
293
294         /* Convert hashlen to bits */
295         hashlen <<= 3;
296
297         ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
298             M_WAITOK | M_ZERO);
299         memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
300         ivpriv->keyhash_len = sizeof(crypto_keyhash);
301         dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
302
303         ivpriv->crypto_session.cri_alg = priv->crypto_alg;
304         ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
305         ivpriv->crypto_session.cri_klen = hashlen;
306         ivpriv->crypto_session.cri_mlen = 0;
307         ivpriv->crypto_session.cri_next = NULL;
308
309         /*
310          * XXX: in principle we also need to check if the block size of the
311          *      cipher is a valid iv size for the block cipher.
312          */
313
314         error = crypto_newsession(&ivpriv->crypto_sid,
315                                   &ivpriv->crypto_session,
316                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
317         if (error) {
318                 kprintf("dm_target_crypt: Error during crypto_newsession "
319                         "for essiv_ivgen, error = %d\n",
320                         error);
321                 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
322                 kfree(ivpriv, M_DMCRYPT);
323                 return ENOTSUP;
324         }
325
326         ivpriv->crp_crd_cache = objcache_create(
327             "dmcrypt-essiv-cache", 0, 0,
328             NULL, NULL, NULL,
329             objcache_malloc_alloc,
330             objcache_malloc_free,
331             &essiv_ivgen_malloc_args );
332
333         *p_ivpriv = ivpriv;
334         return 0;
335 }
336
337 static int 
338 essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
339 {
340         struct essiv_ivgen_priv *ivpriv;
341
342         ivpriv = (struct essiv_ivgen_priv *)arg;
343         KKASSERT(ivpriv != NULL);
344
345         crypto_freesession(ivpriv->crypto_sid);
346
347         objcache_destroy(ivpriv->crp_crd_cache);
348
349         dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
350         kfree(ivpriv, M_DMCRYPT);
351
352         return 0;
353 }
354
355 static int
356 essiv_ivgen_done(struct cryptop *crp)
357 {
358         struct essiv_ivgen_priv *ivpriv;
359         void *free_addr;
360         void *opaque;
361
362
363         if (crp->crp_etype == EAGAIN)
364                 return crypto_dispatch(crp);
365
366         if (crp->crp_etype != 0) {
367                 kprintf("dm_target_crypt: essiv_ivgen_done, "
368                         "crp->crp_etype = %d\n", crp->crp_etype);
369         }
370
371         free_addr = crp->crp_opaque;
372         /*
373          * In-memory structure is:
374          * |  ivpriv  |  opaque  |     crp     |      crd      |
375          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
376          */
377         ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
378         crp->crp_opaque += sizeof(void *);
379         opaque = *((void **)crp->crp_opaque);
380
381         objcache_put(ivpriv->crp_crd_cache, free_addr);
382         dmtc_crypto_dispatch(opaque);
383         return 0;
384 }
385
386 static void
387 essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
388             size_t iv_len, off_t sector, void *opaque)
389 {
390         struct essiv_ivgen_priv *ivpriv;
391         struct cryptodesc *crd;
392         struct cryptop *crp;
393         caddr_t space, alloc_addr;
394         int error;
395
396         ivpriv = priv->ivgen_priv;
397         KKASSERT(ivpriv != NULL);
398
399         /*
400          * In-memory structure is:
401          * |  ivpriv  |  opaque  |     crp     |      crd      |
402          * | (void *) | (void *) |   (cryptop) |  (cryptodesc) |
403          */
404         alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
405         *((struct essiv_ivgen_priv **)space) = ivpriv;
406         space += sizeof(void *);
407         *((void **)space) = opaque;
408         space += sizeof(void *);
409         crp = (struct cryptop *)space;
410         space += sizeof(struct cryptop);
411         crd = (struct cryptodesc *)space;
412
413         bzero(iv, iv_len);
414         bzero(crd, sizeof(struct cryptodesc));
415         bzero(crp, sizeof(struct cryptop));
416         *((off_t *)iv) = htole64(sector + priv->iv_offset);
417         crp->crp_buf = (caddr_t)iv;
418
419         crp->crp_sid = ivpriv->crypto_sid;
420         crp->crp_ilen = crp->crp_olen = iv_len;
421
422         crp->crp_opaque = alloc_addr;
423
424         crp->crp_callback = essiv_ivgen_done;
425
426         crp->crp_desc = crd;
427         crp->crp_etype = 0;
428         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
429
430         crd->crd_alg = priv->crypto_alg;
431 #if 0
432         crd->crd_key = (caddr_t)priv->crypto_keyhash;
433         crd->crd_klen = priv->crypto_klen;
434 #endif
435
436         bzero(crd->crd_iv, sizeof(crd->crd_iv));
437
438         crd->crd_skip = 0;
439         crd->crd_len = iv_len;
440         crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
441         crd->crd_flags |= CRD_F_ENCRYPT;
442         crd->crd_next = NULL;
443
444         error = crypto_dispatch(crp);
445         if (error)
446                 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
447 }
448
449
450 static void
451 plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
452             size_t iv_len, off_t sector, void *opaque)
453 {
454         bzero(iv, iv_len);
455         *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
456         dmtc_crypto_dispatch(opaque);
457 }
458
459 static void
460 plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
461     size_t iv_len, off_t sector, void *opaque)
462 {
463         bzero(iv, iv_len);
464         *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
465         dmtc_crypto_dispatch(opaque);
466 }
467
468 #if 0
469 static void
470 geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
471            size_t iv_len, off_t sector, void *opaque)
472 {
473
474         SHA512_CTX      ctx512;
475         u_int8_t        md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
476
477         memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
478         SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
479         SHA512_Final(md, &ctx512);
480
481         memcpy(iv, md, iv_len);
482         dmtc_crypto_dispatch(opaque);
483 }
484 #endif
485
486 /*
487  * Init function called from dm_table_load_ioctl.
488  * cryptsetup actually passes us this:
489  * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
490  */
491 static int
492 hex2key(char *hex, size_t key_len, u_int8_t *key)
493 {
494         char hex_buf[3];
495         size_t key_idx;
496
497         hex_buf[2] = 0;
498         for (key_idx = 0; key_idx < key_len; ++key_idx) {
499                 hex_buf[0] = *hex++;
500                 hex_buf[1] = *hex++;
501                 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
502         }
503         hex_buf[0] = 0;
504         hex_buf[1] = 0;
505
506         return 0;
507 }
508
509 static int
510 dm_target_crypt_init(dm_dev_t * dmv, void **target_config, char *params)
511 {
512         dm_target_crypt_config_t *priv;
513         size_t len;
514         char **ap, *args[5];
515         char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
516         char *status_str;
517         int i, argc, klen, error;
518         uint64_t iv_offset, block_offset;
519
520         if (params == NULL)
521                 return EINVAL;
522
523         len = strlen(params) + 1;
524         argc = 0;
525
526         status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
527         /*
528          * Parse a string, containing tokens delimited by white space,
529          * into an argument vector
530          */
531         for (ap = args; ap < &args[5] &&
532             (*ap = strsep(&params, " \t")) != NULL;) {
533                 if (**ap != '\0') {
534                         argc++;
535                         ap++;
536                 }
537         }
538
539         if (argc != 5) {
540                 kprintf("dm_target_crypt: not enough arguments, "
541                         "need exactly 5\n");
542                 kfree(status_str, M_DMCRYPT);
543                 return ENOMEM; /* XXX */
544         }
545
546         crypto_alg = strsep(&args[0], "-");
547         crypto_mode = strsep(&args[0], "-");
548         iv_opt = strsep(&args[0], "-");
549         iv_mode = strsep(&iv_opt, ":");
550         key = args[1];
551         iv_offset = strtouq(args[2], NULL, 0);
552         dev = args[3];
553         block_offset = strtouq(args[4], NULL, 0);
554         /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
555         klen = strlen(key) << 2;
556
557 #if 0
558         kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
559                 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
560                 "block_offset=%ju\n",
561                 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
562                 block_offset);
563 #endif
564
565         priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
566
567         /* Insert dmp to global pdev list */
568         if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
569                 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
570                 kfree(status_str, M_DMCRYPT);
571                 return ENOENT;
572         }
573
574         /*
575          * This code checks for valid combinations of algorithm and mode.
576          * Currently supported options are:
577          *
578          * *-cbc
579          * aes-xts
580          * twofish-xts
581          * serpent-xts
582          */
583         if ((strcmp(crypto_mode, "cbc") != 0) &&
584             !((strcmp(crypto_mode, "xts") == 0) &&
585             ((strcmp(crypto_alg, "aes") == 0) ||
586             (strcmp(crypto_alg, "twofish") == 0) ||
587             (strcmp(crypto_alg, "serpent") == 0))))
588         {
589                 kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
590                     " aes-xts, twofish-xts and serpent-xts, "
591                     "invalid mode '%s-%s'\n",
592                     crypto_alg, crypto_mode);
593                 goto notsup;
594         }
595
596         if (!strcmp(crypto_alg, "aes")) {
597                 if (!strcmp(crypto_mode, "xts")) {
598                         priv->crypto_alg = CRYPTO_AES_XTS;
599                         if (klen != 256 && klen != 512)
600                                 goto notsup;
601                 } else if (!strcmp(crypto_mode, "cbc")) {
602                         priv->crypto_alg = CRYPTO_AES_CBC;
603                         if (klen != 128 && klen != 192 && klen != 256)
604                                 goto notsup;
605                 } else {
606                         goto notsup;
607                 }
608                 priv->crypto_klen = klen;
609         } else if (!strcmp(crypto_alg, "twofish")) {
610                 if (!strcmp(crypto_mode, "xts")) {
611                         priv->crypto_alg = CRYPTO_TWOFISH_XTS;
612                         if (klen != 256 && klen != 512)
613                                 goto notsup;
614                 } else if (!strcmp(crypto_mode, "cbc")) {
615                         priv->crypto_alg = CRYPTO_TWOFISH_CBC;
616                         if (klen != 128 && klen != 192 && klen != 256)
617                                 goto notsup;
618                 } else {
619                         goto notsup;
620                 }
621                 priv->crypto_klen = klen;
622         } else if (!strcmp(crypto_alg, "serpent")) {
623                 if (!strcmp(crypto_mode, "xts")) {
624                         priv->crypto_alg = CRYPTO_SERPENT_XTS;
625                         if (klen != 256 && klen != 512)
626                                 goto notsup;
627                 } else if (!strcmp(crypto_mode, "cbc")) {
628                         priv->crypto_alg = CRYPTO_SERPENT_CBC;
629                         if (klen != 128 && klen != 192 && klen != 256)
630                                 goto notsup;
631                 } else {
632                         goto notsup;
633                 }
634                 priv->crypto_klen = klen;
635         } else if (!strcmp(crypto_alg, "blowfish")) {
636                 priv->crypto_alg = CRYPTO_BLF_CBC;
637                 if (klen < 128 || klen > 448 || (klen % 8) != 0)
638                         goto notsup;
639                 priv->crypto_klen = klen;
640         } else if (!strcmp(crypto_alg, "3des") ||
641                    !strncmp(crypto_alg, "des3", 4)) {
642                 priv->crypto_alg = CRYPTO_3DES_CBC;
643                 if (klen != 168)
644                         goto notsup;
645                 priv->crypto_klen = 168;
646         } else if (!strcmp(crypto_alg, "camellia")) {
647                 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
648                 if (klen != 128 && klen != 192 && klen != 256)
649                         goto notsup;
650                 priv->crypto_klen = klen;
651         } else if (!strcmp(crypto_alg, "skipjack")) {
652                 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
653                 if (klen != 80)
654                         goto notsup;
655                 priv->crypto_klen = 80;
656         } else if (!strcmp(crypto_alg, "cast5")) {
657                 priv->crypto_alg = CRYPTO_CAST_CBC;
658                 if (klen != 128)
659                         goto notsup;
660                 priv->crypto_klen = 128;
661         } else if (!strcmp(crypto_alg, "null")) {
662                 priv->crypto_alg = CRYPTO_NULL_CBC;
663                 if (klen != 128)
664                         goto notsup;
665                 priv->crypto_klen = 128;
666         } else {
667                 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
668                         crypto_alg);
669                 goto notsup;
670         }
671
672         /* Save length of param string */
673         priv->params_len = len;
674         priv->block_offset = block_offset;
675         priv->iv_offset = iv_offset - block_offset;
676
677         *target_config = priv;
678
679         dmv->dev_type = DM_CRYPTO_DEV;
680
681         error = hex2key(key, priv->crypto_klen >> 3,
682                         (u_int8_t *)priv->crypto_key);
683
684         if (error) {
685                 kprintf("dm_target_crypt: hex2key failed, "
686                         "invalid key format\n");
687                 goto notsup;
688         }
689
690         /* Handle cmd */
691         for(i = 0; ivgens[i].name != NULL; i++) {
692                 if (!strcmp(iv_mode, ivgens[i].name))
693                         break;
694         }
695
696         if (ivgens[i].name == NULL) {
697                 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
698                         iv_mode);       
699                 goto notsup;
700         }
701
702         /* Call our ivgen constructor */
703         if (ivgens[i].ctor != NULL) {
704                 error = ivgens[i].ctor(priv, iv_opt,
705                     &priv->ivgen_priv);
706                 if (error) {
707                         kprintf("dm_target_crypt: ctor for '%s' failed\n",
708                             ivgens[i].name);
709                         goto notsup;
710                 }
711         }
712
713         priv->ivgen = &ivgens[i];
714
715         priv->crypto_session.cri_alg = priv->crypto_alg;
716         priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
717         priv->crypto_session.cri_klen = priv->crypto_klen;
718         priv->crypto_session.cri_mlen = 0;
719         priv->crypto_session.cri_next = NULL;
720
721         error = crypto_newsession(&priv->crypto_sid,
722                                   &priv->crypto_session,
723                                   CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
724         if (error) {
725                 kprintf("dm_target_crypt: Error during crypto_newsession, "
726                         "error = %d\n",
727                         error);
728                 goto notsup;
729         }
730
731         memset(key, '0', strlen(key));
732         if (iv_opt) {
733                 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
734                     crypto_alg, crypto_mode, iv_mode, iv_opt,
735                     key, iv_offset, dev, block_offset);
736         } else {
737                 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
738                     crypto_alg, crypto_mode, iv_mode,
739                     key, iv_offset, dev, block_offset);
740         }
741         priv->status_str = status_str;
742
743         /* Initialize mpipes */
744         dmtc_init_mpipe(priv);
745
746         return 0;
747
748 notsup:
749         kprintf("dm_target_crypt: ENOTSUP\n");
750         kfree(status_str, M_DMCRYPT);
751         return ENOTSUP;
752 }
753
754 /* Status routine called to get params string. */
755 static char *
756 dm_target_crypt_status(void *target_config)
757 {
758         dm_target_crypt_config_t *priv;
759         char *params;
760
761         priv = target_config;
762
763         /* caller expects use of M_DM */
764         params = kmalloc(DM_MAX_PARAMS_SIZE, M_DM, M_WAITOK);
765
766         ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
767             priv->status_str);
768
769         return params;
770 }
771
772 static int
773 dm_target_crypt_destroy(dm_table_entry_t * table_en)
774 {
775         dm_target_crypt_config_t *priv;
776
777         /*
778          * Disconnect the crypt config before unbusying the target.
779          */
780         priv = table_en->target_config;
781         if (priv == NULL)
782                 return 0;
783         table_en->target_config = NULL;
784         dm_pdev_decr(priv->pdev);
785
786         /*
787          * Clean up the crypt config
788          *
789          * Overwrite the private information before freeing memory to
790          * avoid leaking it.
791          */
792         if (priv->status_str) {
793                 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
794                 kfree(priv->status_str, M_DMCRYPT);
795                 crypto_freesession(priv->crypto_sid);
796         }
797
798         if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
799                 priv->ivgen->dtor(priv, priv->ivgen_priv);
800         }
801
802         /* Destroy mpipes */
803         dmtc_destroy_mpipe(priv);
804
805         dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
806         kfree(priv, M_DMCRYPT);
807
808         return 0;
809 }
810
811 static int
812 dm_target_crypt_deps(dm_table_entry_t * table_en, prop_array_t prop_array)
813 {
814         dm_target_crypt_config_t *priv;
815         struct vattr va;
816
817         int error;
818
819         if (table_en->target_config == NULL)
820                 return ENOENT;
821
822         priv = table_en->target_config;
823
824         if ((error = VOP_GETATTR(priv->pdev->pdev_vnode, &va)) != 0)
825                 return error;
826
827         prop_array_add_uint64(prop_array,
828                               (uint64_t)makeudev(va.va_rmajor, va.va_rminor));
829
830         return 0;
831 }
832
833 /* Unsupported for this target. */
834 static int
835 dm_target_crypt_upcall(dm_table_entry_t * table_en, struct buf * bp)
836 {
837         return 0;
838 }
839
840 /************************************************************************
841  *                      STRATEGY SUPPORT FUNCTIONS                      *
842  ************************************************************************
843  *
844  * READ PATH:   doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
845  * WRITE PATH:  crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
846  */
847
848 /*
849  * Wrapper around crypto_dispatch() to match dispatch_t type
850  */
851 static void
852 dmtc_crypto_dispatch(void *arg)
853 {
854         struct cryptop *crp;
855
856         crp = (struct cryptop *)arg;
857         KKASSERT(crp != NULL);
858         KTR_LOG(dmcrypt_crypto_dispatch, crp);
859         crypto_dispatch(crp);
860 }
861
862 /*
863  * Start IO operation, called from dmstrategy routine.
864  */
865 static int
866 dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
867 {
868         struct bio *bio;
869
870         dm_target_crypt_config_t *priv;
871         priv = table_en->target_config;
872
873         /* Get rid of stuff we can't really handle */
874         if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
875                 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
876                         kprintf("dm_target_crypt_strategy: can't really "
877                                 "handle bp->b_bcount = %d\n",
878                                 bp->b_bcount);
879                         bp->b_error = EINVAL;
880                         bp->b_flags |= B_ERROR | B_INVAL;
881                         biodone(&bp->b_bio1);
882                         return 0;
883                 }
884         }
885
886         KTR_LOG(dmcrypt_crypt_strategy, bp->b_cmd, bp);
887
888         switch (bp->b_cmd) {
889         case BUF_CMD_READ:
890                 bio = push_bio(&bp->b_bio1);
891                 bio->bio_offset = bp->b_bio1.bio_offset +
892                                   priv->block_offset * DEV_BSIZE;
893                 bio->bio_caller_info1.ptr = priv;
894                 bio->bio_done = dmtc_bio_read_done;
895                 vn_strategy(priv->pdev->pdev_vnode, bio);
896                 break;
897         case BUF_CMD_WRITE:
898                 bio = push_bio(&bp->b_bio1);
899                 bio->bio_offset = bp->b_bio1.bio_offset +
900                                   priv->block_offset * DEV_BSIZE;
901                 bio->bio_caller_info1.ptr = priv;
902                 dmtc_crypto_write_start(priv, bio);
903                 break;
904         default:
905                 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
906                 break;
907         }
908         return 0;
909 }
910
911 /*
912  * STRATEGY READ PATH PART 1/3 (after read BIO completes)
913  */
914 static void
915 dmtc_bio_read_done(struct bio *bio)
916 {
917         struct bio *obio;
918
919         dm_target_crypt_config_t *priv;
920
921         KTR_LOG(dmcrypt_bio_read_done, bio->bio_buf);
922
923         /*
924          * If a read error occurs we shortcut the operation, otherwise
925          * go on to stage 2.
926          */
927         if (bio->bio_buf->b_flags & B_ERROR) {
928                 obio = pop_bio(bio);
929                 biodone(obio);
930         } else {
931                 priv = bio->bio_caller_info1.ptr;
932                 dmtc_crypto_read_start(priv, bio);
933         }
934 }
935
936 /*
937  * STRATEGY READ PATH PART 2/3
938  */
939 static void
940 dmtc_crypto_read_retry(void *arg1, void *arg2)
941 {
942         dm_target_crypt_config_t *priv = arg1;
943         struct bio *bio = arg2;
944
945         dmtc_crypto_read_start(priv, bio);
946 }
947
948 static void
949 dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
950 {
951         struct dmtc_helper *dmtc;
952         struct cryptodesc *crd;
953         struct cryptop *crp;
954         struct cryptoini *cri;
955         int i, bytes, sectors, sz;
956         off_t isector;
957         u_char *ptr, *space;
958
959         cri = &priv->crypto_session;
960
961         /*
962          * Note: b_resid no good after read I/O, it will be 0, use
963          *       b_bcount.
964          */
965         bytes = bio->bio_buf->b_bcount;
966         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
967         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
968         sz = sectors * (sizeof(*crp) + sizeof(*crd));
969
970         /*
971          * For reads with bogus page we can't decrypt in place as stuff
972          * can get ripped out from under us.
973          *
974          * XXX actually it looks like we can, and in any case the initial
975          * read already completed and threw crypted data into the buffer
976          * cache buffer.  Disable for now.
977          */
978         space = mpipe_alloc_callback(&priv->read_mpipe,
979                                      dmtc_crypto_read_retry, priv, bio);
980         if (space == NULL)
981                 return;
982
983         dmtc = (struct dmtc_helper *)space;
984         dmtc->free_addr = space;
985         space += sizeof(struct dmtc_helper);
986         dmtc->orig_buf = NULL;
987         dmtc->data_buf = bio->bio_buf->b_data;
988         dmtc->priv = priv;
989         bio->bio_caller_info2.ptr = dmtc;
990         bio->bio_buf->b_error = 0;
991
992         /*
993          * Load crypto descriptors (crp/crd loop)
994          */
995         bzero(space, sz);
996         ptr = space;
997         bio->bio_caller_info3.value = sectors;
998         cpu_sfence();
999 #if 0
1000         kprintf("Read, bytes = %d (b_bcount), "
1001                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1002                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1003 #endif
1004         for (i = 0; i < sectors; i++) {
1005                 crp = (struct cryptop *)ptr;
1006                 ptr += sizeof(*crp);
1007                 crd = (struct cryptodesc *)ptr;
1008                 ptr += sizeof (*crd);
1009
1010                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1011
1012                 crp->crp_sid = priv->crypto_sid;
1013                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1014
1015                 crp->crp_opaque = (void *)bio;
1016
1017                 crp->crp_callback = dmtc_crypto_cb_read_done;
1018                 crp->crp_desc = crd;
1019                 crp->crp_etype = 0;
1020                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1021                                  CRYPTO_F_BATCH;
1022
1023                 crd->crd_alg = priv->crypto_alg;
1024 #if 0
1025                 crd->crd_key = (caddr_t)priv->crypto_key;
1026                 crd->crd_klen = priv->crypto_klen;
1027 #endif
1028
1029                 crd->crd_skip = 0;
1030                 crd->crd_len = DEV_BSIZE /* XXX */;
1031                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1032                 crd->crd_next = NULL;
1033
1034                 crd->crd_flags &= ~CRD_F_ENCRYPT;
1035
1036                 KTR_LOG(dmcrypt_crypto_read_start, crp, bio->bio_buf, i,
1037                     sectors);
1038
1039                 /*
1040                  * Note: last argument is used to generate salt(?) and is
1041                  *       a 64 bit value, but the original code passed an
1042                  *       int.  Changing it now will break pre-existing
1043                  *       crypt volumes.
1044                  */
1045                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1046                                     isector + i, crp);
1047         }
1048 }
1049
1050 /*
1051  * STRATEGY READ PATH PART 3/3
1052  */
1053 static int
1054 dmtc_crypto_cb_read_done(struct cryptop *crp)
1055 {
1056         struct dmtc_helper *dmtc;
1057         struct bio *bio, *obio;
1058         int n;
1059
1060         if (crp->crp_etype == EAGAIN)
1061                 return crypto_dispatch(crp);
1062
1063         bio = (struct bio *)crp->crp_opaque;
1064         KKASSERT(bio != NULL);
1065
1066         /*
1067          * Cumulative error
1068          */
1069         if (crp->crp_etype) {
1070                 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1071                         "crp_etype = %d\n",
1072                         crp->crp_etype);
1073                 bio->bio_buf->b_error = crp->crp_etype;
1074         }
1075
1076         /*
1077          * On the last chunk of the decryption we do any required copybacks
1078          * and complete the I/O.
1079          */
1080         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1081 #if 0
1082         kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1083 #endif
1084
1085         KTR_LOG(dmcrypt_crypto_cb_read_done, crp, bio->bio_buf, n);
1086
1087         if (n == 1) {
1088                 /*
1089                  * For the B_HASBOGUS case we didn't decrypt in place,
1090                  * so we need to copy stuff back into the buf.
1091                  *
1092                  * (disabled for now).
1093                  */
1094                 dmtc = bio->bio_caller_info2.ptr;
1095                 if (bio->bio_buf->b_error) {
1096                         bio->bio_buf->b_flags |= B_ERROR;
1097                 }
1098 #if 0
1099                 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
1100                         memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1101                                bio->bio_buf->b_bcount);
1102                 }
1103 #endif
1104                 mpipe_free(&dmtc->priv->read_mpipe, dmtc->free_addr);
1105                 obio = pop_bio(bio);
1106                 biodone(obio);
1107         }
1108         return 0;
1109 }
1110 /* END OF STRATEGY READ SECTION */
1111
1112 /*
1113  * STRATEGY WRITE PATH PART 1/3
1114  */
1115
1116 static void
1117 dmtc_crypto_write_retry(void *arg1, void *arg2)
1118 {
1119         dm_target_crypt_config_t *priv = arg1;
1120         struct bio *bio = arg2;
1121
1122         KTR_LOG(dmcrypt_crypto_write_retry, bio->bio_buf);
1123
1124         dmtc_crypto_write_start(priv, bio);
1125 }
1126
1127 static void
1128 dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1129 {
1130         struct dmtc_helper *dmtc;
1131         struct cryptodesc *crd;
1132         struct cryptop *crp;
1133         struct cryptoini *cri;
1134         int i, bytes, sectors, sz;
1135         off_t isector;
1136         u_char *ptr, *space;
1137
1138         cri = &priv->crypto_session;
1139
1140         /*
1141          * Use b_bcount for consistency
1142          */
1143         bytes = bio->bio_buf->b_bcount;
1144
1145         isector = bio->bio_offset / DEV_BSIZE;  /* ivgen salt base? */
1146         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1147         sz = sectors * (sizeof(*crp) + sizeof(*crd));
1148
1149         /*
1150          * For writes and reads with bogus page don't decrypt in place.
1151          */
1152         space = mpipe_alloc_callback(&priv->write_mpipe,
1153                                      dmtc_crypto_write_retry, priv, bio);
1154         if (space == NULL)
1155                 return;
1156
1157         dmtc = (struct dmtc_helper *)space;
1158         dmtc->free_addr = space;
1159         space += sizeof(struct dmtc_helper);
1160         memcpy(space + sz, bio->bio_buf->b_data, bytes);
1161
1162         bio->bio_caller_info2.ptr = dmtc;
1163         bio->bio_buf->b_error = 0;
1164
1165         dmtc->orig_buf = bio->bio_buf->b_data;
1166         dmtc->data_buf = space + sz;
1167         dmtc->priv = priv;
1168
1169         /*
1170          * Load crypto descriptors (crp/crd loop)
1171          */
1172         bzero(space, sz);
1173         ptr = space;
1174         bio->bio_caller_info3.value = sectors;
1175         cpu_sfence();
1176 #if 0
1177         kprintf("Write, bytes = %d (b_bcount), "
1178                 "sectors = %d (bio = %p, b_cmd = %d)\n",
1179                 bytes, sectors, bio, bio->bio_buf->b_cmd);
1180 #endif
1181         for (i = 0; i < sectors; i++) {
1182                 crp = (struct cryptop *)ptr;
1183                 ptr += sizeof(*crp);
1184                 crd = (struct cryptodesc *)ptr;
1185                 ptr += sizeof (*crd);
1186
1187                 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1188
1189                 crp->crp_sid = priv->crypto_sid;
1190                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1191
1192                 crp->crp_opaque = (void *)bio;
1193
1194                 crp->crp_callback = dmtc_crypto_cb_write_done;
1195                 crp->crp_desc = crd;
1196                 crp->crp_etype = 0;
1197                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1198                                  CRYPTO_F_BATCH;
1199
1200                 crd->crd_alg = priv->crypto_alg;
1201 #if 0
1202                 crd->crd_key = (caddr_t)priv->crypto_key;
1203                 crd->crd_klen = priv->crypto_klen;
1204 #endif
1205
1206                 crd->crd_skip = 0;
1207                 crd->crd_len = DEV_BSIZE /* XXX */;
1208                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1209                 crd->crd_next = NULL;
1210
1211                 crd->crd_flags |= CRD_F_ENCRYPT;
1212
1213                 /*
1214                  * Note: last argument is used to generate salt(?) and is
1215                  *       a 64 bit value, but the original code passed an
1216                  *       int.  Changing it now will break pre-existing
1217                  *       crypt volumes.
1218                  */
1219
1220                 KTR_LOG(dmcrypt_crypto_write_start, crp, bio->bio_buf,
1221                     i, sectors);
1222
1223                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1224                                     isector + i, crp);
1225         }
1226 }
1227
1228 /*
1229  * STRATEGY WRITE PATH PART 2/3
1230  */
1231 static int
1232 dmtc_crypto_cb_write_done(struct cryptop *crp)
1233 {
1234         struct dmtc_helper *dmtc;
1235         dm_target_crypt_config_t *priv;
1236         struct bio *bio, *obio;
1237         int n;
1238
1239         if (crp->crp_etype == EAGAIN)
1240                 return crypto_dispatch(crp);
1241
1242         bio = (struct bio *)crp->crp_opaque;
1243         KKASSERT(bio != NULL);
1244
1245         /*
1246          * Cumulative error
1247          */
1248         if (crp->crp_etype != 0) {
1249                 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1250                         "crp_etype = %d\n",
1251                 crp->crp_etype);
1252                 bio->bio_buf->b_error = crp->crp_etype;
1253         }
1254
1255         /*
1256          * On the last chunk of the encryption we issue the write
1257          */
1258         n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1259 #if 0
1260         kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1261 #endif
1262
1263         KTR_LOG(dmcrypt_crypto_cb_write_done, crp, bio->bio_buf, n);
1264
1265         if (n == 1) {
1266                 dmtc = bio->bio_caller_info2.ptr;
1267                 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1268
1269                 if (bio->bio_buf->b_error) {
1270                         bio->bio_buf->b_flags |= B_ERROR;
1271                         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1272                         obio = pop_bio(bio);
1273                         biodone(obio);
1274                 } else {
1275                         dmtc->orig_buf = bio->bio_buf->b_data;
1276                         bio->bio_buf->b_data = dmtc->data_buf;
1277                         bio->bio_done = dmtc_bio_write_done;
1278                         vn_strategy(priv->pdev->pdev_vnode, bio);
1279                 }
1280         }
1281         return 0;
1282 }
1283
1284 /*
1285  * STRATEGY WRITE PATH PART 3/3
1286  */
1287 static void
1288 dmtc_bio_write_done(struct bio *bio)
1289 {
1290         struct dmtc_helper *dmtc;
1291         struct bio *obio;
1292
1293         dmtc = bio->bio_caller_info2.ptr;
1294         bio->bio_buf->b_data = dmtc->orig_buf;
1295         mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1296
1297         KTR_LOG(dmcrypt_bio_write_done, bio->bio_buf);
1298
1299         obio = pop_bio(bio);
1300         biodone(obio);
1301 }
1302 /* END OF STRATEGY WRITE SECTION */
1303
1304
1305
1306 /* DUMPING MAGIC */
1307
1308 extern int tsleep_crypto_dump;
1309
1310 static int
1311 dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1312 {
1313         static struct dmtc_dump_helper dump_helper;
1314         dm_target_crypt_config_t *priv;
1315         int id;
1316         static int first_call = 1;
1317
1318         priv = table_en->target_config;
1319
1320         if (first_call) {
1321                 first_call = 0;
1322                 dump_reactivate_cpus();
1323         }
1324
1325         /* Magically enable tsleep */
1326         tsleep_crypto_dump = 1;
1327         id = 0;
1328
1329         /*
1330          * 0 length means flush buffers and return
1331          */
1332         if (length == 0) {
1333                 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1334                         tsleep_crypto_dump = 0;
1335                         return ENXIO;
1336                 }
1337                 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1338                     data, 0, offset, 0);
1339                 tsleep_crypto_dump = 0;
1340                 return 0;
1341         }
1342
1343         bzero(&dump_helper, sizeof(dump_helper));
1344         dump_helper.priv = priv;
1345         dump_helper.data = data;
1346         dump_helper.length = length;
1347         dump_helper.offset = offset +
1348             priv->block_offset * DEV_BSIZE;
1349         dump_helper.ident = &id;
1350         dmtc_crypto_dump_start(priv, &dump_helper);
1351
1352         /*
1353          * Hackery to make stuff appear synchronous. The crypto callback will
1354          * set id to 1 and call wakeup on it. If the request completed
1355          * synchronously, id will be 1 and we won't bother to sleep. If not,
1356          * the crypto request will complete asynchronously and we sleep until
1357          * it's done.
1358          */
1359         if (id == 0)
1360                 tsleep(&dump_helper, 0, "cryptdump", 0);
1361
1362         dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1363             dump_helper.offset);
1364
1365         dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1366             dump_helper.space, 0, dump_helper.offset,
1367             dump_helper.length);
1368
1369         tsleep_crypto_dump = 0;
1370         return 0;
1371 }
1372
1373 static void
1374 dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1375 {
1376         struct cryptodesc *crd;
1377         struct cryptop *crp;
1378         struct cryptoini *cri;
1379         int i, bytes, sectors;
1380         off_t isector;
1381
1382         cri = &priv->crypto_session;
1383
1384         bytes = dump_helper->length;
1385
1386         isector = dump_helper->offset / DEV_BSIZE;      /* ivgen salt base? */
1387         sectors = bytes / DEV_BSIZE;            /* Number of sectors */
1388         dump_helper->sectors = sectors;
1389 #if 0
1390         kprintf("Dump, bytes = %d, "
1391                 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1392 #endif
1393         KKASSERT(dump_helper->length <= 65536);
1394
1395         memcpy(dump_helper->space, dump_helper->data, bytes);
1396
1397         cpu_sfence();
1398
1399         for (i = 0; i < sectors; i++) {
1400                 crp = &dump_helper->crp[i];
1401                 crd = &dump_helper->crd[i];
1402
1403                 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1404
1405                 crp->crp_sid = priv->crypto_sid;
1406                 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1407
1408                 crp->crp_opaque = (void *)dump_helper;
1409
1410                 crp->crp_callback = dmtc_crypto_cb_dump_done;
1411                 crp->crp_desc = crd;
1412                 crp->crp_etype = 0;
1413                 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1414                                  CRYPTO_F_BATCH;
1415
1416                 crd->crd_alg = priv->crypto_alg;
1417
1418                 crd->crd_skip = 0;
1419                 crd->crd_len = DEV_BSIZE /* XXX */;
1420                 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1421                 crd->crd_next = NULL;
1422
1423                 crd->crd_flags |= CRD_F_ENCRYPT;
1424
1425                 /*
1426                  * Note: last argument is used to generate salt(?) and is
1427                  *       a 64 bit value, but the original code passed an
1428                  *       int.  Changing it now will break pre-existing
1429                  *       crypt volumes.
1430                  */
1431                 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1432                                     isector + i, crp);
1433         }
1434 }
1435
1436 static int
1437 dmtc_crypto_cb_dump_done(struct cryptop *crp)
1438 {
1439         struct dmtc_dump_helper *dump_helper;
1440         dm_target_crypt_config_t *priv;
1441         int n;
1442
1443         if (crp->crp_etype == EAGAIN)
1444                 return crypto_dispatch(crp);
1445
1446         dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1447         KKASSERT(dump_helper != NULL);
1448
1449         if (crp->crp_etype != 0) {
1450                 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1451                         "crp_etype = %d\n",
1452                 crp->crp_etype);
1453                 return crp->crp_etype;
1454         }
1455
1456         /*
1457          * On the last chunk of the encryption we return control
1458          */
1459         n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1460
1461         if (n == 1) {
1462                 priv = (dm_target_crypt_config_t *)dump_helper->priv;
1463                 atomic_add_int(dump_helper->ident, 1);
1464                 wakeup(dump_helper);
1465         }
1466
1467         return 0;
1468 }
1469
1470 static int
1471 dmtc_mod_handler(module_t mod, int type, void *unused)
1472 {
1473         dm_target_t *dmt = NULL;
1474         int err = 0;
1475
1476         switch (type) {
1477         case MOD_LOAD:
1478                 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1479                         dm_target_unbusy(dmt);
1480                         return EEXIST;
1481                 }
1482                 dmt = dm_target_alloc("crypt");
1483                 dmt->version[0] = 1;
1484                 dmt->version[1] = 6;
1485                 dmt->version[2] = 0;
1486                 strlcpy(dmt->name, "crypt", DM_MAX_TYPE_NAME);
1487                 dmt->init = &dm_target_crypt_init;
1488                 dmt->status = &dm_target_crypt_status;
1489                 dmt->strategy = &dm_target_crypt_strategy;
1490                 dmt->deps = &dm_target_crypt_deps;
1491                 dmt->destroy = &dm_target_crypt_destroy;
1492                 dmt->upcall = &dm_target_crypt_upcall;
1493                 dmt->dump = &dm_target_crypt_dump;
1494
1495                 err = dm_target_insert(dmt);
1496                 if (!err)
1497                         kprintf("dm_target_crypt: Successfully initialized\n");
1498                 break;
1499
1500         case MOD_UNLOAD:
1501                 err = dm_target_rem("crypt");
1502                 if (err == 0) {
1503                         kprintf("dm_target_crypt: unloaded\n");
1504                 }
1505                 break;
1506
1507         default:
1508                 break;
1509         }
1510
1511         return err;
1512 }
1513
1514 DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);