dm - undo my pointless moving of dm.h
[dragonfly.git] / sys / dev / disk / dm / targets / crypt / dm_target_crypt.c
CommitLineData
dd9da8fe
AH
1/*
2 * Copyright (c) 2010 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Alex Hornung <ahornung@gmail.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35/*
36 * This file implements initial version of device-mapper crypt target.
37 */
38#include <sys/types.h>
39#include <sys/param.h>
ff3e5675 40#include <sys/endian.h>
dd9da8fe
AH
41
42#include <sys/bio.h>
43#include <sys/buf.h>
79b7159f
AH
44#include <sys/globaldata.h>
45#include <sys/kerneldump.h>
dd9da8fe 46#include <sys/malloc.h>
ba65ac12 47#include <sys/mpipe.h>
b559ed14 48#include <sys/md5.h>
79b7159f 49#include <sys/mutex2.h>
dd9da8fe
AH
50#include <sys/vnode.h>
51#include <crypto/sha1.h>
52#include <crypto/sha2/sha2.h>
53#include <opencrypto/cryptodev.h>
b559ed14 54#include <opencrypto/rmd160.h>
c98abe57 55#include <machine/cpufunc.h>
dd9da8fe 56
a84e173e 57#include <dev/disk/dm/dm.h>
dd9da8fe
AH
58MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
59
dd9da8fe 60struct target_crypt_config;
7cfe2226
AH
61
62typedef void dispatch_t(void *);
63typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
64 void *);
65
66typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
67typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
68
69struct iv_generator {
70 const char *name;
71 ivgen_ctor_t *ctor;
72 ivgen_dtor_t *dtor;
73 ivgen_t *gen_iv;
74};
75
76struct essiv_ivgen_priv {
77 struct cryptoini crypto_session;
78 struct objcache *crp_crd_cache;
79 u_int64_t crypto_sid;
80 size_t keyhash_len;
81 u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
82};
dd9da8fe
AH
83
84typedef struct target_crypt_config {
85 size_t params_len;
86 dm_pdev_t *pdev;
aadb5a11 87 char *status_str;
dd9da8fe
AH
88 int crypto_alg;
89 int crypto_klen;
90 u_int8_t crypto_key[512>>3];
79b7159f 91
dd9da8fe 92 u_int64_t crypto_sid;
b559ed14 93 u_int64_t block_offset;
48a0340e 94 int64_t iv_offset;
dd9da8fe 95 SHA512_CTX essivsha512_ctx;
7cfe2226 96
dd9da8fe 97 struct cryptoini crypto_session;
7cfe2226
AH
98
99 struct iv_generator *ivgen;
100 void *ivgen_priv;
dd9da8fe
AH
101} dm_target_crypt_config_t;
102
b559ed14
AH
103struct dmtc_helper {
104 caddr_t free_addr;
105 caddr_t orig_buf;
f6221ad1 106 caddr_t data_buf;
b559ed14
AH
107};
108
79b7159f
AH
109struct dmtc_dump_helper {
110 dm_target_crypt_config_t *priv;
111 void *data;
112 size_t length;
113 off_t offset;
114
115 int sectors;
116 int *ident;
117
118 struct cryptodesc crd[128];
119 struct cryptop crp[128];
120 u_char space[65536];
121};
122
ba65ac12
AH
123#define DMTC_BUF_SIZE_WRITE \
124 MAXPHYS + sizeof(struct dmtc_helper) + \
125 MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
126#define DMTC_BUF_SIZE_READ \
127 sizeof(struct dmtc_helper) + \
128 MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
129
7cfe2226 130static void dmtc_crypto_dispatch(void *arg);
79b7159f
AH
131static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
132 struct dmtc_dump_helper *dump_helper);
f6221ad1
MD
133static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
134 struct bio *bio);
135static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
136 struct bio *bio);
ff3e5675
AH
137static void dmtc_bio_read_done(struct bio *bio);
138static void dmtc_bio_write_done(struct bio *bio);
79b7159f 139static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
ff3e5675
AH
140static int dmtc_crypto_cb_read_done(struct cryptop *crp);
141static int dmtc_crypto_cb_write_done(struct cryptop *crp);
dd9da8fe 142
7cfe2226
AH
143static ivgen_ctor_t essiv_ivgen_ctor;
144static ivgen_dtor_t essiv_ivgen_dtor;
145static ivgen_t essiv_ivgen;
146static ivgen_t plain_ivgen;
147
148static struct iv_generator ivgens[] = {
149 { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
150 .gen_iv = essiv_ivgen },
151 { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
152 { NULL, NULL, NULL, NULL }
153};
154
155struct objcache_malloc_args essiv_ivgen_malloc_args = {
156 2*sizeof(void *) + (sizeof(struct cryptodesc) +
157 sizeof(struct cryptop)), M_DMCRYPT };
158
ba65ac12
AH
159static struct malloc_pipe dmtc_read_mpipe;
160static struct malloc_pipe dmtc_write_mpipe;
161
162static void
163dmtc_init_mpipe(void)
164{
165 int nmax;
166
167 nmax = (physmem*5/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
168
169 if (nmax < 2)
170 nmax = 2;
171
172 kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
173
174 mpipe_init(&dmtc_write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
175 2, nmax, MPF_NOZERO, NULL);
176 mpipe_init(&dmtc_read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
177 2, nmax, MPF_NOZERO, NULL);
178}
179
180static void
181dmtc_destroy_mpipe(void)
182{
183 mpipe_done(&dmtc_write_mpipe);
184 mpipe_done(&dmtc_read_mpipe);
185}
186
f6221ad1 187/*
7cfe2226 188 * Overwrite private information (in buf) to avoid leaking it
f6221ad1 189 */
7cfe2226
AH
190static void
191dmtc_crypto_clear(void *buf, size_t len)
192{
193 memset(buf, 0xFF, len);
194 bzero(buf, len);
195}
196
197/*
198 * ESSIV IV Generator Routines
199 */
ba65ac12 200static int
7cfe2226 201essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
dd9da8fe 202{
7cfe2226
AH
203 struct essiv_ivgen_priv *ivpriv;
204 u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
48a0340e 205 unsigned int klen, hashlen;
7cfe2226 206 int error;
b559ed14
AH
207
208 klen = (priv->crypto_klen >> 3);
209
210 if (iv_hash == NULL)
211 return EINVAL;
212
213 if (!strcmp(iv_hash, "sha1")) {
214 SHA1_CTX ctx;
215
48a0340e 216 hashlen = SHA1_RESULTLEN;
b559ed14
AH
217 SHA1Init(&ctx);
218 SHA1Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 219 SHA1Final(crypto_keyhash, &ctx);
b559ed14
AH
220 } else if (!strcmp(iv_hash, "sha256")) {
221 SHA256_CTX ctx;
222
48a0340e 223 hashlen = SHA256_DIGEST_LENGTH;
b559ed14
AH
224 SHA256_Init(&ctx);
225 SHA256_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 226 SHA256_Final(crypto_keyhash, &ctx);
b559ed14
AH
227 } else if (!strcmp(iv_hash, "sha384")) {
228 SHA384_CTX ctx;
229
48a0340e 230 hashlen = SHA384_DIGEST_LENGTH;
b559ed14
AH
231 SHA384_Init(&ctx);
232 SHA384_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 233 SHA384_Final(crypto_keyhash, &ctx);
b559ed14
AH
234 } else if (!strcmp(iv_hash, "sha512")) {
235 SHA512_CTX ctx;
236
48a0340e 237 hashlen = SHA512_DIGEST_LENGTH;
b559ed14
AH
238 SHA512_Init(&ctx);
239 SHA512_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 240 SHA512_Final(crypto_keyhash, &ctx);
b559ed14
AH
241 } else if (!strcmp(iv_hash, "md5")) {
242 MD5_CTX ctx;
243
48a0340e 244 hashlen = MD5_DIGEST_LENGTH;
b559ed14
AH
245 MD5Init(&ctx);
246 MD5Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 247 MD5Final(crypto_keyhash, &ctx);
f6221ad1
MD
248 } else if (!strcmp(iv_hash, "rmd160") ||
249 !strcmp(iv_hash, "ripemd160")) {
b559ed14
AH
250 RMD160_CTX ctx;
251
48a0340e 252 hashlen = 160/8;
b559ed14
AH
253 RMD160Init(&ctx);
254 RMD160Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 255 RMD160Final(crypto_keyhash, &ctx);
dd9da8fe 256 } else {
b559ed14 257 return EINVAL;
dd9da8fe 258 }
b559ed14 259
48a0340e
AH
260 /* Convert hashlen to bits */
261 hashlen <<= 3;
262
7cfe2226
AH
263 ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
264 M_WAITOK | M_ZERO);
265 memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
266 ivpriv->keyhash_len = sizeof(crypto_keyhash);
267 dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
268
269 ivpriv->crypto_session.cri_alg = priv->crypto_alg;
270 ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
48a0340e 271 ivpriv->crypto_session.cri_klen = hashlen;
7cfe2226
AH
272 ivpriv->crypto_session.cri_mlen = 0;
273 ivpriv->crypto_session.cri_next = NULL;
274
48a0340e
AH
275 /*
276 * XXX: in principle we also need to check if the block size of the
277 * cipher is a valid iv size for the block cipher.
278 */
279
7cfe2226
AH
280 error = crypto_newsession(&ivpriv->crypto_sid,
281 &ivpriv->crypto_session,
282 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
283 if (error) {
284 kprintf("dm_target_crypt: Error during crypto_newsession "
285 "for essiv_ivgen, error = %d\n",
286 error);
287 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
288 kfree(ivpriv, M_DMCRYPT);
289 return ENOTSUP;
290 }
291
292 ivpriv->crp_crd_cache = objcache_create(
293 "dmcrypt-essiv-cache", 0, 0,
294 NULL, NULL, NULL,
295 objcache_malloc_alloc,
296 objcache_malloc_free,
297 &essiv_ivgen_malloc_args );
298
299 *p_ivpriv = ivpriv;
300 return 0;
301}
302
303static int
304essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
305{
306 struct essiv_ivgen_priv *ivpriv;
307
308 ivpriv = (struct essiv_ivgen_priv *)arg;
309 KKASSERT(ivpriv != NULL);
310
311 crypto_freesession(ivpriv->crypto_sid);
312
313 objcache_destroy(ivpriv->crp_crd_cache);
314
315 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
316 kfree(ivpriv, M_DMCRYPT);
317
b559ed14 318 return 0;
dd9da8fe
AH
319}
320
321static int
322essiv_ivgen_done(struct cryptop *crp)
323{
7cfe2226
AH
324 struct essiv_ivgen_priv *ivpriv;
325 void *free_addr;
326 void *opaque;
dd9da8fe 327
79b7159f 328
dd9da8fe
AH
329 if (crp->crp_etype == EAGAIN)
330 return crypto_dispatch(crp);
331
332 if (crp->crp_etype != 0) {
f6221ad1
MD
333 kprintf("dm_target_crypt: essiv_ivgen_done, "
334 "crp->crp_etype = %d\n", crp->crp_etype);
dd9da8fe
AH
335 }
336
7cfe2226
AH
337 free_addr = crp->crp_opaque;
338 /*
339 * In-memory structure is:
340 * | ivpriv | opaque | crp | crd |
341 * | (void *) | (void *) | (cryptop) | (cryptodesc) |
342 */
343 ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
344 crp->crp_opaque += sizeof(void *);
345 opaque = *((void **)crp->crp_opaque);
79b7159f 346
7cfe2226
AH
347 objcache_put(ivpriv->crp_crd_cache, free_addr);
348 dmtc_crypto_dispatch(opaque);
dd9da8fe
AH
349 return 0;
350}
351
352static void
f6221ad1 353essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
7cfe2226 354 size_t iv_len, off_t sector, void *opaque)
dd9da8fe 355{
7cfe2226
AH
356 struct essiv_ivgen_priv *ivpriv;
357 struct cryptodesc *crd;
358 struct cryptop *crp;
359 caddr_t space, alloc_addr;
360 int error;
361
362 ivpriv = priv->ivgen_priv;
363 KKASSERT(ivpriv != NULL);
364
365 /*
366 * In-memory structure is:
367 * | ivpriv | opaque | crp | crd |
368 * | (void *) | (void *) | (cryptop) | (cryptodesc) |
369 */
370 alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
371 *((struct essiv_ivgen_priv **)space) = ivpriv;
372 space += sizeof(void *);
373 *((void **)space) = opaque;
374 space += sizeof(void *);
375 crp = (struct cryptop *)space;
376 space += sizeof(struct cryptop);
377 crd = (struct cryptodesc *)space;
dd9da8fe 378
dd9da8fe 379 bzero(iv, iv_len);
7cfe2226
AH
380 bzero(crd, sizeof(struct cryptodesc));
381 bzero(crp, sizeof(struct cryptop));
ff3e5675 382 *((off_t *)iv) = htole64(sector + priv->iv_offset);
7cfe2226 383 crp->crp_buf = (caddr_t)iv;
dd9da8fe 384
7cfe2226
AH
385 crp->crp_sid = ivpriv->crypto_sid;
386 crp->crp_ilen = crp->crp_olen = iv_len;
dd9da8fe 387
7cfe2226 388 crp->crp_opaque = alloc_addr;
dd9da8fe 389
7cfe2226 390 crp->crp_callback = essiv_ivgen_done;
dd9da8fe 391
7cfe2226
AH
392 crp->crp_desc = crd;
393 crp->crp_etype = 0;
394 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
dd9da8fe 395
7cfe2226
AH
396 crd->crd_alg = priv->crypto_alg;
397#if 0
398 crd->crd_key = (caddr_t)priv->crypto_keyhash;
399 crd->crd_klen = priv->crypto_klen;
400#endif
dd9da8fe 401
7cfe2226 402 bzero(crd->crd_iv, sizeof(crd->crd_iv));
dd9da8fe 403
7cfe2226
AH
404 crd->crd_skip = 0;
405 crd->crd_len = iv_len;
406 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
407 crd->crd_flags |= CRD_F_ENCRYPT;
408 crd->crd_next = NULL;
dd9da8fe 409
7cfe2226 410 error = crypto_dispatch(crp);
dd9da8fe 411 if (error)
e8e2bcda 412 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
7cfe2226 413}
dd9da8fe 414
7cfe2226
AH
415
416static void
417plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
418 size_t iv_len, off_t sector, void *opaque)
419{
420 bzero(iv, iv_len);
7a89864a 421 *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
7cfe2226 422 dmtc_crypto_dispatch(opaque);
dd9da8fe
AH
423}
424
f6221ad1 425
7cfe2226 426#if 0
dd9da8fe 427static void
f6221ad1 428geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
7cfe2226 429 size_t iv_len, off_t sector, void *opaque)
dd9da8fe
AH
430{
431
432 SHA512_CTX ctx512;
433 u_int8_t md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
434
435 memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
436 SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
437 SHA512_Final(md, &ctx512);
438
439 memcpy(iv, md, iv_len);
7cfe2226 440 dmtc_crypto_dispatch(opaque);
dd9da8fe 441}
e8e2bcda 442#endif
dd9da8fe 443
dd9da8fe
AH
444/*
445 * Init function called from dm_table_load_ioctl.
b559ed14
AH
446 * cryptsetup actually passes us this:
447 * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
dd9da8fe 448 */
b559ed14 449static int
619de3db 450hex2key(char *hex, size_t key_len, u_int8_t *key)
b559ed14
AH
451{
452 char hex_buf[3];
453 size_t key_idx;
454
619de3db
MD
455 hex_buf[2] = 0;
456 for (key_idx = 0; key_idx < key_len; ++key_idx) {
b559ed14
AH
457 hex_buf[0] = *hex++;
458 hex_buf[1] = *hex++;
619de3db 459 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
b559ed14 460 }
619de3db
MD
461 hex_buf[0] = 0;
462 hex_buf[1] = 0;
b559ed14
AH
463
464 return 0;
465}
466
ba65ac12 467static int
dd9da8fe
AH
468dm_target_crypt_init(dm_dev_t * dmv, void **target_config, char *params)
469{
470 dm_target_crypt_config_t *priv;
471 size_t len;
472 char **ap, *args[5];
b559ed14
AH
473 char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
474 char *status_str;
7cfe2226 475 int i, argc, klen, error;
b559ed14 476 uint64_t iv_offset, block_offset;
dd9da8fe
AH
477
478 if (params == NULL)
479 return EINVAL;
480
481 len = strlen(params) + 1;
482 argc = 0;
483
dec22111 484 status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
dd9da8fe
AH
485 /*
486 * Parse a string, containing tokens delimited by white space,
487 * into an argument vector
488 */
b559ed14 489 for (ap = args; ap < &args[5] &&
dd9da8fe
AH
490 (*ap = strsep(&params, " \t")) != NULL;) {
491 if (**ap != '\0') {
492 argc++;
493 ap++;
494 }
495 }
496
b559ed14 497 if (argc != 5) {
f6221ad1
MD
498 kprintf("dm_target_crypt: not enough arguments, "
499 "need exactly 5\n");
b559ed14 500 kfree(status_str, M_DMCRYPT);
dd9da8fe
AH
501 return ENOMEM; /* XXX */
502 }
503
b559ed14
AH
504 crypto_alg = strsep(&args[0], "-");
505 crypto_mode = strsep(&args[0], "-");
506 iv_opt = strsep(&args[0], "-");
507 iv_mode = strsep(&iv_opt, ":");
508 key = args[1];
509 iv_offset = strtouq(args[2], NULL, 0);
510 dev = args[3];
511 block_offset = strtouq(args[4], NULL, 0);
512 /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
513 klen = strlen(key) << 2;
514
dec22111 515#if 0
7cfe2226 516 kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
f6221ad1
MD
517 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
518 "block_offset=%ju\n",
b559ed14
AH
519 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
520 block_offset);
dec22111 521#endif
dd9da8fe 522
1446934e
MD
523 priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
524 if (priv == NULL) {
e8e2bcda 525 kprintf("dm_target_crypt: could not allocate memory\n");
b559ed14 526 kfree(status_str, M_DMCRYPT);
dd9da8fe
AH
527 return ENOMEM;
528 }
529
530 /* Insert dmp to global pdev list */
b559ed14 531 if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
e8e2bcda 532 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
b559ed14 533 kfree(status_str, M_DMCRYPT);
dd9da8fe
AH
534 return ENOENT;
535 }
536
4d1658f1
AH
537 if ((strcmp(crypto_mode, "cbc") != 0) &&
538 !((strcmp(crypto_mode, "xts") == 0) && (strcmp(crypto_alg, "aes") == 0)))
178da765
AH
539 {
540 kprintf("dm_target_crypt: only support 'cbc' chaining mode"
541 " and aes-xts, invalid mode '%s-%s'\n",
542 crypto_alg, crypto_mode);
b559ed14
AH
543 goto notsup;
544 }
545
546 if (!strcmp(crypto_alg, "aes")) {
178da765
AH
547 if (!strcmp(crypto_mode, "xts")) {
548 priv->crypto_alg = CRYPTO_AES_XTS;
549 if (klen != 256 && klen != 512)
550 goto notsup;
551 } else if (!strcmp(crypto_mode, "cbc")) {
552 priv->crypto_alg = CRYPTO_AES_CBC;
553 if (klen != 128 && klen != 192 && klen != 256)
554 goto notsup;
555 } else {
dd9da8fe 556 goto notsup;
178da765 557 }
dd9da8fe 558 priv->crypto_klen = klen;
b559ed14 559 } else if (!strcmp(crypto_alg, "blowfish")) {
dd9da8fe 560 priv->crypto_alg = CRYPTO_BLF_CBC;
dd9da8fe
AH
561 if (klen < 128 || klen > 448 || (klen % 8) != 0)
562 goto notsup;
563 priv->crypto_klen = klen;
f6221ad1
MD
564 } else if (!strcmp(crypto_alg, "3des") ||
565 !strncmp(crypto_alg, "des3", 4)) {
dd9da8fe 566 priv->crypto_alg = CRYPTO_3DES_CBC;
b559ed14
AH
567 if (klen != 168)
568 goto notsup;
dd9da8fe 569 priv->crypto_klen = 168;
b559ed14 570 } else if (!strcmp(crypto_alg, "camellia")) {
dd9da8fe 571 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
dd9da8fe
AH
572 if (klen != 128 && klen != 192 && klen != 256)
573 goto notsup;
574 priv->crypto_klen = klen;
b559ed14 575 } else if (!strcmp(crypto_alg, "skipjack")) {
dd9da8fe 576 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
b559ed14
AH
577 if (klen != 80)
578 goto notsup;
dd9da8fe 579 priv->crypto_klen = 80;
b559ed14 580 } else if (!strcmp(crypto_alg, "cast5")) {
dd9da8fe 581 priv->crypto_alg = CRYPTO_CAST_CBC;
b559ed14
AH
582 if (klen != 128)
583 goto notsup;
dd9da8fe 584 priv->crypto_klen = 128;
b559ed14 585 } else if (!strcmp(crypto_alg, "null")) {
dd9da8fe 586 priv->crypto_alg = CRYPTO_NULL_CBC;
b559ed14
AH
587 if (klen != 128)
588 goto notsup;
dd9da8fe 589 priv->crypto_klen = 128;
b559ed14 590 } else {
f6221ad1
MD
591 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
592 crypto_alg);
b559ed14 593 goto notsup;
dd9da8fe
AH
594 }
595
596 /* Save length of param string */
597 priv->params_len = len;
b559ed14 598 priv->block_offset = block_offset;
7a89864a 599 priv->iv_offset = iv_offset - block_offset;
dd9da8fe
AH
600
601 *target_config = priv;
602
603 dmv->dev_type = DM_CRYPTO_DEV;
604
f6221ad1
MD
605 error = hex2key(key, priv->crypto_klen >> 3,
606 (u_int8_t *)priv->crypto_key);
607
b559ed14 608 if (error) {
f6221ad1
MD
609 kprintf("dm_target_crypt: hex2key failed, "
610 "invalid key format\n");
b559ed14
AH
611 goto notsup;
612 }
dd9da8fe 613
7cfe2226
AH
614 /* Handle cmd */
615 for(i = 0; ivgens[i].name != NULL; i++) {
616 if (!strcmp(iv_mode, ivgens[i].name))
617 break;
618 }
619
620 if (ivgens[i].name == NULL) {
621 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
622 iv_mode);
623 goto notsup;
624 }
625
626 /* Call our ivgen constructor */
627 if (ivgens[i].ctor != NULL) {
628 error = ivgens[i].ctor(priv, iv_opt,
629 &priv->ivgen_priv);
b559ed14 630 if (error) {
7cfe2226
AH
631 kprintf("dm_target_crypt: ctor for '%s' failed\n",
632 ivgens[i].name);
aadb5a11 633 goto notsup;
b559ed14 634 }
dd9da8fe
AH
635 }
636
7cfe2226
AH
637 priv->ivgen = &ivgens[i];
638
639 priv->crypto_session.cri_alg = priv->crypto_alg;
dd9da8fe 640 priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
7cfe2226
AH
641 priv->crypto_session.cri_klen = priv->crypto_klen;
642 priv->crypto_session.cri_mlen = 0;
dd9da8fe
AH
643 priv->crypto_session.cri_next = NULL;
644
645 error = crypto_newsession(&priv->crypto_sid,
646 &priv->crypto_session,
647 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
648 if (error) {
f6221ad1
MD
649 kprintf("dm_target_crypt: Error during crypto_newsession, "
650 "error = %d\n",
651 error);
b559ed14 652 goto notsup;
dd9da8fe
AH
653 }
654
dec22111
AH
655 memset(key, '0', strlen(key));
656 if (iv_opt) {
657 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
658 crypto_alg, crypto_mode, iv_mode, iv_opt,
659 key, iv_offset, dev, block_offset);
660 } else {
661 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
662 crypto_alg, crypto_mode, iv_mode,
663 key, iv_offset, dev, block_offset);
664 }
b559ed14 665 priv->status_str = status_str;
dec22111 666
dd9da8fe
AH
667 return 0;
668
669notsup:
e8e2bcda 670 kprintf("dm_target_crypt: ENOTSUP\n");
b559ed14 671 kfree(status_str, M_DMCRYPT);
dd9da8fe
AH
672 return ENOTSUP;
673}
674
675/* Status routine called to get params string. */
ba65ac12 676static char *
dd9da8fe
AH
677dm_target_crypt_status(void *target_config)
678{
679 dm_target_crypt_config_t *priv;
680 char *params;
681
682 priv = target_config;
683
1446934e
MD
684 /* caller expects use of M_DM */
685 params = kmalloc(DM_MAX_PARAMS_SIZE, M_DM, M_WAITOK);
dd9da8fe
AH
686
687 ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
e8e2bcda 688 priv->status_str);
dd9da8fe
AH
689
690 return params;
691}
692
ba65ac12 693static int
f6221ad1
MD
694dm_target_crypt_destroy(dm_table_entry_t * table_en)
695{
696 dm_target_crypt_config_t *priv;
697
698 /*
699 * Disconnect the crypt config before unbusying the target.
700 */
701 priv = table_en->target_config;
702 if (priv == NULL)
703 return 0;
704 table_en->target_config = NULL;
705 dm_pdev_decr(priv->pdev);
706
707 dm_target_unbusy(table_en->target);
708
709 /*
710 * Clean up the crypt config
711 *
712 * Overwrite the private information before freeing memory to
713 * avoid leaking it.
714 */
48a0340e
AH
715 if (priv->status_str) {
716 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
717 kfree(priv->status_str, M_DMCRYPT);
718 crypto_freesession(priv->crypto_sid);
719 }
7cfe2226 720
48a0340e 721 if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
7cfe2226
AH
722 priv->ivgen->dtor(priv, priv->ivgen_priv);
723 }
724
725 dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
f6221ad1
MD
726 kfree(priv, M_DMCRYPT);
727
728 return 0;
729}
730
ba65ac12 731static int
f6221ad1
MD
732dm_target_crypt_deps(dm_table_entry_t * table_en, prop_array_t prop_array)
733{
734 dm_target_crypt_config_t *priv;
735 struct vattr va;
736
737 int error;
738
739 if (table_en->target_config == NULL)
740 return ENOENT;
741
742 priv = table_en->target_config;
743
744 if ((error = VOP_GETATTR(priv->pdev->pdev_vnode, &va)) != 0)
745 return error;
746
747 prop_array_add_uint64(prop_array,
748 (uint64_t)makeudev(va.va_rmajor, va.va_rminor));
749
750 return 0;
751}
752
753/* Unsupported for this target. */
ba65ac12 754static int
f6221ad1
MD
755dm_target_crypt_upcall(dm_table_entry_t * table_en, struct buf * bp)
756{
757 return 0;
758}
759
760/************************************************************************
761 * STRATEGY SUPPORT FUNCTIONS *
762 ************************************************************************
763 *
764 * READ PATH: doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
765 * WRITE PATH: crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
766 */
767
dd9da8fe 768/*
7cfe2226
AH
769 * Wrapper around crypto_dispatch() to match dispatch_t type
770 */
771static void
772dmtc_crypto_dispatch(void *arg)
773{
774 struct cryptop *crp;
775
776 crp = (struct cryptop *)arg;
777 KKASSERT(crp != NULL);
778 crypto_dispatch(crp);
779}
780
781/*
f6221ad1 782 * Start IO operation, called from dmstrategy routine.
dd9da8fe 783 */
ba65ac12 784static int
f6221ad1 785dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
dd9da8fe
AH
786{
787 struct bio *bio;
788
789 dm_target_crypt_config_t *priv;
790 priv = table_en->target_config;
791
792 /* Get rid of stuff we can't really handle */
793 if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
794 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
f6221ad1
MD
795 kprintf("dm_target_crypt_strategy: can't really "
796 "handle bp->b_bcount = %d\n",
797 bp->b_bcount);
dd9da8fe
AH
798 bp->b_error = EINVAL;
799 bp->b_flags |= B_ERROR | B_INVAL;
800 biodone(&bp->b_bio1);
801 return 0;
802 }
803 }
804
805 switch (bp->b_cmd) {
806 case BUF_CMD_READ:
807 bio = push_bio(&bp->b_bio1);
f6221ad1
MD
808 bio->bio_offset = bp->b_bio1.bio_offset +
809 priv->block_offset * DEV_BSIZE;
dd9da8fe 810 bio->bio_caller_info1.ptr = priv;
ff3e5675 811 bio->bio_done = dmtc_bio_read_done;
dd9da8fe
AH
812 vn_strategy(priv->pdev->pdev_vnode, bio);
813 break;
dd9da8fe
AH
814 case BUF_CMD_WRITE:
815 bio = push_bio(&bp->b_bio1);
f6221ad1
MD
816 bio->bio_offset = bp->b_bio1.bio_offset +
817 priv->block_offset * DEV_BSIZE;
dd9da8fe 818 bio->bio_caller_info1.ptr = priv;
f6221ad1 819 dmtc_crypto_write_start(priv, bio);
dd9da8fe 820 break;
dd9da8fe 821 default:
aadb5a11 822 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
f6221ad1 823 break;
dd9da8fe 824 }
dd9da8fe 825 return 0;
dd9da8fe
AH
826}
827
f6221ad1
MD
828/*
829 * STRATEGY READ PATH PART 1/3 (after read BIO completes)
830 */
831static void
832dmtc_bio_read_done(struct bio *bio)
dd9da8fe 833{
f6221ad1
MD
834 struct bio *obio;
835
dd9da8fe
AH
836 dm_target_crypt_config_t *priv;
837
6730d2b7 838 /*
f6221ad1
MD
839 * If a read error occurs we shortcut the operation, otherwise
840 * go on to stage 2.
6730d2b7 841 */
f6221ad1
MD
842 if (bio->bio_buf->b_flags & B_ERROR) {
843 obio = pop_bio(bio);
844 biodone(obio);
845 } else {
846 priv = bio->bio_caller_info1.ptr;
847 dmtc_crypto_read_start(priv, bio);
848 }
849}
dd9da8fe 850
f6221ad1
MD
851/*
852 * STRATEGY READ PATH PART 2/3
853 */
854static void
855dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
856{
857 struct dmtc_helper *dmtc;
858 struct cryptodesc *crd;
859 struct cryptop *crp;
860 struct cryptoini *cri;
ff4a7c8b
MD
861 int i, bytes, sectors, sz;
862 off_t isector;
f6221ad1
MD
863 u_char *ptr, *space;
864
865 cri = &priv->crypto_session;
dd9da8fe 866
ff3e5675 867 /*
f6221ad1
MD
868 * Note: b_resid no good after read I/O, it will be 0, use
869 * b_bcount.
ff3e5675 870 */
f6221ad1 871 bytes = bio->bio_buf->b_bcount;
ff4a7c8b 872 isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
f6221ad1
MD
873 sectors = bytes / DEV_BSIZE; /* Number of sectors */
874 sz = sectors * (sizeof(*crp) + sizeof(*crd));
6730d2b7 875
f6221ad1
MD
876 /*
877 * For reads with bogus page we can't decrypt in place as stuff
878 * can get ripped out from under us.
c98abe57
MD
879 *
880 * XXX actually it looks like we can, and in any case the initial
881 * read already completed and threw crypted data into the buffer
882 * cache buffer. Disable for now.
f6221ad1 883 */
c98abe57 884#if 0
f6221ad1
MD
885 if (bio->bio_buf->b_flags & B_HASBOGUS) {
886 space = kmalloc(sizeof(struct dmtc_helper) + sz + bytes,
887 M_DMCRYPT, M_WAITOK);
888 dmtc = (struct dmtc_helper *)space;
889 dmtc->free_addr = space;
890 space += sizeof(struct dmtc_helper);
c98abe57 891 dmtc->orig_buf = NULL;
f6221ad1 892 dmtc->data_buf = space + sz;
c98abe57
MD
893 memcpy(dmtc->data_buf, bio->bio_buf->b_data, bytes);
894 } else
895#endif
896 {
ba65ac12 897 space = mpipe_alloc_waitok(&dmtc_read_mpipe);
f6221ad1
MD
898 dmtc = (struct dmtc_helper *)space;
899 dmtc->free_addr = space;
900 space += sizeof(struct dmtc_helper);
c98abe57 901 dmtc->orig_buf = NULL;
f6221ad1
MD
902 dmtc->data_buf = bio->bio_buf->b_data;
903 }
904 bio->bio_caller_info2.ptr = dmtc;
905 bio->bio_buf->b_error = 0;
906
907 /*
908 * Load crypto descriptors (crp/crd loop)
909 */
c98abe57 910 bzero(space, sz);
f6221ad1
MD
911 ptr = space;
912 bio->bio_caller_info3.value = sectors;
c98abe57 913 cpu_sfence();
f6221ad1
MD
914#if 0
915 kprintf("Read, bytes = %d (b_bcount), "
916 "sectors = %d (bio = %p, b_cmd = %d)\n",
917 bytes, sectors, bio, bio->bio_buf->b_cmd);
918#endif
919 for (i = 0; i < sectors; i++) {
920 crp = (struct cryptop *)ptr;
921 ptr += sizeof(*crp);
922 crd = (struct cryptodesc *)ptr;
923 ptr += sizeof (*crd);
924
925 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
926
927 crp->crp_sid = priv->crypto_sid;
928 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
929
930 crp->crp_opaque = (void *)bio;
931
932 crp->crp_callback = dmtc_crypto_cb_read_done;
933 crp->crp_desc = crd;
934 crp->crp_etype = 0;
57ffb118
MD
935 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
936 CRYPTO_F_BATCH;
f6221ad1
MD
937
938 crd->crd_alg = priv->crypto_alg;
7cfe2226 939#if 0
f6221ad1
MD
940 crd->crd_key = (caddr_t)priv->crypto_key;
941 crd->crd_klen = priv->crypto_klen;
7cfe2226 942#endif
f6221ad1
MD
943
944 crd->crd_skip = 0;
945 crd->crd_len = DEV_BSIZE /* XXX */;
7cfe2226 946 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
f6221ad1
MD
947 crd->crd_next = NULL;
948
949 crd->crd_flags &= ~CRD_F_ENCRYPT;
950
7cfe2226
AH
951 /*
952 * Note: last argument is used to generate salt(?) and is
953 * a 64 bit value, but the original code passed an
954 * int. Changing it now will break pre-existing
955 * crypt volumes.
956 */
957 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
ff4a7c8b 958 isector + i, crp);
f6221ad1
MD
959 }
960}
961
962/*
963 * STRATEGY READ PATH PART 3/3
964 */
965static int
966dmtc_crypto_cb_read_done(struct cryptop *crp)
967{
968 struct dmtc_helper *dmtc;
969 struct bio *bio, *obio;
970 int n;
971
972 if (crp->crp_etype == EAGAIN)
973 return crypto_dispatch(crp);
974
975 bio = (struct bio *)crp->crp_opaque;
976 KKASSERT(bio != NULL);
977
f6221ad1
MD
978 /*
979 * Cumulative error
980 */
981 if (crp->crp_etype) {
982 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
983 "crp_etype = %d\n",
984 crp->crp_etype);
985 bio->bio_buf->b_error = crp->crp_etype;
986 }
dd9da8fe 987
f6221ad1
MD
988 /*
989 * On the last chunk of the decryption we do any required copybacks
990 * and complete the I/O.
991 */
c98abe57
MD
992 n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
993#if 0
994 kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
995#endif
996
f6221ad1
MD
997 if (n == 1) {
998 /*
999 * For the B_HASBOGUS case we didn't decrypt in place,
1000 * so we need to copy stuff back into the buf.
c98abe57
MD
1001 *
1002 * (disabled for now).
f6221ad1
MD
1003 */
1004 dmtc = bio->bio_caller_info2.ptr;
1005 if (bio->bio_buf->b_error) {
1006 bio->bio_buf->b_flags |= B_ERROR;
c98abe57
MD
1007 }
1008#if 0
1009 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
f6221ad1
MD
1010 memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1011 bio->bio_buf->b_bcount);
1012 }
c98abe57 1013#endif
ba65ac12 1014 mpipe_free(&dmtc_read_mpipe, dmtc->free_addr);
f6221ad1
MD
1015 obio = pop_bio(bio);
1016 biodone(obio);
1017 }
dd9da8fe
AH
1018 return 0;
1019}
f6221ad1 1020/* END OF STRATEGY READ SECTION */
dd9da8fe 1021
f6221ad1
MD
1022/*
1023 * STRATEGY WRITE PATH PART 1/3
1024 */
1025static void
1026dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1027{
1028 struct dmtc_helper *dmtc;
1029 struct cryptodesc *crd;
1030 struct cryptop *crp;
1031 struct cryptoini *cri;
ff4a7c8b
MD
1032 int i, bytes, sectors, sz;
1033 off_t isector;
f6221ad1
MD
1034 u_char *ptr, *space;
1035
1036 cri = &priv->crypto_session;
1037
1038 /*
1039 * Use b_bcount for consistency
1040 */
1041 bytes = bio->bio_buf->b_bcount;
1042
ff4a7c8b 1043 isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
f6221ad1
MD
1044 sectors = bytes / DEV_BSIZE; /* Number of sectors */
1045 sz = sectors * (sizeof(*crp) + sizeof(*crd));
1046
1047 /*
1048 * For writes and reads with bogus page don't decrypt in place.
1049 */
ba65ac12 1050 space = mpipe_alloc_waitok(&dmtc_write_mpipe);
f6221ad1
MD
1051 dmtc = (struct dmtc_helper *)space;
1052 dmtc->free_addr = space;
1053 space += sizeof(struct dmtc_helper);
1054 memcpy(space + sz, bio->bio_buf->b_data, bytes);
1055
1056 bio->bio_caller_info2.ptr = dmtc;
1057 bio->bio_buf->b_error = 0;
1058
1059 dmtc->orig_buf = bio->bio_buf->b_data;
1060 dmtc->data_buf = space + sz;
1061
1062 /*
1063 * Load crypto descriptors (crp/crd loop)
1064 */
c98abe57 1065 bzero(space, sz);
f6221ad1
MD
1066 ptr = space;
1067 bio->bio_caller_info3.value = sectors;
c98abe57 1068 cpu_sfence();
f6221ad1
MD
1069#if 0
1070 kprintf("Write, bytes = %d (b_bcount), "
1071 "sectors = %d (bio = %p, b_cmd = %d)\n",
1072 bytes, sectors, bio, bio->bio_buf->b_cmd);
1073#endif
1074 for (i = 0; i < sectors; i++) {
1075 crp = (struct cryptop *)ptr;
1076 ptr += sizeof(*crp);
1077 crd = (struct cryptodesc *)ptr;
1078 ptr += sizeof (*crd);
1079
1080 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1081
1082 crp->crp_sid = priv->crypto_sid;
1083 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1084
1085 crp->crp_opaque = (void *)bio;
1086
1087 crp->crp_callback = dmtc_crypto_cb_write_done;
1088 crp->crp_desc = crd;
1089 crp->crp_etype = 0;
57ffb118
MD
1090 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1091 CRYPTO_F_BATCH;
f6221ad1
MD
1092
1093 crd->crd_alg = priv->crypto_alg;
7cfe2226 1094#if 0
f6221ad1
MD
1095 crd->crd_key = (caddr_t)priv->crypto_key;
1096 crd->crd_klen = priv->crypto_klen;
7cfe2226
AH
1097#endif
1098
1099 crd->crd_skip = 0;
1100 crd->crd_len = DEV_BSIZE /* XXX */;
1101 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1102 crd->crd_next = NULL;
1103
1104 crd->crd_flags |= CRD_F_ENCRYPT;
f6221ad1 1105
c98abe57
MD
1106 /*
1107 * Note: last argument is used to generate salt(?) and is
1108 * a 64 bit value, but the original code passed an
1109 * int. Changing it now will break pre-existing
1110 * crypt volumes.
1111 */
7cfe2226 1112 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
ff4a7c8b 1113 isector + i, crp);
f6221ad1
MD
1114 }
1115}
1116
1117/*
1118 * STRATEGY WRITE PATH PART 2/3
1119 */
1120static int
1121dmtc_crypto_cb_write_done(struct cryptop *crp)
dd9da8fe 1122{
f6221ad1 1123 struct dmtc_helper *dmtc;
dd9da8fe 1124 dm_target_crypt_config_t *priv;
f6221ad1
MD
1125 struct bio *bio, *obio;
1126 int n;
dd9da8fe 1127
f6221ad1
MD
1128 if (crp->crp_etype == EAGAIN)
1129 return crypto_dispatch(crp);
dd9da8fe 1130
f6221ad1
MD
1131 bio = (struct bio *)crp->crp_opaque;
1132 KKASSERT(bio != NULL);
dd9da8fe 1133
f6221ad1
MD
1134 /*
1135 * Cumulative error
1136 */
1137 if (crp->crp_etype != 0) {
1138 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1139 "crp_etype = %d\n",
1140 crp->crp_etype);
1141 bio->bio_buf->b_error = crp->crp_etype;
1142 }
dd9da8fe 1143
f6221ad1
MD
1144 /*
1145 * On the last chunk of the encryption we issue the write
1146 */
c98abe57
MD
1147 n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1148#if 0
1149 kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1150#endif
1151
f6221ad1
MD
1152 if (n == 1) {
1153 dmtc = bio->bio_caller_info2.ptr;
1154 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
dd9da8fe 1155
f6221ad1
MD
1156 if (bio->bio_buf->b_error) {
1157 bio->bio_buf->b_flags |= B_ERROR;
ba65ac12 1158 mpipe_free(&dmtc_write_mpipe, dmtc->free_addr);
f6221ad1
MD
1159 obio = pop_bio(bio);
1160 biodone(obio);
1161 } else {
1162 dmtc->orig_buf = bio->bio_buf->b_data;
1163 bio->bio_buf->b_data = dmtc->data_buf;
1164 bio->bio_done = dmtc_bio_write_done;
1165 vn_strategy(priv->pdev->pdev_vnode, bio);
1166 }
1167 }
dd9da8fe
AH
1168 return 0;
1169}
1170
f6221ad1
MD
1171/*
1172 * STRATEGY WRITE PATH PART 3/3
1173 */
1174static void
1175dmtc_bio_write_done(struct bio *bio)
dd9da8fe 1176{
f6221ad1
MD
1177 struct dmtc_helper *dmtc;
1178 struct bio *obio;
1179
1180 dmtc = bio->bio_caller_info2.ptr;
1181 bio->bio_buf->b_data = dmtc->orig_buf;
ba65ac12 1182 mpipe_free(&dmtc_write_mpipe, dmtc->free_addr);
f6221ad1
MD
1183 obio = pop_bio(bio);
1184 biodone(obio);
dd9da8fe 1185}
f6221ad1 1186/* END OF STRATEGY WRITE SECTION */
79b7159f
AH
1187
1188
1189
1190/* DUMPING MAGIC */
1191
1192extern int tsleep_crypto_dump;
1193
ba65ac12 1194static int
79b7159f
AH
1195dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1196{
1197 static struct dmtc_dump_helper dump_helper;
1198 dm_target_crypt_config_t *priv;
1199 int id;
1200 static int first_call = 1;
1201
1202 priv = table_en->target_config;
1203
1204 if (first_call) {
1205 first_call = 0;
1206 dump_reactivate_cpus();
1207 }
1208
1209 /* Magically enable tsleep */
1210 tsleep_crypto_dump = 1;
1211 id = 0;
1212
1213 /*
1214 * 0 length means flush buffers and return
1215 */
1216 if (length == 0) {
1217 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1218 tsleep_crypto_dump = 0;
1219 return ENXIO;
1220 }
1221 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1222 data, 0, offset, 0);
1223 tsleep_crypto_dump = 0;
1224 return 0;
1225 }
1226
1227 bzero(&dump_helper, sizeof(dump_helper));
1228 dump_helper.priv = priv;
1229 dump_helper.data = data;
1230 dump_helper.length = length;
1231 dump_helper.offset = offset +
1232 priv->block_offset * DEV_BSIZE;
1233 dump_helper.ident = &id;
1234 dmtc_crypto_dump_start(priv, &dump_helper);
1235
1236 /*
1237 * Hackery to make stuff appear synchronous. The crypto callback will
1238 * set id to 1 and call wakeup on it. If the request completed
1239 * synchronously, id will be 1 and we won't bother to sleep. If not,
1240 * the crypto request will complete asynchronously and we sleep until
1241 * it's done.
1242 */
1243 if (id == 0)
1244 tsleep(&dump_helper, 0, "cryptdump", 0);
1245
1246 dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1247 dump_helper.offset);
1248
1249 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1250 dump_helper.space, 0, dump_helper.offset,
1251 dump_helper.length);
1252
1253 tsleep_crypto_dump = 0;
1254 return 0;
1255}
1256
1257static void
1258dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1259{
1260 struct cryptodesc *crd;
1261 struct cryptop *crp;
1262 struct cryptoini *cri;
1263 int i, bytes, sectors;
1264 off_t isector;
1265
1266 cri = &priv->crypto_session;
1267
1268 bytes = dump_helper->length;
1269
1270 isector = dump_helper->offset / DEV_BSIZE; /* ivgen salt base? */
1271 sectors = bytes / DEV_BSIZE; /* Number of sectors */
1272 dump_helper->sectors = sectors;
1273#if 0
1274 kprintf("Dump, bytes = %d, "
1275 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1276#endif
1277 KKASSERT(dump_helper->length <= 65536);
1278
1279 memcpy(dump_helper->space, dump_helper->data, bytes);
1280
1281 cpu_sfence();
1282
1283 for (i = 0; i < sectors; i++) {
1284 crp = &dump_helper->crp[i];
1285 crd = &dump_helper->crd[i];
1286
1287 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1288
1289 crp->crp_sid = priv->crypto_sid;
1290 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1291
1292 crp->crp_opaque = (void *)dump_helper;
1293
1294 crp->crp_callback = dmtc_crypto_cb_dump_done;
1295 crp->crp_desc = crd;
1296 crp->crp_etype = 0;
1297 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1298 CRYPTO_F_BATCH;
1299
1300 crd->crd_alg = priv->crypto_alg;
1301
1302 crd->crd_skip = 0;
1303 crd->crd_len = DEV_BSIZE /* XXX */;
1304 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1305 crd->crd_next = NULL;
1306
1307 crd->crd_flags |= CRD_F_ENCRYPT;
1308
1309 /*
1310 * Note: last argument is used to generate salt(?) and is
1311 * a 64 bit value, but the original code passed an
1312 * int. Changing it now will break pre-existing
1313 * crypt volumes.
1314 */
1315 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1316 isector + i, crp);
1317 }
1318}
1319
1320static int
1321dmtc_crypto_cb_dump_done(struct cryptop *crp)
1322{
1323 struct dmtc_dump_helper *dump_helper;
1324 dm_target_crypt_config_t *priv;
1325 int n;
1326
1327 if (crp->crp_etype == EAGAIN)
1328 return crypto_dispatch(crp);
1329
1330 dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1331 KKASSERT(dump_helper != NULL);
1332
1333 if (crp->crp_etype != 0) {
1334 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1335 "crp_etype = %d\n",
1336 crp->crp_etype);
1337 return crp->crp_etype;
1338 }
1339
1340 /*
1341 * On the last chunk of the encryption we return control
1342 */
1343 n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1344
1345 if (n == 1) {
1346 priv = (dm_target_crypt_config_t *)dump_helper->priv;
1347 atomic_add_int(dump_helper->ident, 1);
1348 wakeup(dump_helper);
1349 }
1350
1351 return 0;
1352}
ba65ac12
AH
1353
1354static int
1355dmtc_mod_handler(module_t mod, int type, void *unused)
1356{
1357 dm_target_t *dmt = NULL;
1358 int err = 0;
1359
1360 switch (type) {
1361 case MOD_LOAD:
1362 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1363 dm_target_unbusy(dmt);
1364 return EEXIST;
1365 }
1366 dmt = dm_target_alloc("crypt");
1367 dmt->version[0] = 1;
1368 dmt->version[1] = 6;
1369 dmt->version[2] = 0;
1370 strlcpy(dmt->name, "crypt", DM_MAX_TYPE_NAME);
1371 dmt->init = &dm_target_crypt_init;
1372 dmt->status = &dm_target_crypt_status;
1373 dmt->strategy = &dm_target_crypt_strategy;
1374 dmt->deps = &dm_target_crypt_deps;
1375 dmt->destroy = &dm_target_crypt_destroy;
1376 dmt->upcall = &dm_target_crypt_upcall;
1377 dmt->dump = &dm_target_crypt_dump;
1378
1379 dmtc_init_mpipe();
1380
1381 err = dm_target_insert(dmt);
1382 if (err)
1383 dmtc_destroy_mpipe();
5c411e8e
AH
1384 else
1385 kprintf("dm_target_crypt: Successfully initialized\n");
ba65ac12
AH
1386 break;
1387
1388 case MOD_UNLOAD:
1389 err = dm_target_rem("crypt");
7115a22b 1390 if (err == 0) {
ba65ac12 1391 dmtc_destroy_mpipe();
7115a22b
AH
1392 kprintf("dm_target_crypt: unloaded\n");
1393 }
ba65ac12
AH
1394 break;
1395
1396 default:
1397 break;
1398 }
1399
1400 return err;
1401}
1402
1403DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);