Modify ktr(4) to be typesafe
[dragonfly.git] / sys / dev / disk / dm / targets / crypt / dm_target_crypt.c
CommitLineData
dd9da8fe
AH
1/*
2 * Copyright (c) 2010 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Alex Hornung <ahornung@gmail.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35/*
36 * This file implements initial version of device-mapper crypt target.
37 */
38#include <sys/types.h>
39#include <sys/param.h>
ff3e5675 40#include <sys/endian.h>
dd9da8fe
AH
41
42#include <sys/bio.h>
43#include <sys/buf.h>
79b7159f
AH
44#include <sys/globaldata.h>
45#include <sys/kerneldump.h>
dd9da8fe 46#include <sys/malloc.h>
ba65ac12 47#include <sys/mpipe.h>
b559ed14 48#include <sys/md5.h>
79b7159f 49#include <sys/mutex2.h>
dd9da8fe
AH
50#include <sys/vnode.h>
51#include <crypto/sha1.h>
52#include <crypto/sha2/sha2.h>
53#include <opencrypto/cryptodev.h>
b559ed14 54#include <opencrypto/rmd160.h>
c98abe57 55#include <machine/cpufunc.h>
dd9da8fe 56
d536b408
AH
57#include <sys/ktr.h>
58
a84e173e 59#include <dev/disk/dm/dm.h>
dd9da8fe
AH
60MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
61
d536b408
AH
62KTR_INFO_MASTER(dmcrypt);
63
64#if !defined(KTR_DMCRYPT)
65#define KTR_DMCRYPT KTR_ALL
66#endif
67
68KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_dispatch, 0,
5bf48697 69 "crypto_dispatch(%p)", struct cryptop *crp);
d536b408 70KTR_INFO(KTR_DMCRYPT, dmcrypt, crypt_strategy, 0,
5bf48697 71 "crypt_strategy(b_cmd = %d, bp = %p)", int cmd, struct buf *bp);
d536b408
AH
72KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_start, 1,
73 "crypto_write_start(crp = %p, bp = %p, sector = %d/%d)",
5bf48697 74 struct cryptop *crp, struct buf *bp, int i, int sectors);
d536b408
AH
75KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_write_done, 1,
76 "crypto_cb_write_done(crp = %p, bp = %p, n = %d)",
5bf48697 77 struct cryptop *crp, struct buf *bp, int n);
d536b408 78KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_write_done, 1,
5bf48697 79 "bio_write_done(bp = %p)", struct buf *bp);
d536b408 80KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_write_retry, 1,
5bf48697 81 "crypto_write_retry(crp = %p)", struct buf *bp);
d536b408 82KTR_INFO(KTR_DMCRYPT, dmcrypt, bio_read_done, 2,
5bf48697 83 "bio_read_done(bp = %p)", struct buf *bp);
d536b408
AH
84KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_read_start, 2,
85 "crypto_read_start(crp = %p, bp = %p, sector = %d/%d)",
5bf48697 86 struct cryptop *crp, struct buf *bp, int i, int sectors);
d536b408
AH
87KTR_INFO(KTR_DMCRYPT, dmcrypt, crypto_cb_read_done, 2,
88 "crypto_cb_read_done(crp = %p, bp = %p, n = %d)",
5bf48697 89 struct cryptop *crp, struct buf *bp, int n);
d536b408 90
dd9da8fe 91struct target_crypt_config;
7cfe2226
AH
92
93typedef void dispatch_t(void *);
94typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
95 void *);
96
97typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
98typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
99
100struct iv_generator {
101 const char *name;
102 ivgen_ctor_t *ctor;
103 ivgen_dtor_t *dtor;
104 ivgen_t *gen_iv;
105};
106
107struct essiv_ivgen_priv {
108 struct cryptoini crypto_session;
109 struct objcache *crp_crd_cache;
110 u_int64_t crypto_sid;
111 size_t keyhash_len;
112 u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
113};
dd9da8fe
AH
114
115typedef struct target_crypt_config {
116 size_t params_len;
117 dm_pdev_t *pdev;
aadb5a11 118 char *status_str;
dd9da8fe
AH
119 int crypto_alg;
120 int crypto_klen;
121 u_int8_t crypto_key[512>>3];
79b7159f 122
dd9da8fe 123 u_int64_t crypto_sid;
b559ed14 124 u_int64_t block_offset;
48a0340e 125 int64_t iv_offset;
dd9da8fe 126 SHA512_CTX essivsha512_ctx;
7cfe2226 127
dd9da8fe 128 struct cryptoini crypto_session;
7cfe2226
AH
129
130 struct iv_generator *ivgen;
131 void *ivgen_priv;
d536b408
AH
132
133 struct malloc_pipe read_mpipe;
134 struct malloc_pipe write_mpipe;
dd9da8fe
AH
135} dm_target_crypt_config_t;
136
b559ed14 137struct dmtc_helper {
d536b408 138 dm_target_crypt_config_t *priv;
b559ed14
AH
139 caddr_t free_addr;
140 caddr_t orig_buf;
f6221ad1 141 caddr_t data_buf;
b559ed14
AH
142};
143
79b7159f
AH
144struct dmtc_dump_helper {
145 dm_target_crypt_config_t *priv;
146 void *data;
147 size_t length;
148 off_t offset;
149
150 int sectors;
151 int *ident;
152
153 struct cryptodesc crd[128];
154 struct cryptop crp[128];
155 u_char space[65536];
156};
157
ba65ac12
AH
158#define DMTC_BUF_SIZE_WRITE \
159 MAXPHYS + sizeof(struct dmtc_helper) + \
160 MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
161#define DMTC_BUF_SIZE_READ \
162 sizeof(struct dmtc_helper) + \
163 MAXPHYS/DEV_BSIZE*(sizeof(struct cryptop) + sizeof(struct cryptodesc))
164
7cfe2226 165static void dmtc_crypto_dispatch(void *arg);
79b7159f
AH
166static void dmtc_crypto_dump_start(dm_target_crypt_config_t *priv,
167 struct dmtc_dump_helper *dump_helper);
f6221ad1
MD
168static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
169 struct bio *bio);
170static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
171 struct bio *bio);
ff3e5675
AH
172static void dmtc_bio_read_done(struct bio *bio);
173static void dmtc_bio_write_done(struct bio *bio);
79b7159f 174static int dmtc_crypto_cb_dump_done(struct cryptop *crp);
ff3e5675
AH
175static int dmtc_crypto_cb_read_done(struct cryptop *crp);
176static int dmtc_crypto_cb_write_done(struct cryptop *crp);
dd9da8fe 177
7cfe2226
AH
178static ivgen_ctor_t essiv_ivgen_ctor;
179static ivgen_dtor_t essiv_ivgen_dtor;
180static ivgen_t essiv_ivgen;
181static ivgen_t plain_ivgen;
9678bbf5 182static ivgen_t plain64_ivgen;
7cfe2226
AH
183
184static struct iv_generator ivgens[] = {
185 { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
186 .gen_iv = essiv_ivgen },
187 { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
9678bbf5 188 { .name = "plain64", .ctor = NULL, .dtor = NULL, .gen_iv = plain64_ivgen },
7cfe2226
AH
189 { NULL, NULL, NULL, NULL }
190};
191
192struct objcache_malloc_args essiv_ivgen_malloc_args = {
193 2*sizeof(void *) + (sizeof(struct cryptodesc) +
194 sizeof(struct cryptop)), M_DMCRYPT };
195
ba65ac12 196static void
d536b408 197dmtc_init_mpipe(struct target_crypt_config *priv)
ba65ac12
AH
198{
199 int nmax;
200
d536b408 201 nmax = (physmem*2/1000*PAGE_SIZE)/(DMTC_BUF_SIZE_WRITE + DMTC_BUF_SIZE_READ) + 1;
ba65ac12
AH
202
203 if (nmax < 2)
204 nmax = 2;
205
206 kprintf("dm_target_crypt: Setting min/max mpipe buffers: %d/%d\n", 2, nmax);
207
d536b408 208 mpipe_init(&priv->write_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_WRITE,
c9f7645b 209 2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
d536b408 210 mpipe_init(&priv->read_mpipe, M_DMCRYPT, DMTC_BUF_SIZE_READ,
c9f7645b 211 2, nmax, MPF_NOZERO | MPF_CALLBACK, NULL, NULL, NULL);
ba65ac12
AH
212}
213
214static void
d536b408 215dmtc_destroy_mpipe(struct target_crypt_config *priv)
ba65ac12 216{
d536b408
AH
217 mpipe_done(&priv->write_mpipe);
218 mpipe_done(&priv->read_mpipe);
ba65ac12
AH
219}
220
f6221ad1 221/*
7cfe2226 222 * Overwrite private information (in buf) to avoid leaking it
f6221ad1 223 */
7cfe2226
AH
224static void
225dmtc_crypto_clear(void *buf, size_t len)
226{
227 memset(buf, 0xFF, len);
228 bzero(buf, len);
229}
230
231/*
232 * ESSIV IV Generator Routines
233 */
ba65ac12 234static int
7cfe2226 235essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
dd9da8fe 236{
7cfe2226
AH
237 struct essiv_ivgen_priv *ivpriv;
238 u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
48a0340e 239 unsigned int klen, hashlen;
7cfe2226 240 int error;
b559ed14
AH
241
242 klen = (priv->crypto_klen >> 3);
243
244 if (iv_hash == NULL)
245 return EINVAL;
246
247 if (!strcmp(iv_hash, "sha1")) {
248 SHA1_CTX ctx;
249
48a0340e 250 hashlen = SHA1_RESULTLEN;
b559ed14
AH
251 SHA1Init(&ctx);
252 SHA1Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 253 SHA1Final(crypto_keyhash, &ctx);
b559ed14
AH
254 } else if (!strcmp(iv_hash, "sha256")) {
255 SHA256_CTX ctx;
256
48a0340e 257 hashlen = SHA256_DIGEST_LENGTH;
b559ed14
AH
258 SHA256_Init(&ctx);
259 SHA256_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 260 SHA256_Final(crypto_keyhash, &ctx);
b559ed14
AH
261 } else if (!strcmp(iv_hash, "sha384")) {
262 SHA384_CTX ctx;
263
48a0340e 264 hashlen = SHA384_DIGEST_LENGTH;
b559ed14
AH
265 SHA384_Init(&ctx);
266 SHA384_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 267 SHA384_Final(crypto_keyhash, &ctx);
b559ed14
AH
268 } else if (!strcmp(iv_hash, "sha512")) {
269 SHA512_CTX ctx;
270
48a0340e 271 hashlen = SHA512_DIGEST_LENGTH;
b559ed14
AH
272 SHA512_Init(&ctx);
273 SHA512_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 274 SHA512_Final(crypto_keyhash, &ctx);
b559ed14
AH
275 } else if (!strcmp(iv_hash, "md5")) {
276 MD5_CTX ctx;
277
48a0340e 278 hashlen = MD5_DIGEST_LENGTH;
b559ed14
AH
279 MD5Init(&ctx);
280 MD5Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 281 MD5Final(crypto_keyhash, &ctx);
f6221ad1
MD
282 } else if (!strcmp(iv_hash, "rmd160") ||
283 !strcmp(iv_hash, "ripemd160")) {
b559ed14
AH
284 RMD160_CTX ctx;
285
48a0340e 286 hashlen = 160/8;
b559ed14
AH
287 RMD160Init(&ctx);
288 RMD160Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
7cfe2226 289 RMD160Final(crypto_keyhash, &ctx);
dd9da8fe 290 } else {
b559ed14 291 return EINVAL;
dd9da8fe 292 }
b559ed14 293
48a0340e
AH
294 /* Convert hashlen to bits */
295 hashlen <<= 3;
296
7cfe2226
AH
297 ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
298 M_WAITOK | M_ZERO);
299 memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
300 ivpriv->keyhash_len = sizeof(crypto_keyhash);
301 dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
302
303 ivpriv->crypto_session.cri_alg = priv->crypto_alg;
304 ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
48a0340e 305 ivpriv->crypto_session.cri_klen = hashlen;
7cfe2226
AH
306 ivpriv->crypto_session.cri_mlen = 0;
307 ivpriv->crypto_session.cri_next = NULL;
308
48a0340e
AH
309 /*
310 * XXX: in principle we also need to check if the block size of the
311 * cipher is a valid iv size for the block cipher.
312 */
313
7cfe2226
AH
314 error = crypto_newsession(&ivpriv->crypto_sid,
315 &ivpriv->crypto_session,
316 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
317 if (error) {
318 kprintf("dm_target_crypt: Error during crypto_newsession "
319 "for essiv_ivgen, error = %d\n",
320 error);
321 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
322 kfree(ivpriv, M_DMCRYPT);
323 return ENOTSUP;
324 }
325
326 ivpriv->crp_crd_cache = objcache_create(
327 "dmcrypt-essiv-cache", 0, 0,
328 NULL, NULL, NULL,
329 objcache_malloc_alloc,
330 objcache_malloc_free,
331 &essiv_ivgen_malloc_args );
332
333 *p_ivpriv = ivpriv;
334 return 0;
335}
336
337static int
338essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
339{
340 struct essiv_ivgen_priv *ivpriv;
341
342 ivpriv = (struct essiv_ivgen_priv *)arg;
343 KKASSERT(ivpriv != NULL);
344
345 crypto_freesession(ivpriv->crypto_sid);
346
347 objcache_destroy(ivpriv->crp_crd_cache);
348
349 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
350 kfree(ivpriv, M_DMCRYPT);
351
b559ed14 352 return 0;
dd9da8fe
AH
353}
354
355static int
356essiv_ivgen_done(struct cryptop *crp)
357{
7cfe2226
AH
358 struct essiv_ivgen_priv *ivpriv;
359 void *free_addr;
360 void *opaque;
dd9da8fe 361
79b7159f 362
dd9da8fe
AH
363 if (crp->crp_etype == EAGAIN)
364 return crypto_dispatch(crp);
365
366 if (crp->crp_etype != 0) {
f6221ad1
MD
367 kprintf("dm_target_crypt: essiv_ivgen_done, "
368 "crp->crp_etype = %d\n", crp->crp_etype);
dd9da8fe
AH
369 }
370
7cfe2226
AH
371 free_addr = crp->crp_opaque;
372 /*
373 * In-memory structure is:
374 * | ivpriv | opaque | crp | crd |
375 * | (void *) | (void *) | (cryptop) | (cryptodesc) |
376 */
377 ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
378 crp->crp_opaque += sizeof(void *);
379 opaque = *((void **)crp->crp_opaque);
79b7159f 380
7cfe2226
AH
381 objcache_put(ivpriv->crp_crd_cache, free_addr);
382 dmtc_crypto_dispatch(opaque);
dd9da8fe
AH
383 return 0;
384}
385
386static void
f6221ad1 387essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
7cfe2226 388 size_t iv_len, off_t sector, void *opaque)
dd9da8fe 389{
7cfe2226
AH
390 struct essiv_ivgen_priv *ivpriv;
391 struct cryptodesc *crd;
392 struct cryptop *crp;
393 caddr_t space, alloc_addr;
394 int error;
395
396 ivpriv = priv->ivgen_priv;
397 KKASSERT(ivpriv != NULL);
398
399 /*
400 * In-memory structure is:
401 * | ivpriv | opaque | crp | crd |
402 * | (void *) | (void *) | (cryptop) | (cryptodesc) |
403 */
404 alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
405 *((struct essiv_ivgen_priv **)space) = ivpriv;
406 space += sizeof(void *);
407 *((void **)space) = opaque;
408 space += sizeof(void *);
409 crp = (struct cryptop *)space;
410 space += sizeof(struct cryptop);
411 crd = (struct cryptodesc *)space;
dd9da8fe 412
dd9da8fe 413 bzero(iv, iv_len);
7cfe2226
AH
414 bzero(crd, sizeof(struct cryptodesc));
415 bzero(crp, sizeof(struct cryptop));
ff3e5675 416 *((off_t *)iv) = htole64(sector + priv->iv_offset);
7cfe2226 417 crp->crp_buf = (caddr_t)iv;
dd9da8fe 418
7cfe2226
AH
419 crp->crp_sid = ivpriv->crypto_sid;
420 crp->crp_ilen = crp->crp_olen = iv_len;
dd9da8fe 421
7cfe2226 422 crp->crp_opaque = alloc_addr;
dd9da8fe 423
7cfe2226 424 crp->crp_callback = essiv_ivgen_done;
dd9da8fe 425
7cfe2226
AH
426 crp->crp_desc = crd;
427 crp->crp_etype = 0;
428 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
dd9da8fe 429
7cfe2226
AH
430 crd->crd_alg = priv->crypto_alg;
431#if 0
432 crd->crd_key = (caddr_t)priv->crypto_keyhash;
433 crd->crd_klen = priv->crypto_klen;
434#endif
dd9da8fe 435
7cfe2226 436 bzero(crd->crd_iv, sizeof(crd->crd_iv));
dd9da8fe 437
7cfe2226
AH
438 crd->crd_skip = 0;
439 crd->crd_len = iv_len;
440 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
441 crd->crd_flags |= CRD_F_ENCRYPT;
442 crd->crd_next = NULL;
dd9da8fe 443
7cfe2226 444 error = crypto_dispatch(crp);
dd9da8fe 445 if (error)
e8e2bcda 446 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
7cfe2226 447}
dd9da8fe 448
7cfe2226
AH
449
450static void
451plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
452 size_t iv_len, off_t sector, void *opaque)
453{
454 bzero(iv, iv_len);
7a89864a 455 *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
7cfe2226 456 dmtc_crypto_dispatch(opaque);
dd9da8fe
AH
457}
458
9678bbf5
AH
459static void
460plain64_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
461 size_t iv_len, off_t sector, void *opaque)
462{
463 bzero(iv, iv_len);
464 *((uint64_t *)iv) = htole64((uint64_t)(sector + priv->iv_offset));
465 dmtc_crypto_dispatch(opaque);
466}
f6221ad1 467
7cfe2226 468#if 0
dd9da8fe 469static void
f6221ad1 470geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
7cfe2226 471 size_t iv_len, off_t sector, void *opaque)
dd9da8fe
AH
472{
473
474 SHA512_CTX ctx512;
475 u_int8_t md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
476
477 memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
478 SHA512_Update(&ctx512, (u_int8_t*)&sector, sizeof(off_t));
479 SHA512_Final(md, &ctx512);
480
481 memcpy(iv, md, iv_len);
7cfe2226 482 dmtc_crypto_dispatch(opaque);
dd9da8fe 483}
e8e2bcda 484#endif
dd9da8fe 485
dd9da8fe
AH
486/*
487 * Init function called from dm_table_load_ioctl.
b559ed14
AH
488 * cryptsetup actually passes us this:
489 * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
dd9da8fe 490 */
b559ed14 491static int
619de3db 492hex2key(char *hex, size_t key_len, u_int8_t *key)
b559ed14
AH
493{
494 char hex_buf[3];
495 size_t key_idx;
496
619de3db
MD
497 hex_buf[2] = 0;
498 for (key_idx = 0; key_idx < key_len; ++key_idx) {
b559ed14
AH
499 hex_buf[0] = *hex++;
500 hex_buf[1] = *hex++;
619de3db 501 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
b559ed14 502 }
619de3db
MD
503 hex_buf[0] = 0;
504 hex_buf[1] = 0;
b559ed14
AH
505
506 return 0;
507}
508
ba65ac12 509static int
dd9da8fe
AH
510dm_target_crypt_init(dm_dev_t * dmv, void **target_config, char *params)
511{
512 dm_target_crypt_config_t *priv;
513 size_t len;
514 char **ap, *args[5];
b559ed14
AH
515 char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
516 char *status_str;
7cfe2226 517 int i, argc, klen, error;
b559ed14 518 uint64_t iv_offset, block_offset;
dd9da8fe
AH
519
520 if (params == NULL)
521 return EINVAL;
522
523 len = strlen(params) + 1;
524 argc = 0;
525
dec22111 526 status_str = kmalloc(len, M_DMCRYPT, M_WAITOK);
dd9da8fe
AH
527 /*
528 * Parse a string, containing tokens delimited by white space,
529 * into an argument vector
530 */
b559ed14 531 for (ap = args; ap < &args[5] &&
dd9da8fe
AH
532 (*ap = strsep(&params, " \t")) != NULL;) {
533 if (**ap != '\0') {
534 argc++;
535 ap++;
536 }
537 }
538
b559ed14 539 if (argc != 5) {
f6221ad1
MD
540 kprintf("dm_target_crypt: not enough arguments, "
541 "need exactly 5\n");
b559ed14 542 kfree(status_str, M_DMCRYPT);
dd9da8fe
AH
543 return ENOMEM; /* XXX */
544 }
545
b559ed14
AH
546 crypto_alg = strsep(&args[0], "-");
547 crypto_mode = strsep(&args[0], "-");
548 iv_opt = strsep(&args[0], "-");
549 iv_mode = strsep(&iv_opt, ":");
550 key = args[1];
551 iv_offset = strtouq(args[2], NULL, 0);
552 dev = args[3];
553 block_offset = strtouq(args[4], NULL, 0);
554 /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
555 klen = strlen(key) << 2;
556
dec22111 557#if 0
7cfe2226 558 kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
f6221ad1
MD
559 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
560 "block_offset=%ju\n",
b559ed14
AH
561 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
562 block_offset);
dec22111 563#endif
dd9da8fe 564
1446934e 565 priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
dd9da8fe
AH
566
567 /* Insert dmp to global pdev list */
b559ed14 568 if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
e8e2bcda 569 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
b559ed14 570 kfree(status_str, M_DMCRYPT);
dd9da8fe
AH
571 return ENOENT;
572 }
573
ee41ac19
AH
574 /*
575 * This code checks for valid combinations of algorithm and mode.
576 * Currently supported options are:
577 *
578 * *-cbc
579 * aes-xts
580 * twofish-xts
581 * serpent-xts
582 */
4d1658f1 583 if ((strcmp(crypto_mode, "cbc") != 0) &&
ee41ac19
AH
584 !((strcmp(crypto_mode, "xts") == 0) &&
585 ((strcmp(crypto_alg, "aes") == 0) ||
586 (strcmp(crypto_alg, "twofish") == 0) ||
587 (strcmp(crypto_alg, "serpent") == 0))))
178da765 588 {
ee41ac19
AH
589 kprintf("dm_target_crypt: only support 'cbc' chaining mode,"
590 " aes-xts, twofish-xts and serpent-xts, "
591 "invalid mode '%s-%s'\n",
178da765 592 crypto_alg, crypto_mode);
b559ed14
AH
593 goto notsup;
594 }
595
596 if (!strcmp(crypto_alg, "aes")) {
178da765
AH
597 if (!strcmp(crypto_mode, "xts")) {
598 priv->crypto_alg = CRYPTO_AES_XTS;
599 if (klen != 256 && klen != 512)
600 goto notsup;
601 } else if (!strcmp(crypto_mode, "cbc")) {
602 priv->crypto_alg = CRYPTO_AES_CBC;
603 if (klen != 128 && klen != 192 && klen != 256)
604 goto notsup;
605 } else {
dd9da8fe 606 goto notsup;
178da765 607 }
dd9da8fe 608 priv->crypto_klen = klen;
ee41ac19
AH
609 } else if (!strcmp(crypto_alg, "twofish")) {
610 if (!strcmp(crypto_mode, "xts")) {
611 priv->crypto_alg = CRYPTO_TWOFISH_XTS;
612 if (klen != 256 && klen != 512)
613 goto notsup;
614 } else if (!strcmp(crypto_mode, "cbc")) {
615 priv->crypto_alg = CRYPTO_TWOFISH_CBC;
616 if (klen != 128 && klen != 192 && klen != 256)
617 goto notsup;
618 } else {
619 goto notsup;
620 }
621 priv->crypto_klen = klen;
622 } else if (!strcmp(crypto_alg, "serpent")) {
623 if (!strcmp(crypto_mode, "xts")) {
624 priv->crypto_alg = CRYPTO_SERPENT_XTS;
625 if (klen != 256 && klen != 512)
626 goto notsup;
627 } else if (!strcmp(crypto_mode, "cbc")) {
628 priv->crypto_alg = CRYPTO_SERPENT_CBC;
629 if (klen != 128 && klen != 192 && klen != 256)
630 goto notsup;
631 } else {
632 goto notsup;
633 }
634 priv->crypto_klen = klen;
b559ed14 635 } else if (!strcmp(crypto_alg, "blowfish")) {
dd9da8fe 636 priv->crypto_alg = CRYPTO_BLF_CBC;
dd9da8fe
AH
637 if (klen < 128 || klen > 448 || (klen % 8) != 0)
638 goto notsup;
639 priv->crypto_klen = klen;
f6221ad1
MD
640 } else if (!strcmp(crypto_alg, "3des") ||
641 !strncmp(crypto_alg, "des3", 4)) {
dd9da8fe 642 priv->crypto_alg = CRYPTO_3DES_CBC;
b559ed14
AH
643 if (klen != 168)
644 goto notsup;
dd9da8fe 645 priv->crypto_klen = 168;
b559ed14 646 } else if (!strcmp(crypto_alg, "camellia")) {
dd9da8fe 647 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
dd9da8fe
AH
648 if (klen != 128 && klen != 192 && klen != 256)
649 goto notsup;
650 priv->crypto_klen = klen;
b559ed14 651 } else if (!strcmp(crypto_alg, "skipjack")) {
dd9da8fe 652 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
b559ed14
AH
653 if (klen != 80)
654 goto notsup;
dd9da8fe 655 priv->crypto_klen = 80;
b559ed14 656 } else if (!strcmp(crypto_alg, "cast5")) {
dd9da8fe 657 priv->crypto_alg = CRYPTO_CAST_CBC;
b559ed14
AH
658 if (klen != 128)
659 goto notsup;
dd9da8fe 660 priv->crypto_klen = 128;
b559ed14 661 } else if (!strcmp(crypto_alg, "null")) {
dd9da8fe 662 priv->crypto_alg = CRYPTO_NULL_CBC;
b559ed14
AH
663 if (klen != 128)
664 goto notsup;
dd9da8fe 665 priv->crypto_klen = 128;
b559ed14 666 } else {
f6221ad1
MD
667 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
668 crypto_alg);
b559ed14 669 goto notsup;
dd9da8fe
AH
670 }
671
672 /* Save length of param string */
673 priv->params_len = len;
b559ed14 674 priv->block_offset = block_offset;
7a89864a 675 priv->iv_offset = iv_offset - block_offset;
dd9da8fe
AH
676
677 *target_config = priv;
678
679 dmv->dev_type = DM_CRYPTO_DEV;
680
f6221ad1
MD
681 error = hex2key(key, priv->crypto_klen >> 3,
682 (u_int8_t *)priv->crypto_key);
683
b559ed14 684 if (error) {
f6221ad1
MD
685 kprintf("dm_target_crypt: hex2key failed, "
686 "invalid key format\n");
b559ed14
AH
687 goto notsup;
688 }
dd9da8fe 689
7cfe2226
AH
690 /* Handle cmd */
691 for(i = 0; ivgens[i].name != NULL; i++) {
692 if (!strcmp(iv_mode, ivgens[i].name))
693 break;
694 }
695
696 if (ivgens[i].name == NULL) {
697 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
698 iv_mode);
699 goto notsup;
700 }
701
702 /* Call our ivgen constructor */
703 if (ivgens[i].ctor != NULL) {
704 error = ivgens[i].ctor(priv, iv_opt,
705 &priv->ivgen_priv);
b559ed14 706 if (error) {
7cfe2226
AH
707 kprintf("dm_target_crypt: ctor for '%s' failed\n",
708 ivgens[i].name);
aadb5a11 709 goto notsup;
b559ed14 710 }
dd9da8fe
AH
711 }
712
7cfe2226
AH
713 priv->ivgen = &ivgens[i];
714
715 priv->crypto_session.cri_alg = priv->crypto_alg;
dd9da8fe 716 priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
7cfe2226
AH
717 priv->crypto_session.cri_klen = priv->crypto_klen;
718 priv->crypto_session.cri_mlen = 0;
dd9da8fe
AH
719 priv->crypto_session.cri_next = NULL;
720
721 error = crypto_newsession(&priv->crypto_sid,
722 &priv->crypto_session,
723 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
724 if (error) {
f6221ad1
MD
725 kprintf("dm_target_crypt: Error during crypto_newsession, "
726 "error = %d\n",
727 error);
b559ed14 728 goto notsup;
dd9da8fe
AH
729 }
730
dec22111
AH
731 memset(key, '0', strlen(key));
732 if (iv_opt) {
733 ksprintf(status_str, "%s-%s-%s:%s %s %ju %s %ju",
734 crypto_alg, crypto_mode, iv_mode, iv_opt,
735 key, iv_offset, dev, block_offset);
736 } else {
737 ksprintf(status_str, "%s-%s-%s %s %ju %s %ju",
738 crypto_alg, crypto_mode, iv_mode,
739 key, iv_offset, dev, block_offset);
740 }
b559ed14 741 priv->status_str = status_str;
dec22111 742
d536b408
AH
743 /* Initialize mpipes */
744 dmtc_init_mpipe(priv);
745
dd9da8fe
AH
746 return 0;
747
748notsup:
e8e2bcda 749 kprintf("dm_target_crypt: ENOTSUP\n");
b559ed14 750 kfree(status_str, M_DMCRYPT);
dd9da8fe
AH
751 return ENOTSUP;
752}
753
754/* Status routine called to get params string. */
ba65ac12 755static char *
dd9da8fe
AH
756dm_target_crypt_status(void *target_config)
757{
758 dm_target_crypt_config_t *priv;
759 char *params;
760
761 priv = target_config;
762
1446934e
MD
763 /* caller expects use of M_DM */
764 params = kmalloc(DM_MAX_PARAMS_SIZE, M_DM, M_WAITOK);
dd9da8fe
AH
765
766 ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
e8e2bcda 767 priv->status_str);
dd9da8fe
AH
768
769 return params;
770}
771
ba65ac12 772static int
f6221ad1
MD
773dm_target_crypt_destroy(dm_table_entry_t * table_en)
774{
775 dm_target_crypt_config_t *priv;
776
777 /*
778 * Disconnect the crypt config before unbusying the target.
779 */
780 priv = table_en->target_config;
781 if (priv == NULL)
782 return 0;
783 table_en->target_config = NULL;
784 dm_pdev_decr(priv->pdev);
785
f6221ad1
MD
786 /*
787 * Clean up the crypt config
788 *
789 * Overwrite the private information before freeing memory to
790 * avoid leaking it.
791 */
48a0340e
AH
792 if (priv->status_str) {
793 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
794 kfree(priv->status_str, M_DMCRYPT);
795 crypto_freesession(priv->crypto_sid);
796 }
7cfe2226 797
48a0340e 798 if ((priv->ivgen) && (priv->ivgen->dtor != NULL)) {
7cfe2226
AH
799 priv->ivgen->dtor(priv, priv->ivgen_priv);
800 }
801
d536b408
AH
802 /* Destroy mpipes */
803 dmtc_destroy_mpipe(priv);
804
7cfe2226 805 dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
f6221ad1
MD
806 kfree(priv, M_DMCRYPT);
807
808 return 0;
809}
810
ba65ac12 811static int
f6221ad1
MD
812dm_target_crypt_deps(dm_table_entry_t * table_en, prop_array_t prop_array)
813{
814 dm_target_crypt_config_t *priv;
815 struct vattr va;
816
817 int error;
818
819 if (table_en->target_config == NULL)
820 return ENOENT;
821
822 priv = table_en->target_config;
823
824 if ((error = VOP_GETATTR(priv->pdev->pdev_vnode, &va)) != 0)
825 return error;
826
827 prop_array_add_uint64(prop_array,
828 (uint64_t)makeudev(va.va_rmajor, va.va_rminor));
829
830 return 0;
831}
832
833/* Unsupported for this target. */
ba65ac12 834static int
f6221ad1
MD
835dm_target_crypt_upcall(dm_table_entry_t * table_en, struct buf * bp)
836{
837 return 0;
838}
839
840/************************************************************************
841 * STRATEGY SUPPORT FUNCTIONS *
842 ************************************************************************
843 *
844 * READ PATH: doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
845 * WRITE PATH: crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
846 */
847
dd9da8fe 848/*
7cfe2226
AH
849 * Wrapper around crypto_dispatch() to match dispatch_t type
850 */
851static void
852dmtc_crypto_dispatch(void *arg)
853{
854 struct cryptop *crp;
855
856 crp = (struct cryptop *)arg;
857 KKASSERT(crp != NULL);
d536b408 858 KTR_LOG(dmcrypt_crypto_dispatch, crp);
7cfe2226
AH
859 crypto_dispatch(crp);
860}
d536b408 861
7cfe2226 862/*
f6221ad1 863 * Start IO operation, called from dmstrategy routine.
dd9da8fe 864 */
ba65ac12 865static int
f6221ad1 866dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
dd9da8fe
AH
867{
868 struct bio *bio;
869
870 dm_target_crypt_config_t *priv;
871 priv = table_en->target_config;
872
873 /* Get rid of stuff we can't really handle */
874 if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
875 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
f6221ad1
MD
876 kprintf("dm_target_crypt_strategy: can't really "
877 "handle bp->b_bcount = %d\n",
878 bp->b_bcount);
dd9da8fe
AH
879 bp->b_error = EINVAL;
880 bp->b_flags |= B_ERROR | B_INVAL;
881 biodone(&bp->b_bio1);
882 return 0;
883 }
884 }
885
d536b408
AH
886 KTR_LOG(dmcrypt_crypt_strategy, bp->b_cmd, bp);
887
dd9da8fe
AH
888 switch (bp->b_cmd) {
889 case BUF_CMD_READ:
890 bio = push_bio(&bp->b_bio1);
f6221ad1
MD
891 bio->bio_offset = bp->b_bio1.bio_offset +
892 priv->block_offset * DEV_BSIZE;
dd9da8fe 893 bio->bio_caller_info1.ptr = priv;
ff3e5675 894 bio->bio_done = dmtc_bio_read_done;
dd9da8fe
AH
895 vn_strategy(priv->pdev->pdev_vnode, bio);
896 break;
dd9da8fe
AH
897 case BUF_CMD_WRITE:
898 bio = push_bio(&bp->b_bio1);
f6221ad1
MD
899 bio->bio_offset = bp->b_bio1.bio_offset +
900 priv->block_offset * DEV_BSIZE;
dd9da8fe 901 bio->bio_caller_info1.ptr = priv;
f6221ad1 902 dmtc_crypto_write_start(priv, bio);
dd9da8fe 903 break;
dd9da8fe 904 default:
aadb5a11 905 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
f6221ad1 906 break;
dd9da8fe 907 }
dd9da8fe 908 return 0;
dd9da8fe
AH
909}
910
f6221ad1
MD
911/*
912 * STRATEGY READ PATH PART 1/3 (after read BIO completes)
913 */
914static void
915dmtc_bio_read_done(struct bio *bio)
dd9da8fe 916{
f6221ad1
MD
917 struct bio *obio;
918
dd9da8fe
AH
919 dm_target_crypt_config_t *priv;
920
d536b408
AH
921 KTR_LOG(dmcrypt_bio_read_done, bio->bio_buf);
922
6730d2b7 923 /*
f6221ad1
MD
924 * If a read error occurs we shortcut the operation, otherwise
925 * go on to stage 2.
6730d2b7 926 */
f6221ad1
MD
927 if (bio->bio_buf->b_flags & B_ERROR) {
928 obio = pop_bio(bio);
929 biodone(obio);
930 } else {
931 priv = bio->bio_caller_info1.ptr;
932 dmtc_crypto_read_start(priv, bio);
933 }
934}
dd9da8fe 935
f6221ad1
MD
936/*
937 * STRATEGY READ PATH PART 2/3
938 */
939static void
c9f7645b
MD
940dmtc_crypto_read_retry(void *arg1, void *arg2)
941{
942 dm_target_crypt_config_t *priv = arg1;
943 struct bio *bio = arg2;
944
945 dmtc_crypto_read_start(priv, bio);
946}
947
948static void
f6221ad1
MD
949dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
950{
951 struct dmtc_helper *dmtc;
952 struct cryptodesc *crd;
953 struct cryptop *crp;
954 struct cryptoini *cri;
ff4a7c8b
MD
955 int i, bytes, sectors, sz;
956 off_t isector;
f6221ad1
MD
957 u_char *ptr, *space;
958
959 cri = &priv->crypto_session;
dd9da8fe 960
ff3e5675 961 /*
f6221ad1
MD
962 * Note: b_resid no good after read I/O, it will be 0, use
963 * b_bcount.
ff3e5675 964 */
f6221ad1 965 bytes = bio->bio_buf->b_bcount;
ff4a7c8b 966 isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
f6221ad1
MD
967 sectors = bytes / DEV_BSIZE; /* Number of sectors */
968 sz = sectors * (sizeof(*crp) + sizeof(*crd));
6730d2b7 969
f6221ad1
MD
970 /*
971 * For reads with bogus page we can't decrypt in place as stuff
972 * can get ripped out from under us.
c98abe57
MD
973 *
974 * XXX actually it looks like we can, and in any case the initial
975 * read already completed and threw crypted data into the buffer
976 * cache buffer. Disable for now.
f6221ad1 977 */
d536b408 978 space = mpipe_alloc_callback(&priv->read_mpipe,
c9f7645b
MD
979 dmtc_crypto_read_retry, priv, bio);
980 if (space == NULL)
981 return;
982
983 dmtc = (struct dmtc_helper *)space;
984 dmtc->free_addr = space;
985 space += sizeof(struct dmtc_helper);
986 dmtc->orig_buf = NULL;
987 dmtc->data_buf = bio->bio_buf->b_data;
d536b408 988 dmtc->priv = priv;
f6221ad1
MD
989 bio->bio_caller_info2.ptr = dmtc;
990 bio->bio_buf->b_error = 0;
991
992 /*
993 * Load crypto descriptors (crp/crd loop)
994 */
c98abe57 995 bzero(space, sz);
f6221ad1
MD
996 ptr = space;
997 bio->bio_caller_info3.value = sectors;
c98abe57 998 cpu_sfence();
f6221ad1
MD
999#if 0
1000 kprintf("Read, bytes = %d (b_bcount), "
1001 "sectors = %d (bio = %p, b_cmd = %d)\n",
1002 bytes, sectors, bio, bio->bio_buf->b_cmd);
1003#endif
1004 for (i = 0; i < sectors; i++) {
1005 crp = (struct cryptop *)ptr;
1006 ptr += sizeof(*crp);
1007 crd = (struct cryptodesc *)ptr;
1008 ptr += sizeof (*crd);
1009
1010 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1011
1012 crp->crp_sid = priv->crypto_sid;
1013 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1014
1015 crp->crp_opaque = (void *)bio;
1016
1017 crp->crp_callback = dmtc_crypto_cb_read_done;
1018 crp->crp_desc = crd;
1019 crp->crp_etype = 0;
57ffb118
MD
1020 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1021 CRYPTO_F_BATCH;
f6221ad1
MD
1022
1023 crd->crd_alg = priv->crypto_alg;
7cfe2226 1024#if 0
f6221ad1
MD
1025 crd->crd_key = (caddr_t)priv->crypto_key;
1026 crd->crd_klen = priv->crypto_klen;
7cfe2226 1027#endif
f6221ad1
MD
1028
1029 crd->crd_skip = 0;
1030 crd->crd_len = DEV_BSIZE /* XXX */;
7cfe2226 1031 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
f6221ad1
MD
1032 crd->crd_next = NULL;
1033
1034 crd->crd_flags &= ~CRD_F_ENCRYPT;
1035
d536b408
AH
1036 KTR_LOG(dmcrypt_crypto_read_start, crp, bio->bio_buf, i,
1037 sectors);
1038
7cfe2226
AH
1039 /*
1040 * Note: last argument is used to generate salt(?) and is
1041 * a 64 bit value, but the original code passed an
1042 * int. Changing it now will break pre-existing
1043 * crypt volumes.
1044 */
1045 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
ff4a7c8b 1046 isector + i, crp);
f6221ad1
MD
1047 }
1048}
1049
1050/*
1051 * STRATEGY READ PATH PART 3/3
1052 */
1053static int
1054dmtc_crypto_cb_read_done(struct cryptop *crp)
1055{
1056 struct dmtc_helper *dmtc;
1057 struct bio *bio, *obio;
1058 int n;
1059
1060 if (crp->crp_etype == EAGAIN)
1061 return crypto_dispatch(crp);
1062
1063 bio = (struct bio *)crp->crp_opaque;
1064 KKASSERT(bio != NULL);
1065
f6221ad1
MD
1066 /*
1067 * Cumulative error
1068 */
1069 if (crp->crp_etype) {
1070 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
1071 "crp_etype = %d\n",
1072 crp->crp_etype);
1073 bio->bio_buf->b_error = crp->crp_etype;
1074 }
dd9da8fe 1075
f6221ad1
MD
1076 /*
1077 * On the last chunk of the decryption we do any required copybacks
1078 * and complete the I/O.
1079 */
c98abe57
MD
1080 n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1081#if 0
1082 kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
1083#endif
1084
d536b408
AH
1085 KTR_LOG(dmcrypt_crypto_cb_read_done, crp, bio->bio_buf, n);
1086
f6221ad1
MD
1087 if (n == 1) {
1088 /*
1089 * For the B_HASBOGUS case we didn't decrypt in place,
1090 * so we need to copy stuff back into the buf.
c98abe57
MD
1091 *
1092 * (disabled for now).
f6221ad1
MD
1093 */
1094 dmtc = bio->bio_caller_info2.ptr;
1095 if (bio->bio_buf->b_error) {
1096 bio->bio_buf->b_flags |= B_ERROR;
c98abe57
MD
1097 }
1098#if 0
1099 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
f6221ad1
MD
1100 memcpy(bio->bio_buf->b_data, dmtc->data_buf,
1101 bio->bio_buf->b_bcount);
1102 }
c98abe57 1103#endif
d536b408 1104 mpipe_free(&dmtc->priv->read_mpipe, dmtc->free_addr);
f6221ad1
MD
1105 obio = pop_bio(bio);
1106 biodone(obio);
1107 }
dd9da8fe
AH
1108 return 0;
1109}
f6221ad1 1110/* END OF STRATEGY READ SECTION */
dd9da8fe 1111
f6221ad1
MD
1112/*
1113 * STRATEGY WRITE PATH PART 1/3
1114 */
c9f7645b
MD
1115
1116static void
1117dmtc_crypto_write_retry(void *arg1, void *arg2)
1118{
1119 dm_target_crypt_config_t *priv = arg1;
1120 struct bio *bio = arg2;
1121
d536b408
AH
1122 KTR_LOG(dmcrypt_crypto_write_retry, bio->bio_buf);
1123
c9f7645b
MD
1124 dmtc_crypto_write_start(priv, bio);
1125}
1126
f6221ad1
MD
1127static void
1128dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1129{
1130 struct dmtc_helper *dmtc;
1131 struct cryptodesc *crd;
1132 struct cryptop *crp;
1133 struct cryptoini *cri;
ff4a7c8b
MD
1134 int i, bytes, sectors, sz;
1135 off_t isector;
f6221ad1
MD
1136 u_char *ptr, *space;
1137
1138 cri = &priv->crypto_session;
1139
1140 /*
1141 * Use b_bcount for consistency
1142 */
1143 bytes = bio->bio_buf->b_bcount;
1144
ff4a7c8b 1145 isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
f6221ad1
MD
1146 sectors = bytes / DEV_BSIZE; /* Number of sectors */
1147 sz = sectors * (sizeof(*crp) + sizeof(*crd));
1148
1149 /*
1150 * For writes and reads with bogus page don't decrypt in place.
1151 */
d536b408 1152 space = mpipe_alloc_callback(&priv->write_mpipe,
c9f7645b
MD
1153 dmtc_crypto_write_retry, priv, bio);
1154 if (space == NULL)
1155 return;
1156
f6221ad1
MD
1157 dmtc = (struct dmtc_helper *)space;
1158 dmtc->free_addr = space;
1159 space += sizeof(struct dmtc_helper);
1160 memcpy(space + sz, bio->bio_buf->b_data, bytes);
1161
1162 bio->bio_caller_info2.ptr = dmtc;
1163 bio->bio_buf->b_error = 0;
1164
1165 dmtc->orig_buf = bio->bio_buf->b_data;
1166 dmtc->data_buf = space + sz;
d536b408 1167 dmtc->priv = priv;
f6221ad1
MD
1168
1169 /*
1170 * Load crypto descriptors (crp/crd loop)
1171 */
c98abe57 1172 bzero(space, sz);
f6221ad1
MD
1173 ptr = space;
1174 bio->bio_caller_info3.value = sectors;
c98abe57 1175 cpu_sfence();
f6221ad1
MD
1176#if 0
1177 kprintf("Write, bytes = %d (b_bcount), "
1178 "sectors = %d (bio = %p, b_cmd = %d)\n",
1179 bytes, sectors, bio, bio->bio_buf->b_cmd);
1180#endif
1181 for (i = 0; i < sectors; i++) {
1182 crp = (struct cryptop *)ptr;
1183 ptr += sizeof(*crp);
1184 crd = (struct cryptodesc *)ptr;
1185 ptr += sizeof (*crd);
1186
1187 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1188
1189 crp->crp_sid = priv->crypto_sid;
1190 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1191
1192 crp->crp_opaque = (void *)bio;
1193
1194 crp->crp_callback = dmtc_crypto_cb_write_done;
1195 crp->crp_desc = crd;
1196 crp->crp_etype = 0;
57ffb118
MD
1197 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1198 CRYPTO_F_BATCH;
f6221ad1
MD
1199
1200 crd->crd_alg = priv->crypto_alg;
7cfe2226 1201#if 0
f6221ad1
MD
1202 crd->crd_key = (caddr_t)priv->crypto_key;
1203 crd->crd_klen = priv->crypto_klen;
7cfe2226
AH
1204#endif
1205
1206 crd->crd_skip = 0;
1207 crd->crd_len = DEV_BSIZE /* XXX */;
1208 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1209 crd->crd_next = NULL;
1210
1211 crd->crd_flags |= CRD_F_ENCRYPT;
f6221ad1 1212
c98abe57
MD
1213 /*
1214 * Note: last argument is used to generate salt(?) and is
1215 * a 64 bit value, but the original code passed an
1216 * int. Changing it now will break pre-existing
1217 * crypt volumes.
1218 */
d536b408
AH
1219
1220 KTR_LOG(dmcrypt_crypto_write_start, crp, bio->bio_buf,
1221 i, sectors);
1222
7cfe2226 1223 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
ff4a7c8b 1224 isector + i, crp);
f6221ad1
MD
1225 }
1226}
1227
1228/*
1229 * STRATEGY WRITE PATH PART 2/3
1230 */
1231static int
1232dmtc_crypto_cb_write_done(struct cryptop *crp)
dd9da8fe 1233{
f6221ad1 1234 struct dmtc_helper *dmtc;
dd9da8fe 1235 dm_target_crypt_config_t *priv;
f6221ad1
MD
1236 struct bio *bio, *obio;
1237 int n;
dd9da8fe 1238
f6221ad1
MD
1239 if (crp->crp_etype == EAGAIN)
1240 return crypto_dispatch(crp);
dd9da8fe 1241
f6221ad1
MD
1242 bio = (struct bio *)crp->crp_opaque;
1243 KKASSERT(bio != NULL);
dd9da8fe 1244
f6221ad1
MD
1245 /*
1246 * Cumulative error
1247 */
1248 if (crp->crp_etype != 0) {
1249 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1250 "crp_etype = %d\n",
1251 crp->crp_etype);
1252 bio->bio_buf->b_error = crp->crp_etype;
1253 }
dd9da8fe 1254
f6221ad1
MD
1255 /*
1256 * On the last chunk of the encryption we issue the write
1257 */
c98abe57
MD
1258 n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1259#if 0
1260 kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1261#endif
1262
d536b408
AH
1263 KTR_LOG(dmcrypt_crypto_cb_write_done, crp, bio->bio_buf, n);
1264
f6221ad1
MD
1265 if (n == 1) {
1266 dmtc = bio->bio_caller_info2.ptr;
1267 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
dd9da8fe 1268
f6221ad1
MD
1269 if (bio->bio_buf->b_error) {
1270 bio->bio_buf->b_flags |= B_ERROR;
d536b408 1271 mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
f6221ad1
MD
1272 obio = pop_bio(bio);
1273 biodone(obio);
1274 } else {
1275 dmtc->orig_buf = bio->bio_buf->b_data;
1276 bio->bio_buf->b_data = dmtc->data_buf;
1277 bio->bio_done = dmtc_bio_write_done;
1278 vn_strategy(priv->pdev->pdev_vnode, bio);
1279 }
1280 }
dd9da8fe
AH
1281 return 0;
1282}
1283
f6221ad1
MD
1284/*
1285 * STRATEGY WRITE PATH PART 3/3
1286 */
1287static void
1288dmtc_bio_write_done(struct bio *bio)
dd9da8fe 1289{
f6221ad1
MD
1290 struct dmtc_helper *dmtc;
1291 struct bio *obio;
1292
1293 dmtc = bio->bio_caller_info2.ptr;
1294 bio->bio_buf->b_data = dmtc->orig_buf;
d536b408
AH
1295 mpipe_free(&dmtc->priv->write_mpipe, dmtc->free_addr);
1296
1297 KTR_LOG(dmcrypt_bio_write_done, bio->bio_buf);
1298
f6221ad1
MD
1299 obio = pop_bio(bio);
1300 biodone(obio);
dd9da8fe 1301}
f6221ad1 1302/* END OF STRATEGY WRITE SECTION */
79b7159f
AH
1303
1304
1305
1306/* DUMPING MAGIC */
1307
1308extern int tsleep_crypto_dump;
1309
ba65ac12 1310static int
79b7159f
AH
1311dm_target_crypt_dump(dm_table_entry_t *table_en, void *data, size_t length, off_t offset)
1312{
1313 static struct dmtc_dump_helper dump_helper;
1314 dm_target_crypt_config_t *priv;
1315 int id;
1316 static int first_call = 1;
1317
1318 priv = table_en->target_config;
1319
1320 if (first_call) {
1321 first_call = 0;
1322 dump_reactivate_cpus();
1323 }
1324
1325 /* Magically enable tsleep */
1326 tsleep_crypto_dump = 1;
1327 id = 0;
1328
1329 /*
1330 * 0 length means flush buffers and return
1331 */
1332 if (length == 0) {
1333 if (priv->pdev->pdev_vnode->v_rdev == NULL) {
1334 tsleep_crypto_dump = 0;
1335 return ENXIO;
1336 }
1337 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1338 data, 0, offset, 0);
1339 tsleep_crypto_dump = 0;
1340 return 0;
1341 }
1342
1343 bzero(&dump_helper, sizeof(dump_helper));
1344 dump_helper.priv = priv;
1345 dump_helper.data = data;
1346 dump_helper.length = length;
1347 dump_helper.offset = offset +
1348 priv->block_offset * DEV_BSIZE;
1349 dump_helper.ident = &id;
1350 dmtc_crypto_dump_start(priv, &dump_helper);
1351
1352 /*
1353 * Hackery to make stuff appear synchronous. The crypto callback will
1354 * set id to 1 and call wakeup on it. If the request completed
1355 * synchronously, id will be 1 and we won't bother to sleep. If not,
1356 * the crypto request will complete asynchronously and we sleep until
1357 * it's done.
1358 */
1359 if (id == 0)
1360 tsleep(&dump_helper, 0, "cryptdump", 0);
1361
1362 dump_helper.offset = dm_pdev_correct_dump_offset(priv->pdev,
1363 dump_helper.offset);
1364
1365 dev_ddump(priv->pdev->pdev_vnode->v_rdev,
1366 dump_helper.space, 0, dump_helper.offset,
1367 dump_helper.length);
1368
1369 tsleep_crypto_dump = 0;
1370 return 0;
1371}
1372
1373static void
1374dmtc_crypto_dump_start(dm_target_crypt_config_t *priv, struct dmtc_dump_helper *dump_helper)
1375{
1376 struct cryptodesc *crd;
1377 struct cryptop *crp;
1378 struct cryptoini *cri;
1379 int i, bytes, sectors;
1380 off_t isector;
1381
1382 cri = &priv->crypto_session;
1383
1384 bytes = dump_helper->length;
1385
1386 isector = dump_helper->offset / DEV_BSIZE; /* ivgen salt base? */
1387 sectors = bytes / DEV_BSIZE; /* Number of sectors */
1388 dump_helper->sectors = sectors;
1389#if 0
1390 kprintf("Dump, bytes = %d, "
1391 "sectors = %d, LENGTH=%zu\n", bytes, sectors, dump_helper->length);
1392#endif
1393 KKASSERT(dump_helper->length <= 65536);
1394
1395 memcpy(dump_helper->space, dump_helper->data, bytes);
1396
1397 cpu_sfence();
1398
1399 for (i = 0; i < sectors; i++) {
1400 crp = &dump_helper->crp[i];
1401 crd = &dump_helper->crd[i];
1402
1403 crp->crp_buf = dump_helper->space + i * DEV_BSIZE;
1404
1405 crp->crp_sid = priv->crypto_sid;
1406 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1407
1408 crp->crp_opaque = (void *)dump_helper;
1409
1410 crp->crp_callback = dmtc_crypto_cb_dump_done;
1411 crp->crp_desc = crd;
1412 crp->crp_etype = 0;
1413 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1414 CRYPTO_F_BATCH;
1415
1416 crd->crd_alg = priv->crypto_alg;
1417
1418 crd->crd_skip = 0;
1419 crd->crd_len = DEV_BSIZE /* XXX */;
1420 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1421 crd->crd_next = NULL;
1422
1423 crd->crd_flags |= CRD_F_ENCRYPT;
1424
1425 /*
1426 * Note: last argument is used to generate salt(?) and is
1427 * a 64 bit value, but the original code passed an
1428 * int. Changing it now will break pre-existing
1429 * crypt volumes.
1430 */
1431 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1432 isector + i, crp);
1433 }
1434}
1435
1436static int
1437dmtc_crypto_cb_dump_done(struct cryptop *crp)
1438{
1439 struct dmtc_dump_helper *dump_helper;
1440 dm_target_crypt_config_t *priv;
1441 int n;
1442
1443 if (crp->crp_etype == EAGAIN)
1444 return crypto_dispatch(crp);
1445
1446 dump_helper = (struct dmtc_dump_helper *)crp->crp_opaque;
1447 KKASSERT(dump_helper != NULL);
1448
1449 if (crp->crp_etype != 0) {
1450 kprintf("dm_target_crypt: dmtc_crypto_cb_dump_done "
1451 "crp_etype = %d\n",
1452 crp->crp_etype);
1453 return crp->crp_etype;
1454 }
1455
1456 /*
1457 * On the last chunk of the encryption we return control
1458 */
1459 n = atomic_fetchadd_int(&dump_helper->sectors, -1);
1460
1461 if (n == 1) {
1462 priv = (dm_target_crypt_config_t *)dump_helper->priv;
1463 atomic_add_int(dump_helper->ident, 1);
1464 wakeup(dump_helper);
1465 }
1466
1467 return 0;
1468}
ba65ac12
AH
1469
1470static int
1471dmtc_mod_handler(module_t mod, int type, void *unused)
1472{
1473 dm_target_t *dmt = NULL;
1474 int err = 0;
1475
1476 switch (type) {
1477 case MOD_LOAD:
1478 if ((dmt = dm_target_lookup("crypt")) != NULL) {
1479 dm_target_unbusy(dmt);
1480 return EEXIST;
1481 }
1482 dmt = dm_target_alloc("crypt");
1483 dmt->version[0] = 1;
1484 dmt->version[1] = 6;
1485 dmt->version[2] = 0;
1486 strlcpy(dmt->name, "crypt", DM_MAX_TYPE_NAME);
1487 dmt->init = &dm_target_crypt_init;
1488 dmt->status = &dm_target_crypt_status;
1489 dmt->strategy = &dm_target_crypt_strategy;
1490 dmt->deps = &dm_target_crypt_deps;
1491 dmt->destroy = &dm_target_crypt_destroy;
1492 dmt->upcall = &dm_target_crypt_upcall;
1493 dmt->dump = &dm_target_crypt_dump;
1494
ba65ac12 1495 err = dm_target_insert(dmt);
d536b408 1496 if (!err)
5c411e8e 1497 kprintf("dm_target_crypt: Successfully initialized\n");
ba65ac12
AH
1498 break;
1499
1500 case MOD_UNLOAD:
1501 err = dm_target_rem("crypt");
7115a22b 1502 if (err == 0) {
7115a22b
AH
1503 kprintf("dm_target_crypt: unloaded\n");
1504 }
ba65ac12
AH
1505 break;
1506
1507 default:
1508 break;
1509 }
1510
1511 return err;
1512}
1513
1514DM_TARGET_MODULE(dm_target_crypt, dmtc_mod_handler);