2 * Copyright (c) 2010 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Alex Hornung <ahornung@gmail.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * This file implements initial version of device-mapper crypt target.
38 #include <sys/types.h>
39 #include <sys/param.h>
40 #include <sys/endian.h>
44 #include <sys/malloc.h>
46 #include <sys/vnode.h>
47 #include <crypto/sha1.h>
48 #include <crypto/sha2/sha2.h>
49 #include <opencrypto/cryptodev.h>
50 #include <opencrypto/rmd160.h>
51 #include <machine/cpufunc.h>
54 MALLOC_DEFINE(M_DMCRYPT, "dm_crypt", "Device Mapper Target Crypt");
56 struct target_crypt_config;
58 typedef void dispatch_t(void *);
59 typedef void ivgen_t(struct target_crypt_config *, u_int8_t *, size_t, off_t,
62 typedef int ivgen_ctor_t(struct target_crypt_config *, char *, void **);
63 typedef int ivgen_dtor_t(struct target_crypt_config *, void *);
72 struct essiv_ivgen_priv {
73 struct cryptoini crypto_session;
74 struct objcache *crp_crd_cache;
77 u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
80 typedef struct target_crypt_config {
86 u_int8_t crypto_key[512>>3];
89 u_int64_t block_offset;
91 SHA512_CTX essivsha512_ctx;
93 struct cryptoini crypto_session;
95 struct iv_generator *ivgen;
97 } dm_target_crypt_config_t;
105 static void dmtc_crypto_dispatch(void *arg);
106 static void dmtc_crypto_read_start(dm_target_crypt_config_t *priv,
108 static void dmtc_crypto_write_start(dm_target_crypt_config_t *priv,
110 static void dmtc_bio_read_done(struct bio *bio);
111 static void dmtc_bio_write_done(struct bio *bio);
112 static int dmtc_crypto_cb_read_done(struct cryptop *crp);
113 static int dmtc_crypto_cb_write_done(struct cryptop *crp);
115 static ivgen_ctor_t essiv_ivgen_ctor;
116 static ivgen_dtor_t essiv_ivgen_dtor;
117 static ivgen_t essiv_ivgen;
118 static ivgen_t plain_ivgen;
120 static struct iv_generator ivgens[] = {
121 { .name = "essiv", .ctor = essiv_ivgen_ctor, .dtor = essiv_ivgen_dtor,
122 .gen_iv = essiv_ivgen },
123 { .name = "plain", .ctor = NULL, .dtor = NULL, .gen_iv = plain_ivgen },
124 { NULL, NULL, NULL, NULL }
127 struct objcache_malloc_args essiv_ivgen_malloc_args = {
128 2*sizeof(void *) + (sizeof(struct cryptodesc) +
129 sizeof(struct cryptop)), M_DMCRYPT };
132 * Overwrite private information (in buf) to avoid leaking it
135 dmtc_crypto_clear(void *buf, size_t len)
137 memset(buf, 0xFF, len);
142 * ESSIV IV Generator Routines
145 essiv_ivgen_ctor(struct target_crypt_config *priv, char *iv_hash, void **p_ivpriv)
147 struct essiv_ivgen_priv *ivpriv;
148 u_int8_t crypto_keyhash[SHA512_DIGEST_LENGTH];
152 klen = (priv->crypto_klen >> 3);
157 if (!strcmp(iv_hash, "sha1")) {
160 if (klen != SHA1_RESULTLEN)
164 SHA1Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
165 SHA1Final(crypto_keyhash, &ctx);
166 } else if (!strcmp(iv_hash, "sha256")) {
169 if (klen != SHA256_DIGEST_LENGTH)
173 SHA256_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
174 SHA256_Final(crypto_keyhash, &ctx);
175 } else if (!strcmp(iv_hash, "sha384")) {
178 if (klen != SHA384_DIGEST_LENGTH)
182 SHA384_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
183 SHA384_Final(crypto_keyhash, &ctx);
184 } else if (!strcmp(iv_hash, "sha512")) {
187 if (klen != SHA512_DIGEST_LENGTH)
191 SHA512_Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
192 SHA512_Final(crypto_keyhash, &ctx);
193 } else if (!strcmp(iv_hash, "md5")) {
196 if (klen != MD5_DIGEST_LENGTH)
200 MD5Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
201 MD5Final(crypto_keyhash, &ctx);
202 } else if (!strcmp(iv_hash, "rmd160") ||
203 !strcmp(iv_hash, "ripemd160")) {
210 RMD160Update(&ctx, priv->crypto_key, priv->crypto_klen>>3);
211 RMD160Final(crypto_keyhash, &ctx);
216 ivpriv = kmalloc(sizeof(struct essiv_ivgen_priv), M_DMCRYPT,
218 memcpy(ivpriv->crypto_keyhash, crypto_keyhash, sizeof(crypto_keyhash));
219 ivpriv->keyhash_len = sizeof(crypto_keyhash);
220 dmtc_crypto_clear(crypto_keyhash, sizeof(crypto_keyhash));
222 ivpriv->crypto_session.cri_alg = priv->crypto_alg;
223 ivpriv->crypto_session.cri_key = (u_int8_t *)ivpriv->crypto_keyhash;
224 ivpriv->crypto_session.cri_klen = priv->crypto_klen;
225 ivpriv->crypto_session.cri_mlen = 0;
226 ivpriv->crypto_session.cri_next = NULL;
228 error = crypto_newsession(&ivpriv->crypto_sid,
229 &ivpriv->crypto_session,
230 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
232 kprintf("dm_target_crypt: Error during crypto_newsession "
233 "for essiv_ivgen, error = %d\n",
235 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
236 kfree(ivpriv, M_DMCRYPT);
240 ivpriv->crp_crd_cache = objcache_create(
241 "dmcrypt-essiv-cache", 0, 0,
243 objcache_malloc_alloc,
244 objcache_malloc_free,
245 &essiv_ivgen_malloc_args );
252 essiv_ivgen_dtor(struct target_crypt_config *priv, void *arg)
254 struct essiv_ivgen_priv *ivpriv;
256 ivpriv = (struct essiv_ivgen_priv *)arg;
257 KKASSERT(ivpriv != NULL);
259 crypto_freesession(ivpriv->crypto_sid);
261 objcache_destroy(ivpriv->crp_crd_cache);
263 dmtc_crypto_clear(ivpriv->crypto_keyhash, ivpriv->keyhash_len);
264 kfree(ivpriv, M_DMCRYPT);
270 essiv_ivgen_done(struct cryptop *crp)
272 struct essiv_ivgen_priv *ivpriv;
276 if (crp->crp_etype == EAGAIN)
277 return crypto_dispatch(crp);
279 if (crp->crp_etype != 0) {
280 kprintf("dm_target_crypt: essiv_ivgen_done, "
281 "crp->crp_etype = %d\n", crp->crp_etype);
284 free_addr = crp->crp_opaque;
286 * In-memory structure is:
287 * | ivpriv | opaque | crp | crd |
288 * | (void *) | (void *) | (cryptop) | (cryptodesc) |
290 ivpriv = *((struct essiv_ivgen_priv **)crp->crp_opaque);
291 crp->crp_opaque += sizeof(void *);
292 opaque = *((void **)crp->crp_opaque);
294 objcache_put(ivpriv->crp_crd_cache, free_addr);
295 dmtc_crypto_dispatch(opaque);
300 essiv_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
301 size_t iv_len, off_t sector, void *opaque)
303 struct essiv_ivgen_priv *ivpriv;
304 struct cryptodesc *crd;
306 caddr_t space, alloc_addr;
309 ivpriv = priv->ivgen_priv;
310 KKASSERT(ivpriv != NULL);
313 * In-memory structure is:
314 * | ivpriv | opaque | crp | crd |
315 * | (void *) | (void *) | (cryptop) | (cryptodesc) |
317 alloc_addr = space = objcache_get(ivpriv->crp_crd_cache, M_WAITOK);
318 *((struct essiv_ivgen_priv **)space) = ivpriv;
319 space += sizeof(void *);
320 *((void **)space) = opaque;
321 space += sizeof(void *);
322 crp = (struct cryptop *)space;
323 space += sizeof(struct cryptop);
324 crd = (struct cryptodesc *)space;
327 bzero(crd, sizeof(struct cryptodesc));
328 bzero(crp, sizeof(struct cryptop));
329 *((off_t *)iv) = htole64(sector + priv->iv_offset);
330 crp->crp_buf = (caddr_t)iv;
332 crp->crp_sid = ivpriv->crypto_sid;
333 crp->crp_ilen = crp->crp_olen = iv_len;
335 crp->crp_opaque = alloc_addr;
337 crp->crp_callback = essiv_ivgen_done;
341 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL | CRYPTO_F_BATCH;
343 crd->crd_alg = priv->crypto_alg;
345 crd->crd_key = (caddr_t)priv->crypto_keyhash;
346 crd->crd_klen = priv->crypto_klen;
349 bzero(crd->crd_iv, sizeof(crd->crd_iv));
352 crd->crd_len = iv_len;
353 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
354 crd->crd_flags |= CRD_F_ENCRYPT;
355 crd->crd_next = NULL;
357 error = crypto_dispatch(crp);
359 kprintf("dm_target_crypt: essiv_ivgen, error = %d\n", error);
364 plain_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
365 size_t iv_len, off_t sector, void *opaque)
368 *((uint32_t *)iv) = htole32((uint32_t)(sector + priv->iv_offset));
369 dmtc_crypto_dispatch(opaque);
375 geli_ivgen(dm_target_crypt_config_t *priv, u_int8_t *iv,
376 size_t iv_len, off_t sector, void *opaque)
380 u_int8_t md[SHA512_DIGEST_LENGTH]; /* Max. Digest Size */
382 memcpy(&ctx512, &priv->essivsha512_ctx, sizeof(SHA512_CTX));
383 SHA512_Update(&ctx512, (u_int8_t*)§or, sizeof(off_t));
384 SHA512_Final(md, &ctx512);
386 memcpy(iv, md, iv_len);
387 dmtc_crypto_dispatch(opaque);
391 #ifdef DM_TARGET_MODULE
393 * Every target can be compiled directly to dm driver or as a
394 * separate module this part of target is used for loading targets
396 * Target can be unloaded from kernel only if there are no users of
397 * it e.g. there are no devices which uses that target.
399 #include <sys/kernel.h>
400 #include <sys/module.h>
403 dm_target_crypt_modcmd(modcmd_t cmd, void *arg)
410 case MODULE_CMD_INIT:
411 if ((dmt = dm_target_lookup("crypt")) != NULL) {
412 dm_target_unbusy(dmt);
415 dmt = dm_target_alloc("crypt");
420 strlcpy(dmt->name, "crypt", DM_MAX_TYPE_NAME);
421 dmt->init = &dm_target_crypt_init;
422 dmt->status = &dm_target_crypt_status;
423 dmt->strategy = &dm_target_crypt_strategy;
424 dmt->deps = &dm_target_crypt_deps;
425 dmt->destroy = &dm_target_crypt_destroy;
426 dmt->upcall = &dm_target_crypt_upcall;
428 r = dm_target_insert(dmt);
432 case MODULE_CMD_FINI:
433 r = dm_target_rem("crypt");
436 case MODULE_CMD_STAT:
449 * Init function called from dm_table_load_ioctl.
450 * cryptsetup actually passes us this:
451 * aes-cbc-essiv:sha256 7997f8af... 0 /dev/ad0s0a 8
454 hex2key(char *hex, size_t key_len, u_int8_t *key)
460 for (key_idx = 0; key_idx < key_len; ++key_idx) {
463 key[key_idx] = (u_int8_t)strtoul(hex_buf, NULL, 16);
472 dm_target_crypt_init(dm_dev_t * dmv, void **target_config, char *params)
474 dm_target_crypt_config_t *priv;
477 char *crypto_alg, *crypto_mode, *iv_mode, *iv_opt, *key, *dev;
479 int i, argc, klen, error;
480 uint64_t iv_offset, block_offset;
485 len = strlen(params) + 1;
488 status_str = kstrdup(params, M_DMCRYPT);
490 * Parse a string, containing tokens delimited by white space,
491 * into an argument vector
493 for (ap = args; ap < &args[5] &&
494 (*ap = strsep(¶ms, " \t")) != NULL;) {
502 kprintf("dm_target_crypt: not enough arguments, "
504 kfree(status_str, M_DMCRYPT);
505 return ENOMEM; /* XXX */
508 crypto_alg = strsep(&args[0], "-");
509 crypto_mode = strsep(&args[0], "-");
510 iv_opt = strsep(&args[0], "-");
511 iv_mode = strsep(&iv_opt, ":");
513 iv_offset = strtouq(args[2], NULL, 0);
515 block_offset = strtouq(args[4], NULL, 0);
516 /* bits / 8 = bytes, 1 byte = 2 hexa chars, so << 2 */
517 klen = strlen(key) << 2;
519 kprintf("dm_target_crypt - new: dev=%s, crypto_alg=%s, crypto_mode=%s, "
520 "iv_mode=%s, iv_opt=%s, key=%s, iv_offset=%ju, "
521 "block_offset=%ju\n",
522 dev, crypto_alg, crypto_mode, iv_mode, iv_opt, key, iv_offset,
525 priv = kmalloc(sizeof(dm_target_crypt_config_t), M_DMCRYPT, M_WAITOK);
527 kprintf("dm_target_crypt: could not allocate memory\n");
528 kfree(status_str, M_DMCRYPT);
532 /* Insert dmp to global pdev list */
533 if ((priv->pdev = dm_pdev_insert(dev)) == NULL) {
534 kprintf("dm_target_crypt: dm_pdev_insert failed\n");
535 kfree(status_str, M_DMCRYPT);
539 if (strcmp(crypto_mode, "cbc") != 0) {
540 kprintf("dm_target_crypt: only support 'cbc' chaining mode, "
541 "invalid mode '%s'\n", crypto_mode);
545 if (!strcmp(crypto_alg, "aes")) {
546 priv->crypto_alg = CRYPTO_AES_CBC;
547 if (klen != 128 && klen != 192 && klen != 256)
549 priv->crypto_klen = klen;
550 } else if (!strcmp(crypto_alg, "blowfish")) {
551 priv->crypto_alg = CRYPTO_BLF_CBC;
552 if (klen < 128 || klen > 448 || (klen % 8) != 0)
554 priv->crypto_klen = klen;
556 } else if (!strcmp(crypto_alg, "3des") ||
557 !strncmp(crypto_alg, "des3", 4)) {
558 priv->crypto_alg = CRYPTO_3DES_CBC;
561 priv->crypto_klen = 168;
562 } else if (!strcmp(crypto_alg, "camellia")) {
563 priv->crypto_alg = CRYPTO_CAMELLIA_CBC;
564 if (klen != 128 && klen != 192 && klen != 256)
566 priv->crypto_klen = klen;
567 } else if (!strcmp(crypto_alg, "skipjack")) {
568 priv->crypto_alg = CRYPTO_SKIPJACK_CBC;
571 priv->crypto_klen = 80;
572 } else if (!strcmp(crypto_alg, "cast5")) {
573 priv->crypto_alg = CRYPTO_CAST_CBC;
576 priv->crypto_klen = 128;
577 } else if (!strcmp(crypto_alg, "null")) {
578 priv->crypto_alg = CRYPTO_NULL_CBC;
581 priv->crypto_klen = 128;
583 kprintf("dm_target_crypt: Unsupported crypto algorithm: %s\n",
588 /* Save length of param string */
589 priv->params_len = len;
590 priv->block_offset = block_offset;
591 priv->iv_offset = iv_offset - block_offset;
593 *target_config = priv;
595 dmv->dev_type = DM_CRYPTO_DEV;
597 error = hex2key(key, priv->crypto_klen >> 3,
598 (u_int8_t *)priv->crypto_key);
601 kprintf("dm_target_crypt: hex2key failed, "
602 "invalid key format\n");
607 for(i = 0; ivgens[i].name != NULL; i++) {
608 if (!strcmp(iv_mode, ivgens[i].name))
612 if (ivgens[i].name == NULL) {
613 kprintf("dm_target_crypt: iv_mode='%s' unsupported\n",
618 /* Call our ivgen constructor */
619 if (ivgens[i].ctor != NULL) {
620 error = ivgens[i].ctor(priv, iv_opt,
623 kprintf("dm_target_crypt: ctor for '%s' failed\n",
629 priv->ivgen = &ivgens[i];
631 priv->crypto_session.cri_alg = priv->crypto_alg;
632 priv->crypto_session.cri_key = (u_int8_t *)priv->crypto_key;
633 priv->crypto_session.cri_klen = priv->crypto_klen;
634 priv->crypto_session.cri_mlen = 0;
635 priv->crypto_session.cri_next = NULL;
637 error = crypto_newsession(&priv->crypto_sid,
638 &priv->crypto_session,
639 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE);
641 kprintf("dm_target_crypt: Error during crypto_newsession, "
647 priv->status_str = status_str;
651 kprintf("dm_target_crypt: ENOTSUP\n");
652 kfree(status_str, M_DMCRYPT);
656 /* Status routine called to get params string. */
658 dm_target_crypt_status(void *target_config)
660 dm_target_crypt_config_t *priv;
663 priv = target_config;
665 /* caller expects use of M_DM */
666 params = kmalloc(DM_MAX_PARAMS_SIZE, M_DM, M_WAITOK);
668 ksnprintf(params, DM_MAX_PARAMS_SIZE, "%s",
675 dm_target_crypt_destroy(dm_table_entry_t * table_en)
677 dm_target_crypt_config_t *priv;
680 * Disconnect the crypt config before unbusying the target.
682 priv = table_en->target_config;
685 table_en->target_config = NULL;
686 dm_pdev_decr(priv->pdev);
688 dm_target_unbusy(table_en->target);
691 * Clean up the crypt config
693 * Overwrite the private information before freeing memory to
696 dmtc_crypto_clear(priv->status_str, strlen(priv->status_str));
697 kfree(priv->status_str, M_DMCRYPT);
699 crypto_freesession(priv->crypto_sid);
701 if (priv->ivgen->dtor != NULL) {
702 priv->ivgen->dtor(priv, priv->ivgen_priv);
705 dmtc_crypto_clear(priv, sizeof(dm_target_crypt_config_t));
706 kfree(priv, M_DMCRYPT);
712 dm_target_crypt_deps(dm_table_entry_t * table_en, prop_array_t prop_array)
714 dm_target_crypt_config_t *priv;
719 if (table_en->target_config == NULL)
722 priv = table_en->target_config;
724 if ((error = VOP_GETATTR(priv->pdev->pdev_vnode, &va)) != 0)
727 prop_array_add_uint64(prop_array,
728 (uint64_t)makeudev(va.va_rmajor, va.va_rminor));
733 /* Unsupported for this target. */
735 dm_target_crypt_upcall(dm_table_entry_t * table_en, struct buf * bp)
740 /************************************************************************
741 * STRATEGY SUPPORT FUNCTIONS *
742 ************************************************************************
744 * READ PATH: doio -> bio_read_done -> crypto_work -> crypto_cb_read_done
745 * WRITE PATH: crypto_work -> crypto_cb_write_done -> doio -> bio_write_done
749 * Wrapper around crypto_dispatch() to match dispatch_t type
752 dmtc_crypto_dispatch(void *arg)
756 crp = (struct cryptop *)arg;
757 KKASSERT(crp != NULL);
758 crypto_dispatch(crp);
762 * Start IO operation, called from dmstrategy routine.
765 dm_target_crypt_strategy(dm_table_entry_t *table_en, struct buf *bp)
769 dm_target_crypt_config_t *priv;
770 priv = table_en->target_config;
772 /* Get rid of stuff we can't really handle */
773 if ((bp->b_cmd == BUF_CMD_READ) || (bp->b_cmd == BUF_CMD_WRITE)) {
774 if (((bp->b_bcount % DEV_BSIZE) != 0) || (bp->b_bcount == 0)) {
775 kprintf("dm_target_crypt_strategy: can't really "
776 "handle bp->b_bcount = %d\n",
778 bp->b_error = EINVAL;
779 bp->b_flags |= B_ERROR | B_INVAL;
780 biodone(&bp->b_bio1);
787 bio = push_bio(&bp->b_bio1);
788 bio->bio_offset = bp->b_bio1.bio_offset +
789 priv->block_offset * DEV_BSIZE;
790 bio->bio_caller_info1.ptr = priv;
791 bio->bio_done = dmtc_bio_read_done;
792 vn_strategy(priv->pdev->pdev_vnode, bio);
795 bio = push_bio(&bp->b_bio1);
796 bio->bio_offset = bp->b_bio1.bio_offset +
797 priv->block_offset * DEV_BSIZE;
798 bio->bio_caller_info1.ptr = priv;
799 dmtc_crypto_write_start(priv, bio);
802 vn_strategy(priv->pdev->pdev_vnode, &bp->b_bio1);
809 * STRATEGY READ PATH PART 1/3 (after read BIO completes)
812 dmtc_bio_read_done(struct bio *bio)
816 dm_target_crypt_config_t *priv;
819 * If a read error occurs we shortcut the operation, otherwise
822 if (bio->bio_buf->b_flags & B_ERROR) {
826 priv = bio->bio_caller_info1.ptr;
827 dmtc_crypto_read_start(priv, bio);
832 * STRATEGY READ PATH PART 2/3
835 dmtc_crypto_read_start(dm_target_crypt_config_t *priv, struct bio *bio)
837 struct dmtc_helper *dmtc;
838 struct cryptodesc *crd;
840 struct cryptoini *cri;
841 int i, bytes, sectors, sz;
845 cri = &priv->crypto_session;
848 * Note: b_resid no good after read I/O, it will be 0, use
851 bytes = bio->bio_buf->b_bcount;
852 isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
853 sectors = bytes / DEV_BSIZE; /* Number of sectors */
854 sz = sectors * (sizeof(*crp) + sizeof(*crd));
857 * For reads with bogus page we can't decrypt in place as stuff
858 * can get ripped out from under us.
860 * XXX actually it looks like we can, and in any case the initial
861 * read already completed and threw crypted data into the buffer
862 * cache buffer. Disable for now.
865 if (bio->bio_buf->b_flags & B_HASBOGUS) {
866 space = kmalloc(sizeof(struct dmtc_helper) + sz + bytes,
867 M_DMCRYPT, M_WAITOK);
868 dmtc = (struct dmtc_helper *)space;
869 dmtc->free_addr = space;
870 space += sizeof(struct dmtc_helper);
871 dmtc->orig_buf = NULL;
872 dmtc->data_buf = space + sz;
873 memcpy(dmtc->data_buf, bio->bio_buf->b_data, bytes);
877 space = kmalloc(sizeof(struct dmtc_helper) + sz,
878 M_DMCRYPT, M_WAITOK);
879 dmtc = (struct dmtc_helper *)space;
880 dmtc->free_addr = space;
881 space += sizeof(struct dmtc_helper);
882 dmtc->orig_buf = NULL;
883 dmtc->data_buf = bio->bio_buf->b_data;
885 bio->bio_caller_info2.ptr = dmtc;
886 bio->bio_buf->b_error = 0;
889 * Load crypto descriptors (crp/crd loop)
893 bio->bio_caller_info3.value = sectors;
896 kprintf("Read, bytes = %d (b_bcount), "
897 "sectors = %d (bio = %p, b_cmd = %d)\n",
898 bytes, sectors, bio, bio->bio_buf->b_cmd);
900 for (i = 0; i < sectors; i++) {
901 crp = (struct cryptop *)ptr;
903 crd = (struct cryptodesc *)ptr;
904 ptr += sizeof (*crd);
906 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
908 crp->crp_sid = priv->crypto_sid;
909 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
911 crp->crp_opaque = (void *)bio;
913 crp->crp_callback = dmtc_crypto_cb_read_done;
916 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
919 crd->crd_alg = priv->crypto_alg;
921 crd->crd_key = (caddr_t)priv->crypto_key;
922 crd->crd_klen = priv->crypto_klen;
926 crd->crd_len = DEV_BSIZE /* XXX */;
927 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
928 crd->crd_next = NULL;
930 crd->crd_flags &= ~CRD_F_ENCRYPT;
933 * Note: last argument is used to generate salt(?) and is
934 * a 64 bit value, but the original code passed an
935 * int. Changing it now will break pre-existing
938 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
944 * STRATEGY READ PATH PART 3/3
947 dmtc_crypto_cb_read_done(struct cryptop *crp)
949 struct dmtc_helper *dmtc;
950 struct bio *bio, *obio;
953 if (crp->crp_etype == EAGAIN)
954 return crypto_dispatch(crp);
956 bio = (struct bio *)crp->crp_opaque;
957 KKASSERT(bio != NULL);
962 if (crp->crp_etype) {
963 kprintf("dm_target_crypt: dmtc_crypto_cb_read_done "
966 bio->bio_buf->b_error = crp->crp_etype;
970 * On the last chunk of the decryption we do any required copybacks
971 * and complete the I/O.
973 n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
975 kprintf("dmtc_crypto_cb_read_done %p, n = %d\n", bio, n);
980 * For the B_HASBOGUS case we didn't decrypt in place,
981 * so we need to copy stuff back into the buf.
983 * (disabled for now).
985 dmtc = bio->bio_caller_info2.ptr;
986 if (bio->bio_buf->b_error) {
987 bio->bio_buf->b_flags |= B_ERROR;
990 else if (bio->bio_buf->b_flags & B_HASBOGUS) {
991 memcpy(bio->bio_buf->b_data, dmtc->data_buf,
992 bio->bio_buf->b_bcount);
995 kfree(dmtc->free_addr, M_DMCRYPT);
1001 /* END OF STRATEGY READ SECTION */
1004 * STRATEGY WRITE PATH PART 1/3
1007 dmtc_crypto_write_start(dm_target_crypt_config_t *priv, struct bio *bio)
1009 struct dmtc_helper *dmtc;
1010 struct cryptodesc *crd;
1011 struct cryptop *crp;
1012 struct cryptoini *cri;
1013 int i, bytes, sectors, sz;
1015 u_char *ptr, *space;
1017 cri = &priv->crypto_session;
1020 * Use b_bcount for consistency
1022 bytes = bio->bio_buf->b_bcount;
1024 isector = bio->bio_offset / DEV_BSIZE; /* ivgen salt base? */
1025 sectors = bytes / DEV_BSIZE; /* Number of sectors */
1026 sz = sectors * (sizeof(*crp) + sizeof(*crd));
1029 * For writes and reads with bogus page don't decrypt in place.
1031 space = kmalloc(sizeof(struct dmtc_helper) + sz + bytes,
1032 M_DMCRYPT, M_WAITOK);
1033 dmtc = (struct dmtc_helper *)space;
1034 dmtc->free_addr = space;
1035 space += sizeof(struct dmtc_helper);
1036 memcpy(space + sz, bio->bio_buf->b_data, bytes);
1038 bio->bio_caller_info2.ptr = dmtc;
1039 bio->bio_buf->b_error = 0;
1041 dmtc->orig_buf = bio->bio_buf->b_data;
1042 dmtc->data_buf = space + sz;
1045 * Load crypto descriptors (crp/crd loop)
1049 bio->bio_caller_info3.value = sectors;
1052 kprintf("Write, bytes = %d (b_bcount), "
1053 "sectors = %d (bio = %p, b_cmd = %d)\n",
1054 bytes, sectors, bio, bio->bio_buf->b_cmd);
1056 for (i = 0; i < sectors; i++) {
1057 crp = (struct cryptop *)ptr;
1058 ptr += sizeof(*crp);
1059 crd = (struct cryptodesc *)ptr;
1060 ptr += sizeof (*crd);
1062 crp->crp_buf = dmtc->data_buf + i * DEV_BSIZE;
1064 crp->crp_sid = priv->crypto_sid;
1065 crp->crp_ilen = crp->crp_olen = DEV_BSIZE;
1067 crp->crp_opaque = (void *)bio;
1069 crp->crp_callback = dmtc_crypto_cb_write_done;
1070 crp->crp_desc = crd;
1072 crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_REL |
1075 crd->crd_alg = priv->crypto_alg;
1077 crd->crd_key = (caddr_t)priv->crypto_key;
1078 crd->crd_klen = priv->crypto_klen;
1082 crd->crd_len = DEV_BSIZE /* XXX */;
1083 crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
1084 crd->crd_next = NULL;
1086 crd->crd_flags |= CRD_F_ENCRYPT;
1089 * Note: last argument is used to generate salt(?) and is
1090 * a 64 bit value, but the original code passed an
1091 * int. Changing it now will break pre-existing
1094 priv->ivgen->gen_iv(priv, crd->crd_iv, sizeof(crd->crd_iv),
1100 * STRATEGY WRITE PATH PART 2/3
1103 dmtc_crypto_cb_write_done(struct cryptop *crp)
1105 struct dmtc_helper *dmtc;
1106 dm_target_crypt_config_t *priv;
1107 struct bio *bio, *obio;
1110 if (crp->crp_etype == EAGAIN)
1111 return crypto_dispatch(crp);
1113 bio = (struct bio *)crp->crp_opaque;
1114 KKASSERT(bio != NULL);
1119 if (crp->crp_etype != 0) {
1120 kprintf("dm_target_crypt: dmtc_crypto_cb_write_done "
1123 bio->bio_buf->b_error = crp->crp_etype;
1127 * On the last chunk of the encryption we issue the write
1129 n = atomic_fetchadd_int(&bio->bio_caller_info3.value, -1);
1131 kprintf("dmtc_crypto_cb_write_done %p, n = %d\n", bio, n);
1135 dmtc = bio->bio_caller_info2.ptr;
1136 priv = (dm_target_crypt_config_t *)bio->bio_caller_info1.ptr;
1138 if (bio->bio_buf->b_error) {
1139 bio->bio_buf->b_flags |= B_ERROR;
1140 kfree(dmtc->free_addr, M_DMCRYPT);
1141 obio = pop_bio(bio);
1144 dmtc->orig_buf = bio->bio_buf->b_data;
1145 bio->bio_buf->b_data = dmtc->data_buf;
1146 bio->bio_done = dmtc_bio_write_done;
1147 vn_strategy(priv->pdev->pdev_vnode, bio);
1154 * STRATEGY WRITE PATH PART 3/3
1157 dmtc_bio_write_done(struct bio *bio)
1159 struct dmtc_helper *dmtc;
1162 dmtc = bio->bio_caller_info2.ptr;
1163 bio->bio_buf->b_data = dmtc->orig_buf;
1164 kfree(dmtc->free_addr, M_DMCRYPT);
1165 obio = pop_bio(bio);
1168 /* END OF STRATEGY WRITE SECTION */