[dragonfly.git] / sbin / hammer2 / crypto.c

/*
 * Copyright (c) 2011-2012 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@dragonflybsd.org>
 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
 * by Alex Hornung <alexh@dragonflybsd.org>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "hammer2.h"
#include <sys/endian.h>

/*
 * Setup crypto for pthreads
 */
static pthread_mutex_t *crypto_locks;
int crypto_count;

static
unsigned long
hammer2_crypto_id_callback(void)
{
	return ((unsigned long)(uintptr_t)pthread_self());
}

static
void
hammer2_crypto_locking_callback(int mode, int type,
				const char *file __unused, int line __unused)
{
	assert(type >= 0 && type < crypto_count);
	if (mode & CRYPTO_LOCK) {
		pthread_mutex_lock(&crypto_locks[type]);
	} else {
		pthread_mutex_unlock(&crypto_locks[type]);
	}
}

void
hammer2_crypto_setup(void)
{
	crypto_count = CRYPTO_num_locks();
	crypto_locks = calloc(crypto_count, sizeof(crypto_locks[0]));
	CRYPTO_set_id_callback(hammer2_crypto_id_callback);
	CRYPTO_set_locking_callback(hammer2_crypto_locking_callback);
}

static
int
_gcm_init(hammer2_ioq_t *ioq, char *key, int klen, char *iv_fixed, int ivlen,
	  int enc)
{
	int i, ok;

	if (klen < 32 /* 256 bits */ || ivlen < 4 /* 32 bits */) {
		if (DebugOpt)
			fprintf(stderr, "Not enough key or iv material\n");
		return -1;
	}

	printf("%s key: ", enc ? "Encryption" : "Decryption");
	for (i = 0; i < HAMMER2_AES_KEY_SIZE; ++i)
		printf("%02x", (unsigned char)key[i]);
	printf("\n");

	printf("%s iv:  ", enc ? "Encryption" : "Decryption");
	for (i = 0; i < HAMMER2_CRYPTO_IV_FIXED_SIZE; ++i)
		printf("%02x", (unsigned char)iv_fixed[i]);
	printf(" (fixed part only)\n");


	EVP_CIPHER_CTX_init(&ioq->ctx);

	if (enc)
		ok = EVP_EncryptInit_ex(&ioq->ctx, EVP_aes_256_gcm(), NULL,
					key, NULL);
	else
		ok = EVP_DecryptInit_ex(&ioq->ctx, EVP_aes_256_gcm(), NULL,
					key, NULL);
	if (!ok)
		goto fail;

	/*
	 * According to the original Galois/Counter Mode of Operation (GCM)
	 * proposal, only IVs that are exactly 96 bits get used without any
	 * further processing. Other IV sizes cause the GHASH() operation
	 * to be applied to the IV, which is more costly.
	 *
	 * The NIST SP 800-38D also recommends using a 96 bit IV for the same
	 * reasons. We actually follow the deterministic construction
	 * recommended in NIST SP 800-38D with a 64 bit invocation field as an
	 * integer counter and a random, session-specific fixed field.
	 *
	 * This means that we can essentially use the same session key and
	 * IV fixed field for up to 2^64 invocations of the authenticated
	 * encryption or decryption.
	 *
	 * With a chunk size of 64 bytes, this adds up to 1 zettabyte of
	 * traffic.
	 */
	ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_SET_IVLEN,
				 HAMMER2_CRYPTO_IV_SIZE, NULL);
	if (!ok)
		goto fail;

	memset(ioq->iv, 0, HAMMER2_CRYPTO_IV_SIZE);
	memcpy(ioq->iv, iv_fixed, HAMMER2_CRYPTO_IV_FIXED_SIZE);

	/*
	 * Strictly speaking, padding is irrelevant with a counter mode
	 * encryption.
	 *
	 * However, setting padding to 0, even if using a counter mode such
	 * as GCM, will cause an error in _finish if the pt/ct size is not
	 * a multiple of the cipher block size.
	 */
	EVP_CIPHER_CTX_set_padding(&ioq->ctx, 0);

	return 0;

fail:
	if (DebugOpt)
		fprintf(stderr, "Error during _gcm_init\n");
	return -1;
}

static
int
_gcm_iv_increment(char *iv)
{
	/*
	 * Deterministic construction according to NIST SP 800-38D, with
	 * 64 bit invocation field as integer counter.
	 *
	 * In other words, our 96 bit IV consists of a 32 bit fixed field
	 * unique to the session and a 64 bit integer counter.
	 */

	uint64_t *c = (uint64_t *)(&iv[HAMMER2_CRYPTO_IV_FIXED_SIZE]);

	/* Increment invocation field integer counter */
	/*
	 * XXX: this should ideally be an atomic update, but we don't have
	 * an atomic_fetchadd_64 for i386 yet
	 */
	*c = htobe64(be64toh(*c)+1);

	/*
	 * Detect wrap-around, which means it is time to renegotiate
	 * the session to get a new key and/or fixed field.
	 */
	return (c == 0) ? 0 : 1;
}

static
int
hammer2_crypto_encrypt_chunk(hammer2_ioq_t *ioq, char *ct, char *pt,
			     int in_size, int *out_size)
{
	int ok;
#ifdef CRYPTO_DEBUG
	int i;
#endif
	int u_len, f_len;

	*out_size = 0;

	/* Re-initialize with new IV (but without redoing the key schedule) */
	ok = EVP_EncryptInit_ex(&ioq->ctx, NULL, NULL, NULL, ioq->iv);
	if (!ok)
		goto fail;

	ok = EVP_EncryptUpdate(&ioq->ctx, ct, &u_len, pt, in_size);
	if (!ok)
		goto fail;

	ok = EVP_EncryptFinal(&ioq->ctx, ct + u_len, &f_len);
	if (!ok)
		goto fail;

	/* Retrieve auth tag */
	ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_GET_TAG,
				 HAMMER2_CRYPTO_TAG_SIZE,
				 ct + u_len + f_len);
	if (!ok)
		goto fail;

#ifdef CRYPTO_DEBUG
	printf("enc_chunk iv: ");
	for (i = 0; i < HAMMER2_CRYPTO_IV_SIZE; i++)
		printf("%02x", (unsigned char)ioq->iv[i]);
	printf("\n");

	printf("enc_chunk pt: ");
	for (i = 0; i < in_size; i++)
		printf("%02x", (unsigned char)pt[i]);
	printf("\n");

	printf("enc_chunk ct: ");
	for (i = 0; i < in_size; i++)
		printf("%02x", (unsigned char)ct[i]);
	printf("\n");

	printf("enc_chunk tag: ");
	for (i = 0; i < HAMMER2_CRYPTO_TAG_SIZE; i++)
		printf("%02x", (unsigned char)ct[i + u_len + f_len]);
	printf("\n");
#endif

	ok = _gcm_iv_increment(ioq->iv);
	if (!ok) {
		ioq->error = HAMMER2_IOQ_ERROR_IVWRAP;
		goto fail_out;
	}

	*out_size = u_len + f_len + HAMMER2_CRYPTO_TAG_SIZE;

	return 0;

fail:
	ioq->error = HAMMER2_IOQ_ERROR_ALGO;
fail_out:
	if (DebugOpt)
		fprintf(stderr, "error during encrypt_chunk\n");
	return -1;
}

static
int
hammer2_crypto_decrypt_chunk(hammer2_ioq_t *ioq, char *ct, char *pt,
			     int out_size, int *consume_size)
{
	int ok;
#ifdef CRYPTO_DEBUG
	int i;
#endif
	int u_len, f_len;

	*consume_size = 0;

	/* Re-initialize with new IV (but without redoing the key schedule) */
	ok = EVP_DecryptInit_ex(&ioq->ctx, NULL, NULL, NULL, ioq->iv);
	if (!ok) {
		ioq->error = HAMMER2_IOQ_ERROR_ALGO;
		goto fail_out;
	}

#ifdef CRYPTO_DEBUG
	printf("dec_chunk iv: ");
	for (i = 0; i < HAMMER2_CRYPTO_IV_SIZE; i++)
		printf("%02x", (unsigned char)ioq->iv[i]);
	printf("\n");

	printf("dec_chunk ct: ");
	for (i = 0; i < out_size; i++)
		printf("%02x", (unsigned char)ct[i]);
	printf("\n");

	printf("dec_chunk tag: ");
	for (i = 0; i < HAMMER2_CRYPTO_TAG_SIZE; i++)
		printf("%02x", (unsigned char)ct[out_size + i]);
	printf("\n");
#endif

	ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_SET_TAG,
				 HAMMER2_CRYPTO_TAG_SIZE,
				 ct + out_size);
	if (!ok) {
		ioq->error = HAMMER2_IOQ_ERROR_ALGO;
		goto fail_out;
	}

	ok = EVP_DecryptUpdate(&ioq->ctx, pt, &u_len, ct, out_size);
	if (!ok)
		goto fail;

	ok = EVP_DecryptFinal(&ioq->ctx, pt + u_len, &f_len);
	if (!ok)
		goto fail;

	ok = _gcm_iv_increment(ioq->iv);
	if (!ok) {
		ioq->error = HAMMER2_IOQ_ERROR_IVWRAP;
		goto fail_out;
	}

	*consume_size = u_len + f_len + HAMMER2_CRYPTO_TAG_SIZE;

#ifdef CRYPTO_DEBUG
	printf("dec_chunk pt: ");
	for (i = 0; i < out_size; i++)
		printf("%02x", (unsigned char)pt[i]);
	printf("\n");
#endif

	return 0;

fail:
	ioq->error = HAMMER2_IOQ_ERROR_MACFAIL;
fail_out:
	if (DebugOpt)
		fprintf(stderr, "error during decrypt_chunk (likely authentication error)\n");
	return -1;
}

/*
 * Synchronously negotiate crypto for a new session.  This must occur
 * within 10 seconds or the connection is error'd out.
 *
 * We work off the IP address and/or reverse DNS.  The IP address is
 * checked first, followed by the IP address at various levels of granularity,
 * followed by the full domain name and domain names at various levels of
 * granularity.
 *
 *	/etc/hammer2/remote/<name>.pub	- Contains a public key
 *	/etc/hammer2/remote/<name>.none	- Indicates no encryption (empty file)
 *					  (e.g. localhost.none).
 *
 * We first attempt to locate a public key file based on the peer address or
 * peer FQDN.
 *
 *	<name>.none	- No further negotiation is needed.  We simply return.
 *			  All communication proceeds without encryption.
 *			  No public key handshake occurs in this situation.
 *			  (both ends must match).
 *
 *	<name>.pub	- We have located the public key for the peer.  Both
 *			  sides transmit a block encrypted with their private
 *			  keys and the peer's public key.
 *
 *			  Both sides receive a block and decrypt it.
 *
 *			  Both sides formulate a reply using the decrypted
 *			  block and transmit it.
 *
 *			  communication proceeds with the negotiated session
 *			  key (typically AES-256-CBC).
 *
 * If we fail to locate the appropriate file and no floating.db exists the
 * connection is terminated without further action.
 *
 * If floating.db exists the connection proceeds with a floating negotiation.
 */
typedef union {
	struct sockaddr sa;
	struct sockaddr_in sa_in;
	struct sockaddr_in6 sa_in6;
} sockaddr_any_t;

void
hammer2_crypto_negotiate(hammer2_iocom_t *iocom)
{
	sockaddr_any_t sa;
	socklen_t salen = sizeof(sa);
	char peername[128];
	char realname[128];
	hammer2_handshake_t handtx;
	hammer2_handshake_t handrx;
	char buf1[sizeof(handtx)];
	char buf2[sizeof(handtx)];
	char *ptr;
	char *path;
	struct stat st;
	FILE *fp;
	RSA *keys[3] = { NULL, NULL, NULL };
	size_t i;
	size_t blksize;
	size_t blkmask;
	ssize_t n;
	int fd;
	int error;

	/*
	 * Get the peer IP address for the connection as a string.
	 */
	if (getpeername(iocom->sock_fd, &sa.sa, &salen) < 0) {
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_NOPEER;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "accept: getpeername() failed\n");
		goto done;
	}
	if (getnameinfo(&sa.sa, salen, peername, sizeof(peername),
			NULL, 0, NI_NUMERICHOST) < 0) {
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_NOPEER;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "accept: cannot decode sockaddr\n");
		goto done;
	}
	if (DebugOpt) {
		if (realhostname_sa(realname, sizeof(realname),
				    &sa.sa, salen) == HOSTNAME_FOUND) {
			fprintf(stderr, "accept from %s (%s)\n",
				peername, realname);
		} else {
			fprintf(stderr, "accept from %s\n", peername);
		}
	}

	/*
	 * Find the remote host's public key
	 *
	 * If the link is not to be encrypted (<ip>.none located) we shortcut
	 * the handshake entirely.  No buffers are exchanged.
	 */
	asprintf(&path, "%s/%s.pub", HAMMER2_PATH_REMOTE, peername);
	if ((fp = fopen(path, "r")) == NULL) {
		free(path);
		asprintf(&path, "%s/%s.none",
			 HAMMER2_PATH_REMOTE, peername);
		if (stat(path, &st) < 0) {
			iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_NORKEY;
			iocom->flags |= HAMMER2_IOCOMF_EOF;
			if (DebugOpt)
				fprintf(stderr, "auth failure: unknown host\n");
			goto done;
		}
		if (DebugOpt)
			fprintf(stderr, "auth succeeded, unencrypted link\n");
		goto done;
	}
	if (fp) {
		keys[0] = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
		fclose(fp);
		if (keys[0] == NULL) {
			iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_KEYFMT;
			iocom->flags |= HAMMER2_IOCOMF_EOF;
			if (DebugOpt)
				fprintf(stderr,
					"auth failure: bad key format\n");
			goto done;
		}
	}

	/*
	 * Get our public and private keys
	 */
	free(path);
	asprintf(&path, HAMMER2_DEFAULT_DIR "/rsa.pub");
	if ((fp = fopen(path, "r")) == NULL) {
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_NOLKEY;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		goto done;
	}
	keys[1] = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
	fclose(fp);
	if (keys[1] == NULL) {
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_KEYFMT;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "auth failure: bad host key format\n");
		goto done;
	}

	free(path);
	asprintf(&path, HAMMER2_DEFAULT_DIR "/rsa.prv");
	if ((fp = fopen(path, "r")) == NULL) {
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_NOLKEY;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "auth failure: bad host key format\n");
		goto done;
	}
	keys[2] = PEM_read_RSAPrivateKey(fp, NULL, NULL, NULL);
	fclose(fp);
	if (keys[2] == NULL) {
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_KEYFMT;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "auth failure: bad host key format\n");
		goto done;
	}
	free(path);
	path = NULL;

	/*
	 * public key encrypt/decrypt block size.
	 */
	if (keys[0]) {
		blksize = (size_t)RSA_size(keys[0]);
		if (blksize != (size_t)RSA_size(keys[1]) ||
		    blksize != (size_t)RSA_size(keys[2]) ||
		    sizeof(handtx) % blksize != 0) {
			iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_KEYFMT;
			iocom->flags |= HAMMER2_IOCOMF_EOF;
			if (DebugOpt)
				fprintf(stderr, "auth failure: "
						"key size mismatch\n");
			goto done;
		}
	} else {
		blksize = sizeof(handtx);
	}
	blkmask = blksize - 1;

	bzero(&handrx, sizeof(handrx));
	bzero(&handtx, sizeof(handtx));

	/*
	 * Fill all unused fields (particular all junk fields) with random
	 * data, and also set the session key.
	 */
	fd = open("/dev/urandom", O_RDONLY);
	if (fd < 0 ||
	    fstat(fd, &st) < 0 ||	/* something wrong */
	    S_ISREG(st.st_mode) ||	/* supposed to be a RNG dev! */
	    read(fd, &handtx, sizeof(handtx)) != sizeof(handtx)) {
urandfail:
		if (fd >= 0)
			close(fd);
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_BADURANDOM;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "auth failure: bad rng\n");
		goto done;
	}
	if (bcmp(&handrx, &handtx, sizeof(handtx)) == 0)
		goto urandfail;			/* read all zeros */
	close(fd);
	/* ERR_load_crypto_strings(); openssl debugging */

	/*
	 * Handshake with the remote.
	 *
	 *	Encrypt with my private and remote's public
	 *	Decrypt with my private and remote's public
	 *
	 * When encrypting we have to make sure our buffer fits within the
	 * modulus, which typically requires bit 7 o the first byte to be
	 * zero.  To be safe make sure that bit 7 and bit 6 is zero.
	 */
	snprintf(handtx.quickmsg, sizeof(handtx.quickmsg), "Testing 1 2 3");
	handtx.magic = HAMMER2_MSGHDR_MAGIC;
	handtx.version = 1;
	handtx.flags = 0;
	assert(sizeof(handtx.verf) * 4 == sizeof(handtx.sess));
	bzero(handtx.verf, sizeof(handtx.verf));

	handtx.pad1[0] &= 0x3f;	/* message must fit within modulus */
	handtx.pad2[0] &= 0x3f;	/* message must fit within modulus */

	for (i = 0; i < sizeof(handtx.sess); ++i)
		handtx.verf[i / 4] ^= handtx.sess[i];

	/*
	 * Write handshake buffer to remote
	 */
	for (i = 0; i < sizeof(handtx); i += blksize) {
		ptr = (char *)&handtx + i;
		if (keys[0]) {
			/*
			 * Since we are double-encrypting we have to make
			 * sure that the result of the first stage does
			 * not blow out the modulus for the second stage.
			 *
			 * The pointer is pointing to the pad*[] area so
			 * we can mess with that until the first stage
			 * is legal.
			 */
			do {
				++*(int *)(ptr + 4);
				if (RSA_private_encrypt(blksize, ptr, buf1,
					    keys[2], RSA_NO_PADDING) < 0) {
					iocom->ioq_rx.error =
						HAMMER2_IOQ_ERROR_KEYXCHGFAIL;
				}
			} while (buf1[0] & 0xC0);

			if (RSA_public_encrypt(blksize, buf1, buf2,
					    keys[0], RSA_NO_PADDING) < 0) {
				iocom->ioq_rx.error =
					HAMMER2_IOQ_ERROR_KEYXCHGFAIL;
			}
		}
		if (write(iocom->sock_fd, buf2, blksize) != (ssize_t)blksize) {
			fprintf(stderr, "WRITE ERROR\n");
		}
	}
	if (iocom->ioq_rx.error) {
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "auth failure: key exchange failure "
					"during encryption\n");
		goto done;
	}

	/*
	 * Read handshake buffer from remote
	 */
	i = 0;
	while (i < sizeof(handrx)) {
		ptr = (char *)&handrx + i;
		n = read(iocom->sock_fd, ptr, blksize - (i & blkmask));
		if (n <= 0)
			break;
		ptr -= (i & blkmask);
		i += n;
		if (keys[0] && (i & blkmask) == 0) {
			if (RSA_private_decrypt(blksize, ptr, buf1,
					   keys[2], RSA_NO_PADDING) < 0)
				iocom->ioq_rx.error =
						HAMMER2_IOQ_ERROR_KEYXCHGFAIL;
			if (RSA_public_decrypt(blksize, buf1, ptr,
					   keys[0], RSA_NO_PADDING) < 0)
				iocom->ioq_rx.error =
						HAMMER2_IOQ_ERROR_KEYXCHGFAIL;
		}
	}
	if (iocom->ioq_rx.error) {
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "auth failure: key exchange failure "
					"during decryption\n");
		goto done;
	}

	/*
	 * Validate the received data.  Try to make this a constant-time
	 * algorithm.
	 */
	if (i != sizeof(handrx)) {
keyxchgfail:
		iocom->ioq_rx.error = HAMMER2_IOQ_ERROR_KEYXCHGFAIL;
		iocom->flags |= HAMMER2_IOCOMF_EOF;
		if (DebugOpt)
			fprintf(stderr, "auth failure: key exchange failure\n");
		goto done;
	}

	if (handrx.magic == HAMMER2_MSGHDR_MAGIC_REV) {
		handrx.version = bswap16(handrx.version);
		handrx.flags = bswap32(handrx.flags);
	}
	for (i = 0; i < sizeof(handrx.sess); ++i)
		handrx.verf[i / 4] ^= handrx.sess[i];
	n = 0;
	for (i = 0; i < sizeof(handrx.verf); ++i)
		n += handrx.verf[i];
	if (handrx.version != 1)
		++n;
	if (n != 0)
		goto keyxchgfail;

	assert(HAMMER2_AES_KEY_SIZE * 2 == sizeof(handrx.sess));
	/*
	 * Use separate session keys and session fixed IVs for receive and
	 * transmit.
	 */
	error = _gcm_init(&iocom->ioq_rx, handrx.sess, HAMMER2_AES_KEY_SIZE,
	    handrx.sess + HAMMER2_AES_KEY_SIZE,
	    sizeof(handrx.sess) - HAMMER2_AES_KEY_SIZE,
	    0 /* decryption */);
	if (error)
		goto keyxchgfail;

	error = _gcm_init(&iocom->ioq_tx, handtx.sess, HAMMER2_AES_KEY_SIZE,
	    handtx.sess + HAMMER2_AES_KEY_SIZE,
	    sizeof(handtx.sess) - HAMMER2_AES_KEY_SIZE,
	    1 /* encryption */);
	if (error)
		goto keyxchgfail;

	iocom->flags |= HAMMER2_IOCOMF_CRYPTED;

	if (DebugOpt)
		fprintf(stderr, "auth success: %s\n", handrx.quickmsg);
done:
	if (path)
		free(path);
	if (keys[0])
		RSA_free(keys[0]);
	if (keys[1])
		RSA_free(keys[1]);
	if (keys[1])
		RSA_free(keys[2]);
}

/*
 * Decrypt pending data in the ioq's fifo.  The data is decrypted in-place.
 */
void
hammer2_crypto_decrypt(hammer2_iocom_t *iocom __unused, hammer2_ioq_t *ioq)
{
	int p_len;
	int used;
	int error;
	char buf[512];

	/*
	 * fifo_beg to fifo_cdx is data already decrypted.
	 * fifo_cdn to fifo_end is data not yet decrypted.
	 */
	p_len = ioq->fifo_end - ioq->fifo_cdn; /* data not yet decrypted */

	if (p_len == 0)
		return;

	while (p_len >= HAMMER2_CRYPTO_TAG_SIZE + HAMMER2_CRYPTO_CHUNK_SIZE) {
		bcopy(ioq->buf + ioq->fifo_cdn, buf,
		      HAMMER2_CRYPTO_TAG_SIZE + HAMMER2_CRYPTO_CHUNK_SIZE);
		error = hammer2_crypto_decrypt_chunk(ioq, buf,
						     ioq->buf + ioq->fifo_cdx,
						     HAMMER2_CRYPTO_CHUNK_SIZE,
						     &used);
#ifdef CRYPTO_DEBUG
		printf("dec: p_len: %d, used: %d, fifo_cdn: %ju, fifo_cdx: %ju\n",
		       p_len, used, ioq->fifo_cdn, ioq->fifo_cdx);
#endif
		p_len -= used;
		ioq->fifo_cdn += used;
		ioq->fifo_cdx += HAMMER2_CRYPTO_CHUNK_SIZE;
#ifdef CRYPTO_DEBUG
		printf("dec: p_len: %d, used: %d, fifo_cdn: %ju, fifo_cdx: %ju\n",
		       p_len, used, ioq->fifo_cdn, ioq->fifo_cdx);
#endif
	}
}

/*
 * *nactp is set to the number of ORIGINAL bytes consumed by the encrypter.
 * The FIFO may contain more data.
 */
int
hammer2_crypto_encrypt(hammer2_iocom_t *iocom __unused, hammer2_ioq_t *ioq,
		       struct iovec *iov, int n, size_t *nactp)
{
	int p_len, used, ct_used;
	int i;
	int error;
	size_t nmax;

	nmax = sizeof(ioq->buf) - ioq->fifo_end;	/* max new bytes */

	*nactp = 0;
	for (i = 0; i < n && nmax; ++i) {
		used = 0;
		p_len = iov[i].iov_len;
		assert((p_len & HAMMER2_AES_KEY_MASK) == 0);

		while (p_len >= HAMMER2_CRYPTO_CHUNK_SIZE &&
		    nmax >= HAMMER2_CRYPTO_CHUNK_SIZE + HAMMER2_CRYPTO_TAG_SIZE) {
			error = hammer2_crypto_encrypt_chunk(ioq,
			    ioq->buf + ioq->fifo_cdx,
			    (char *)iov[i].iov_base + used,
			    HAMMER2_CRYPTO_CHUNK_SIZE, &ct_used);
#ifdef CRYPTO_DEBUG
			printf("nactp: %ju, p_len: %d, ct_used: %d, used: %d, nmax: %ju\n",
			       *nactp, p_len, ct_used, used, nmax);
#endif

			*nactp += (size_t)HAMMER2_CRYPTO_CHUNK_SIZE;	/* plaintext count */
			used += HAMMER2_CRYPTO_CHUNK_SIZE;
			p_len -= HAMMER2_CRYPTO_CHUNK_SIZE;

			ioq->fifo_cdx += (size_t)ct_used;	/* crypted count */
			ioq->fifo_cdn += (size_t)ct_used;	/* crypted count */
			ioq->fifo_end += (size_t)ct_used;
			nmax -= (size_t)ct_used;
#ifdef CRYPTO_DEBUG
			printf("nactp: %ju, p_len: %d, ct_used: %d, used: %d, nmax: %ju\n",
			       *nactp, p_len, ct_used, used, nmax);
#endif
		}
	}
	iov[0].iov_base = ioq->buf + ioq->fifo_beg;
	iov[0].iov_len = ioq->fifo_cdx - ioq->fifo_beg;

	return (1);
}