libdmsg: Mark some return values unused.
[dragonfly.git] / lib / libdmsg / crypto.c
CommitLineData
62efe6ec
MD
1/*
2 * Copyright (c) 2011-2012 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
766ad73f 7 * by Alex Hornung <alexh@dragonflybsd.org>
62efe6ec
MD
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
18 * distribution.
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
0c3a8cd0 37#include "dmsg_local.h"
62efe6ec 38
02454b3e
MD
39/*
40 * Setup crypto for pthreads
41 */
42static pthread_mutex_t *crypto_locks;
43int crypto_count;
44
0c3a8cd0
MD
45static int dmsg_crypto_gcm_init(dmsg_ioq_t *, char *, int, char *, int, int);
46static int dmsg_crypto_gcm_encrypt_chunk(dmsg_ioq_t *, char *, char *, int, int *);
47static int dmsg_crypto_gcm_decrypt_chunk(dmsg_ioq_t *, char *, char *, int, int *);
487b3235
AH
48
49/*
0c3a8cd0 50 * NOTE: the order of this table needs to match the DMSG_CRYPTO_ALGO_*_IDX
487b3235
AH
51 * defines in network.h.
52 */
53static struct crypto_algo crypto_algos[] = {
54 {
55 .name = "aes-256-gcm",
0c3a8cd0
MD
56 .keylen = DMSG_CRYPTO_GCM_KEY_SIZE,
57 .taglen = DMSG_CRYPTO_GCM_TAG_SIZE,
58 .init = dmsg_crypto_gcm_init,
59 .enc_chunk = dmsg_crypto_gcm_encrypt_chunk,
60 .dec_chunk = dmsg_crypto_gcm_decrypt_chunk
487b3235
AH
61 },
62 { NULL, 0, 0, NULL, NULL, NULL }
63};
64
02454b3e
MD
65static
66unsigned long
0c3a8cd0 67dmsg_crypto_id_callback(void)
02454b3e
MD
68{
69 return ((unsigned long)(uintptr_t)pthread_self());
70}
71
72static
73void
0c3a8cd0 74dmsg_crypto_locking_callback(int mode, int type,
02454b3e
MD
75 const char *file __unused, int line __unused)
76{
77 assert(type >= 0 && type < crypto_count);
78 if (mode & CRYPTO_LOCK) {
79 pthread_mutex_lock(&crypto_locks[type]);
80 } else {
81 pthread_mutex_unlock(&crypto_locks[type]);
82 }
83}
84
85void
0c3a8cd0 86dmsg_crypto_setup(void)
02454b3e
MD
87{
88 crypto_count = CRYPTO_num_locks();
89 crypto_locks = calloc(crypto_count, sizeof(crypto_locks[0]));
0c3a8cd0
MD
90 CRYPTO_set_id_callback(dmsg_crypto_id_callback);
91 CRYPTO_set_locking_callback(dmsg_crypto_locking_callback);
02454b3e
MD
92}
93
766ad73f
AH
94static
95int
0c3a8cd0 96dmsg_crypto_gcm_init(dmsg_ioq_t *ioq, char *key, int klen,
487b3235 97 char *iv_fixed, int ivlen, int enc)
766ad73f
AH
98{
99 int i, ok;
100
0c3a8cd0
MD
101 if (klen < DMSG_CRYPTO_GCM_KEY_SIZE ||
102 ivlen < DMSG_CRYPTO_GCM_IV_FIXED_SIZE) {
103 if (DMsgDebugOpt)
766ad73f
AH
104 fprintf(stderr, "Not enough key or iv material\n");
105 return -1;
106 }
107
108 printf("%s key: ", enc ? "Encryption" : "Decryption");
0c3a8cd0 109 for (i = 0; i < DMSG_CRYPTO_GCM_KEY_SIZE; ++i)
766ad73f
AH
110 printf("%02x", (unsigned char)key[i]);
111 printf("\n");
112
113 printf("%s iv: ", enc ? "Encryption" : "Decryption");
0c3a8cd0 114 for (i = 0; i < DMSG_CRYPTO_GCM_IV_FIXED_SIZE; ++i)
766ad73f
AH
115 printf("%02x", (unsigned char)iv_fixed[i]);
116 printf(" (fixed part only)\n");
117
766ad73f
AH
118 EVP_CIPHER_CTX_init(&ioq->ctx);
119
120 if (enc)
121 ok = EVP_EncryptInit_ex(&ioq->ctx, EVP_aes_256_gcm(), NULL,
122 key, NULL);
123 else
124 ok = EVP_DecryptInit_ex(&ioq->ctx, EVP_aes_256_gcm(), NULL,
125 key, NULL);
126 if (!ok)
127 goto fail;
128
129 /*
130 * According to the original Galois/Counter Mode of Operation (GCM)
131 * proposal, only IVs that are exactly 96 bits get used without any
132 * further processing. Other IV sizes cause the GHASH() operation
133 * to be applied to the IV, which is more costly.
134 *
135 * The NIST SP 800-38D also recommends using a 96 bit IV for the same
136 * reasons. We actually follow the deterministic construction
137 * recommended in NIST SP 800-38D with a 64 bit invocation field as an
138 * integer counter and a random, session-specific fixed field.
139 *
140 * This means that we can essentially use the same session key and
141 * IV fixed field for up to 2^64 invocations of the authenticated
142 * encryption or decryption.
143 *
144 * With a chunk size of 64 bytes, this adds up to 1 zettabyte of
145 * traffic.
146 */
147 ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_SET_IVLEN,
0c3a8cd0 148 DMSG_CRYPTO_GCM_IV_SIZE, NULL);
766ad73f
AH
149 if (!ok)
150 goto fail;
151
0c3a8cd0
MD
152 memset(ioq->iv, 0, DMSG_CRYPTO_GCM_IV_SIZE);
153 memcpy(ioq->iv, iv_fixed, DMSG_CRYPTO_GCM_IV_FIXED_SIZE);
766ad73f
AH
154
155 /*
156 * Strictly speaking, padding is irrelevant with a counter mode
157 * encryption.
158 *
159 * However, setting padding to 0, even if using a counter mode such
160 * as GCM, will cause an error in _finish if the pt/ct size is not
161 * a multiple of the cipher block size.
162 */
163 EVP_CIPHER_CTX_set_padding(&ioq->ctx, 0);
164
165 return 0;
166
167fail:
0c3a8cd0 168 if (DMsgDebugOpt)
766ad73f
AH
169 fprintf(stderr, "Error during _gcm_init\n");
170 return -1;
171}
172
173static
174int
175_gcm_iv_increment(char *iv)
176{
177 /*
178 * Deterministic construction according to NIST SP 800-38D, with
179 * 64 bit invocation field as integer counter.
180 *
181 * In other words, our 96 bit IV consists of a 32 bit fixed field
182 * unique to the session and a 64 bit integer counter.
183 */
184
0c3a8cd0 185 uint64_t *c = (uint64_t *)(&iv[DMSG_CRYPTO_GCM_IV_FIXED_SIZE]);
766ad73f
AH
186
187 /* Increment invocation field integer counter */
04b0b195 188 *c = htobe64(be64toh(*c)+1);
766ad73f
AH
189
190 /*
191 * Detect wrap-around, which means it is time to renegotiate
192 * the session to get a new key and/or fixed field.
193 */
fd1d02a5 194 return (c == 0) ? 0 : 1;
766ad73f
AH
195}
196
197static
198int
0c3a8cd0 199dmsg_crypto_gcm_encrypt_chunk(dmsg_ioq_t *ioq, char *ct, char *pt,
487b3235 200 int in_size, int *out_size)
766ad73f
AH
201{
202 int ok;
766ad73f
AH
203 int u_len, f_len;
204
205 *out_size = 0;
206
207 /* Re-initialize with new IV (but without redoing the key schedule) */
208 ok = EVP_EncryptInit_ex(&ioq->ctx, NULL, NULL, NULL, ioq->iv);
209 if (!ok)
210 goto fail;
211
212 ok = EVP_EncryptUpdate(&ioq->ctx, ct, &u_len, pt, in_size);
213 if (!ok)
214 goto fail;
215
216 ok = EVP_EncryptFinal(&ioq->ctx, ct + u_len, &f_len);
217 if (!ok)
218 goto fail;
219
220 /* Retrieve auth tag */
221 ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_GET_TAG,
0c3a8cd0 222 DMSG_CRYPTO_GCM_TAG_SIZE,
766ad73f
AH
223 ct + u_len + f_len);
224 if (!ok)
225 goto fail;
226
fd1d02a5
AH
227 ok = _gcm_iv_increment(ioq->iv);
228 if (!ok) {
0c3a8cd0 229 ioq->error = DMSG_IOQ_ERROR_IVWRAP;
fd1d02a5
AH
230 goto fail_out;
231 }
766ad73f 232
0c3a8cd0 233 *out_size = u_len + f_len + DMSG_CRYPTO_GCM_TAG_SIZE;
766ad73f
AH
234
235 return 0;
236
237fail:
0c3a8cd0 238 ioq->error = DMSG_IOQ_ERROR_ALGO;
fd1d02a5 239fail_out:
0c3a8cd0 240 if (DMsgDebugOpt)
766ad73f
AH
241 fprintf(stderr, "error during encrypt_chunk\n");
242 return -1;
243}
244
245static
246int
0c3a8cd0 247dmsg_crypto_gcm_decrypt_chunk(dmsg_ioq_t *ioq, char *ct, char *pt,
487b3235 248 int out_size, int *consume_size)
766ad73f
AH
249{
250 int ok;
766ad73f
AH
251 int u_len, f_len;
252
253 *consume_size = 0;
254
255 /* Re-initialize with new IV (but without redoing the key schedule) */
256 ok = EVP_DecryptInit_ex(&ioq->ctx, NULL, NULL, NULL, ioq->iv);
fd1d02a5 257 if (!ok) {
0c3a8cd0 258 ioq->error = DMSG_IOQ_ERROR_ALGO;
fd1d02a5
AH
259 goto fail_out;
260 }
766ad73f 261
766ad73f 262 ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_SET_TAG,
0c3a8cd0 263 DMSG_CRYPTO_GCM_TAG_SIZE,
766ad73f 264 ct + out_size);
fd1d02a5 265 if (!ok) {
0c3a8cd0 266 ioq->error = DMSG_IOQ_ERROR_ALGO;
fd1d02a5
AH
267 goto fail_out;
268 }
766ad73f
AH
269
270 ok = EVP_DecryptUpdate(&ioq->ctx, pt, &u_len, ct, out_size);
271 if (!ok)
272 goto fail;
273
274 ok = EVP_DecryptFinal(&ioq->ctx, pt + u_len, &f_len);
275 if (!ok)
276 goto fail;
277
fd1d02a5
AH
278 ok = _gcm_iv_increment(ioq->iv);
279 if (!ok) {
0c3a8cd0 280 ioq->error = DMSG_IOQ_ERROR_IVWRAP;
fd1d02a5
AH
281 goto fail_out;
282 }
766ad73f 283
0c3a8cd0 284 *consume_size = u_len + f_len + DMSG_CRYPTO_GCM_TAG_SIZE;
766ad73f
AH
285
286 return 0;
287
288fail:
0c3a8cd0 289 ioq->error = DMSG_IOQ_ERROR_MACFAIL;
fd1d02a5 290fail_out:
0c3a8cd0 291 if (DMsgDebugOpt)
766ad73f
AH
292 fprintf(stderr, "error during decrypt_chunk (likely authentication error)\n");
293 return -1;
294}
295
62efe6ec
MD
296/*
297 * Synchronously negotiate crypto for a new session. This must occur
298 * within 10 seconds or the connection is error'd out.
299 *
300 * We work off the IP address and/or reverse DNS. The IP address is
301 * checked first, followed by the IP address at various levels of granularity,
302 * followed by the full domain name and domain names at various levels of
303 * granularity.
304 *
305 * /etc/hammer2/remote/<name>.pub - Contains a public key
306 * /etc/hammer2/remote/<name>.none - Indicates no encryption (empty file)
307 * (e.g. localhost.none).
308 *
309 * We first attempt to locate a public key file based on the peer address or
310 * peer FQDN.
311 *
312 * <name>.none - No further negotiation is needed. We simply return.
313 * All communication proceeds without encryption.
314 * No public key handshake occurs in this situation.
315 * (both ends must match).
316 *
317 * <name>.pub - We have located the public key for the peer. Both
318 * sides transmit a block encrypted with their private
319 * keys and the peer's public key.
320 *
321 * Both sides receive a block and decrypt it.
322 *
323 * Both sides formulate a reply using the decrypted
324 * block and transmit it.
325 *
326 * communication proceeds with the negotiated session
327 * key (typically AES-256-CBC).
328 *
329 * If we fail to locate the appropriate file and no floating.db exists the
330 * connection is terminated without further action.
331 *
332 * If floating.db exists the connection proceeds with a floating negotiation.
333 */
334typedef union {
335 struct sockaddr sa;
336 struct sockaddr_in sa_in;
337 struct sockaddr_in6 sa_in6;
338} sockaddr_any_t;
339
340void
0c3a8cd0 341dmsg_crypto_negotiate(dmsg_iocom_t *iocom)
62efe6ec
MD
342{
343 sockaddr_any_t sa;
344 socklen_t salen = sizeof(sa);
345 char peername[128];
346 char realname[128];
0c3a8cd0
MD
347 dmsg_handshake_t handtx;
348 dmsg_handshake_t handrx;
5cf97ec5
MD
349 char buf1[sizeof(handtx)];
350 char buf2[sizeof(handtx)];
62efe6ec
MD
351 char *ptr;
352 char *path;
353 struct stat st;
354 FILE *fp;
355 RSA *keys[3] = { NULL, NULL, NULL };
356 size_t i;
357 size_t blksize;
358 size_t blkmask;
359 ssize_t n;
360 int fd;
766ad73f 361 int error;
62efe6ec
MD
362
363 /*
364 * Get the peer IP address for the connection as a string.
365 */
366 if (getpeername(iocom->sock_fd, &sa.sa, &salen) < 0) {
0c3a8cd0 367 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOPEER;
a2179323 368 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 369 if (DMsgDebugOpt)
62efe6ec
MD
370 fprintf(stderr, "accept: getpeername() failed\n");
371 goto done;
372 }
373 if (getnameinfo(&sa.sa, salen, peername, sizeof(peername),
374 NULL, 0, NI_NUMERICHOST) < 0) {
0c3a8cd0 375 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOPEER;
a2179323 376 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 377 if (DMsgDebugOpt)
62efe6ec
MD
378 fprintf(stderr, "accept: cannot decode sockaddr\n");
379 goto done;
380 }
0c3a8cd0 381 if (DMsgDebugOpt) {
62efe6ec
MD
382 if (realhostname_sa(realname, sizeof(realname),
383 &sa.sa, salen) == HOSTNAME_FOUND) {
384 fprintf(stderr, "accept from %s (%s)\n",
385 peername, realname);
386 } else {
387 fprintf(stderr, "accept from %s\n", peername);
388 }
389 }
390
391 /*
392 * Find the remote host's public key
9b8b748f
MD
393 *
394 * If the link is not to be encrypted (<ip>.none located) we shortcut
395 * the handshake entirely. No buffers are exchanged.
62efe6ec 396 */
0c3a8cd0 397 asprintf(&path, "%s/%s.pub", DMSG_PATH_REMOTE, peername);
62efe6ec
MD
398 if ((fp = fopen(path, "r")) == NULL) {
399 free(path);
400 asprintf(&path, "%s/%s.none",
0c3a8cd0 401 DMSG_PATH_REMOTE, peername);
62efe6ec 402 if (stat(path, &st) < 0) {
0c3a8cd0 403 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NORKEY;
a2179323 404 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 405 if (DMsgDebugOpt)
62efe6ec
MD
406 fprintf(stderr, "auth failure: unknown host\n");
407 goto done;
408 }
0c3a8cd0 409 if (DMsgDebugOpt)
62efe6ec 410 fprintf(stderr, "auth succeeded, unencrypted link\n");
9b8b748f 411 goto done;
62efe6ec
MD
412 }
413 if (fp) {
414 keys[0] = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
415 fclose(fp);
416 if (keys[0] == NULL) {
0c3a8cd0 417 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
a2179323 418 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 419 if (DMsgDebugOpt)
62efe6ec
MD
420 fprintf(stderr,
421 "auth failure: bad key format\n");
422 goto done;
423 }
424 }
425
426 /*
427 * Get our public and private keys
428 */
429 free(path);
0c3a8cd0 430 asprintf(&path, DMSG_DEFAULT_DIR "/rsa.pub");
62efe6ec 431 if ((fp = fopen(path, "r")) == NULL) {
0c3a8cd0 432 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOLKEY;
a2179323 433 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
62efe6ec
MD
434 goto done;
435 }
436 keys[1] = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
437 fclose(fp);
438 if (keys[1] == NULL) {
0c3a8cd0 439 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
a2179323 440 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 441 if (DMsgDebugOpt)
62efe6ec
MD
442 fprintf(stderr, "auth failure: bad host key format\n");
443 goto done;
444 }
445
446 free(path);
0c3a8cd0 447 asprintf(&path, DMSG_DEFAULT_DIR "/rsa.prv");
62efe6ec 448 if ((fp = fopen(path, "r")) == NULL) {
0c3a8cd0 449 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOLKEY;
a2179323 450 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 451 if (DMsgDebugOpt)
62efe6ec
MD
452 fprintf(stderr, "auth failure: bad host key format\n");
453 goto done;
454 }
455 keys[2] = PEM_read_RSAPrivateKey(fp, NULL, NULL, NULL);
456 fclose(fp);
457 if (keys[2] == NULL) {
0c3a8cd0 458 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
a2179323 459 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 460 if (DMsgDebugOpt)
62efe6ec
MD
461 fprintf(stderr, "auth failure: bad host key format\n");
462 goto done;
463 }
464 free(path);
465 path = NULL;
466
467 /*
468 * public key encrypt/decrypt block size.
469 */
470 if (keys[0]) {
471 blksize = (size_t)RSA_size(keys[0]);
472 if (blksize != (size_t)RSA_size(keys[1]) ||
473 blksize != (size_t)RSA_size(keys[2]) ||
474 sizeof(handtx) % blksize != 0) {
0c3a8cd0 475 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
a2179323 476 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 477 if (DMsgDebugOpt)
62efe6ec
MD
478 fprintf(stderr, "auth failure: "
479 "key size mismatch\n");
480 goto done;
481 }
482 } else {
483 blksize = sizeof(handtx);
484 }
485 blkmask = blksize - 1;
486
487 bzero(&handrx, sizeof(handrx));
488 bzero(&handtx, sizeof(handtx));
489
490 /*
491 * Fill all unused fields (particular all junk fields) with random
492 * data, and also set the session key.
493 */
494 fd = open("/dev/urandom", O_RDONLY);
495 if (fd < 0 ||
496 fstat(fd, &st) < 0 || /* something wrong */
497 S_ISREG(st.st_mode) || /* supposed to be a RNG dev! */
498 read(fd, &handtx, sizeof(handtx)) != sizeof(handtx)) {
499urandfail:
500 if (fd >= 0)
501 close(fd);
0c3a8cd0 502 iocom->ioq_rx.error = DMSG_IOQ_ERROR_BADURANDOM;
a2179323 503 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 504 if (DMsgDebugOpt)
62efe6ec
MD
505 fprintf(stderr, "auth failure: bad rng\n");
506 goto done;
507 }
508 if (bcmp(&handrx, &handtx, sizeof(handtx)) == 0)
509 goto urandfail; /* read all zeros */
510 close(fd);
5cf97ec5 511 /* ERR_load_crypto_strings(); openssl debugging */
62efe6ec
MD
512
513 /*
514 * Handshake with the remote.
515 *
516 * Encrypt with my private and remote's public
517 * Decrypt with my private and remote's public
518 *
519 * When encrypting we have to make sure our buffer fits within the
520 * modulus, which typically requires bit 7 o the first byte to be
521 * zero. To be safe make sure that bit 7 and bit 6 is zero.
522 */
523 snprintf(handtx.quickmsg, sizeof(handtx.quickmsg), "Testing 1 2 3");
5bc5bca2 524 handtx.magic = DMSG_HDR_MAGIC;
62efe6ec
MD
525 handtx.version = 1;
526 handtx.flags = 0;
527 assert(sizeof(handtx.verf) * 4 == sizeof(handtx.sess));
528 bzero(handtx.verf, sizeof(handtx.verf));
529
530 handtx.pad1[0] &= 0x3f; /* message must fit within modulus */
531 handtx.pad2[0] &= 0x3f; /* message must fit within modulus */
532
533 for (i = 0; i < sizeof(handtx.sess); ++i)
534 handtx.verf[i / 4] ^= handtx.sess[i];
535
536 /*
537 * Write handshake buffer to remote
538 */
539 for (i = 0; i < sizeof(handtx); i += blksize) {
540 ptr = (char *)&handtx + i;
541 if (keys[0]) {
542 /*
543 * Since we are double-encrypting we have to make
544 * sure that the result of the first stage does
545 * not blow out the modulus for the second stage.
546 *
547 * The pointer is pointing to the pad*[] area so
548 * we can mess with that until the first stage
549 * is legal.
550 */
551 do {
552 ++*(int *)(ptr + 4);
5cf97ec5 553 if (RSA_private_encrypt(blksize, ptr, buf1,
62efe6ec
MD
554 keys[2], RSA_NO_PADDING) < 0) {
555 iocom->ioq_rx.error =
0c3a8cd0 556 DMSG_IOQ_ERROR_KEYXCHGFAIL;
62efe6ec 557 }
5cf97ec5 558 } while (buf1[0] & 0xC0);
62efe6ec 559
5cf97ec5 560 if (RSA_public_encrypt(blksize, buf1, buf2,
62efe6ec
MD
561 keys[0], RSA_NO_PADDING) < 0) {
562 iocom->ioq_rx.error =
0c3a8cd0 563 DMSG_IOQ_ERROR_KEYXCHGFAIL;
62efe6ec
MD
564 }
565 }
5cf97ec5 566 if (write(iocom->sock_fd, buf2, blksize) != (ssize_t)blksize) {
62efe6ec
MD
567 fprintf(stderr, "WRITE ERROR\n");
568 }
569 }
570 if (iocom->ioq_rx.error) {
a2179323 571 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 572 if (DMsgDebugOpt)
62efe6ec
MD
573 fprintf(stderr, "auth failure: key exchange failure "
574 "during encryption\n");
575 goto done;
576 }
577
578 /*
579 * Read handshake buffer from remote
580 */
581 i = 0;
582 while (i < sizeof(handrx)) {
583 ptr = (char *)&handrx + i;
584 n = read(iocom->sock_fd, ptr, blksize - (i & blkmask));
585 if (n <= 0)
586 break;
587 ptr -= (i & blkmask);
588 i += n;
589 if (keys[0] && (i & blkmask) == 0) {
5cf97ec5 590 if (RSA_private_decrypt(blksize, ptr, buf1,
62efe6ec
MD
591 keys[2], RSA_NO_PADDING) < 0)
592 iocom->ioq_rx.error =
0c3a8cd0 593 DMSG_IOQ_ERROR_KEYXCHGFAIL;
5cf97ec5 594 if (RSA_public_decrypt(blksize, buf1, ptr,
62efe6ec
MD
595 keys[0], RSA_NO_PADDING) < 0)
596 iocom->ioq_rx.error =
0c3a8cd0 597 DMSG_IOQ_ERROR_KEYXCHGFAIL;
62efe6ec
MD
598 }
599 }
600 if (iocom->ioq_rx.error) {
a2179323 601 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 602 if (DMsgDebugOpt)
62efe6ec
MD
603 fprintf(stderr, "auth failure: key exchange failure "
604 "during decryption\n");
605 goto done;
606 }
607
608 /*
609 * Validate the received data. Try to make this a constant-time
610 * algorithm.
611 */
612 if (i != sizeof(handrx)) {
613keyxchgfail:
0c3a8cd0 614 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYXCHGFAIL;
a2179323 615 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
0c3a8cd0 616 if (DMsgDebugOpt)
62efe6ec
MD
617 fprintf(stderr, "auth failure: key exchange failure\n");
618 goto done;
619 }
620
5bc5bca2 621 if (handrx.magic == DMSG_HDR_MAGIC_REV) {
62efe6ec
MD
622 handrx.version = bswap16(handrx.version);
623 handrx.flags = bswap32(handrx.flags);
624 }
625 for (i = 0; i < sizeof(handrx.sess); ++i)
626 handrx.verf[i / 4] ^= handrx.sess[i];
627 n = 0;
628 for (i = 0; i < sizeof(handrx.verf); ++i)
629 n += handrx.verf[i];
630 if (handrx.version != 1)
631 ++n;
632 if (n != 0)
633 goto keyxchgfail;
634
5cf97ec5 635 /*
766ad73f
AH
636 * Use separate session keys and session fixed IVs for receive and
637 * transmit.
5cf97ec5 638 */
0c3a8cd0
MD
639 error = crypto_algos[DMSG_CRYPTO_ALGO].init(&iocom->ioq_rx, handrx.sess,
640 crypto_algos[DMSG_CRYPTO_ALGO].keylen,
641 handrx.sess + crypto_algos[DMSG_CRYPTO_ALGO].keylen,
642 sizeof(handrx.sess) - crypto_algos[DMSG_CRYPTO_ALGO].keylen,
766ad73f
AH
643 0 /* decryption */);
644 if (error)
645 goto keyxchgfail;
5cf97ec5 646
0c3a8cd0
MD
647 error = crypto_algos[DMSG_CRYPTO_ALGO].init(&iocom->ioq_tx, handtx.sess,
648 crypto_algos[DMSG_CRYPTO_ALGO].keylen,
649 handtx.sess + crypto_algos[DMSG_CRYPTO_ALGO].keylen,
650 sizeof(handtx.sess) - crypto_algos[DMSG_CRYPTO_ALGO].keylen,
766ad73f
AH
651 1 /* encryption */);
652 if (error)
653 goto keyxchgfail;
5cf97ec5 654
a2179323 655 atomic_set_int(&iocom->flags, DMSG_IOCOMF_CRYPTED);
5cf97ec5 656
0c3a8cd0 657 if (DMsgDebugOpt)
5cf97ec5 658 fprintf(stderr, "auth success: %s\n", handrx.quickmsg);
62efe6ec
MD
659done:
660 if (path)
661 free(path);
662 if (keys[0])
663 RSA_free(keys[0]);
664 if (keys[1])
665 RSA_free(keys[1]);
666 if (keys[1])
667 RSA_free(keys[2]);
668}
5cf97ec5
MD
669
670/*
671 * Decrypt pending data in the ioq's fifo. The data is decrypted in-place.
672 */
673void
0c3a8cd0 674dmsg_crypto_decrypt(dmsg_iocom_t *iocom __unused, dmsg_ioq_t *ioq)
5cf97ec5
MD
675{
676 int p_len;
766ad73f 677 int used;
579191cb 678 __unused int error; /* XXX */
5cf97ec5
MD
679 char buf[512];
680
766ad73f
AH
681 /*
682 * fifo_beg to fifo_cdx is data already decrypted.
683 * fifo_cdn to fifo_end is data not yet decrypted.
684 */
685 p_len = ioq->fifo_end - ioq->fifo_cdn; /* data not yet decrypted */
3033ecc8 686
5cf97ec5
MD
687 if (p_len == 0)
688 return;
766ad73f 689
0c3a8cd0
MD
690 while (p_len >= crypto_algos[DMSG_CRYPTO_ALGO].taglen +
691 DMSG_CRYPTO_CHUNK_SIZE) {
766ad73f 692 bcopy(ioq->buf + ioq->fifo_cdn, buf,
0c3a8cd0
MD
693 crypto_algos[DMSG_CRYPTO_ALGO].taglen +
694 DMSG_CRYPTO_CHUNK_SIZE);
695 error = crypto_algos[DMSG_CRYPTO_ALGO].dec_chunk(
487b3235
AH
696 ioq, buf,
697 ioq->buf + ioq->fifo_cdx,
0c3a8cd0 698 DMSG_CRYPTO_CHUNK_SIZE,
487b3235 699 &used);
766ad73f
AH
700#ifdef CRYPTO_DEBUG
701 printf("dec: p_len: %d, used: %d, fifo_cdn: %ju, fifo_cdx: %ju\n",
702 p_len, used, ioq->fifo_cdn, ioq->fifo_cdx);
703#endif
704 p_len -= used;
705 ioq->fifo_cdn += used;
0c3a8cd0 706 ioq->fifo_cdx += DMSG_CRYPTO_CHUNK_SIZE;
766ad73f
AH
707#ifdef CRYPTO_DEBUG
708 printf("dec: p_len: %d, used: %d, fifo_cdn: %ju, fifo_cdx: %ju\n",
709 p_len, used, ioq->fifo_cdn, ioq->fifo_cdx);
710#endif
5cf97ec5 711 }
5cf97ec5
MD
712}
713
714/*
3033ecc8
MD
715 * *nactp is set to the number of ORIGINAL bytes consumed by the encrypter.
716 * The FIFO may contain more data.
5cf97ec5 717 */
5cf97ec5 718int
0c3a8cd0
MD
719dmsg_crypto_encrypt(dmsg_iocom_t *iocom __unused, dmsg_ioq_t *ioq,
720 struct iovec *iov, int n, size_t *nactp)
5cf97ec5 721{
766ad73f 722 int p_len, used, ct_used;
5cf97ec5 723 int i;
579191cb 724 __unused int error; /* XXX */
3033ecc8 725 size_t nmax;
5cf97ec5 726
3033ecc8 727 nmax = sizeof(ioq->buf) - ioq->fifo_end; /* max new bytes */
5cf97ec5 728
3033ecc8
MD
729 *nactp = 0;
730 for (i = 0; i < n && nmax; ++i) {
766ad73f 731 used = 0;
5cf97ec5 732 p_len = iov[i].iov_len;
5bc5bca2 733 assert((p_len & DMSG_ALIGNMASK) == 0);
766ad73f 734
0c3a8cd0
MD
735 while (p_len >= DMSG_CRYPTO_CHUNK_SIZE &&
736 nmax >= DMSG_CRYPTO_CHUNK_SIZE +
737 (size_t)crypto_algos[DMSG_CRYPTO_ALGO].taglen) {
738 error = crypto_algos[DMSG_CRYPTO_ALGO].enc_chunk(
487b3235 739 ioq,
766ad73f
AH
740 ioq->buf + ioq->fifo_cdx,
741 (char *)iov[i].iov_base + used,
0c3a8cd0 742 DMSG_CRYPTO_CHUNK_SIZE, &ct_used);
766ad73f
AH
743#ifdef CRYPTO_DEBUG
744 printf("nactp: %ju, p_len: %d, ct_used: %d, used: %d, nmax: %ju\n",
745 *nactp, p_len, ct_used, used, nmax);
746#endif
747
0c3a8cd0
MD
748 *nactp += (size_t)DMSG_CRYPTO_CHUNK_SIZE; /* plaintext count */
749 used += DMSG_CRYPTO_CHUNK_SIZE;
750 p_len -= DMSG_CRYPTO_CHUNK_SIZE;
766ad73f
AH
751
752 ioq->fifo_cdx += (size_t)ct_used; /* crypted count */
753 ioq->fifo_cdn += (size_t)ct_used; /* crypted count */
754 ioq->fifo_end += (size_t)ct_used;
755 nmax -= (size_t)ct_used;
756#ifdef CRYPTO_DEBUG
757 printf("nactp: %ju, p_len: %d, ct_used: %d, used: %d, nmax: %ju\n",
758 *nactp, p_len, ct_used, used, nmax);
759#endif
760 }
5cf97ec5
MD
761 }
762 iov[0].iov_base = ioq->buf + ioq->fifo_beg;
763 iov[0].iov_len = ioq->fifo_cdx - ioq->fifo_beg;
764
765 return (1);
766}