2 * Copyright (c) 2011-2012 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@dragonflybsd.org>
6 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
7 * by Alex Hornung <alexh@dragonflybsd.org>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in
17 * the documentation and/or other materials provided with the
19 * 3. Neither the name of The DragonFly Project nor the names of its
20 * contributors may be used to endorse or promote products derived
21 * from this software without specific, prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 #include "dmsg_local.h"
40 * Setup crypto for pthreads
42 static pthread_mutex_t *crypto_locks;
45 static int dmsg_crypto_gcm_init(dmsg_ioq_t *, char *, int, char *, int, int);
46 static int dmsg_crypto_gcm_encrypt_chunk(dmsg_ioq_t *, char *, char *, int, int *);
47 static int dmsg_crypto_gcm_decrypt_chunk(dmsg_ioq_t *, char *, char *, int, int *);
50 * NOTE: the order of this table needs to match the DMSG_CRYPTO_ALGO_*_IDX
51 * defines in network.h.
53 static struct crypto_algo crypto_algos[] = {
55 .name = "aes-256-gcm",
56 .keylen = DMSG_CRYPTO_GCM_KEY_SIZE,
57 .taglen = DMSG_CRYPTO_GCM_TAG_SIZE,
58 .init = dmsg_crypto_gcm_init,
59 .enc_chunk = dmsg_crypto_gcm_encrypt_chunk,
60 .dec_chunk = dmsg_crypto_gcm_decrypt_chunk
62 { NULL, 0, 0, NULL, NULL, NULL }
67 dmsg_crypto_id_callback(void)
69 return ((unsigned long)(uintptr_t)pthread_self());
74 dmsg_crypto_locking_callback(int mode, int type,
75 const char *file __unused, int line __unused)
77 assert(type >= 0 && type < crypto_count);
78 if (mode & CRYPTO_LOCK) {
79 pthread_mutex_lock(&crypto_locks[type]);
81 pthread_mutex_unlock(&crypto_locks[type]);
86 dmsg_crypto_setup(void)
88 crypto_count = CRYPTO_num_locks();
89 crypto_locks = calloc(crypto_count, sizeof(crypto_locks[0]));
90 CRYPTO_set_id_callback(dmsg_crypto_id_callback);
91 CRYPTO_set_locking_callback(dmsg_crypto_locking_callback);
96 dmsg_crypto_gcm_init(dmsg_ioq_t *ioq, char *key, int klen,
97 char *iv_fixed, int ivlen, int enc)
101 if (klen < DMSG_CRYPTO_GCM_KEY_SIZE ||
102 ivlen < DMSG_CRYPTO_GCM_IV_FIXED_SIZE) {
104 fprintf(stderr, "Not enough key or iv material\n");
108 printf("%s key: ", enc ? "Encryption" : "Decryption");
109 for (i = 0; i < DMSG_CRYPTO_GCM_KEY_SIZE; ++i)
110 printf("%02x", (unsigned char)key[i]);
113 printf("%s iv: ", enc ? "Encryption" : "Decryption");
114 for (i = 0; i < DMSG_CRYPTO_GCM_IV_FIXED_SIZE; ++i)
115 printf("%02x", (unsigned char)iv_fixed[i]);
116 printf(" (fixed part only)\n");
118 EVP_CIPHER_CTX_init(&ioq->ctx);
121 ok = EVP_EncryptInit_ex(&ioq->ctx, EVP_aes_256_gcm(), NULL,
124 ok = EVP_DecryptInit_ex(&ioq->ctx, EVP_aes_256_gcm(), NULL,
130 * According to the original Galois/Counter Mode of Operation (GCM)
131 * proposal, only IVs that are exactly 96 bits get used without any
132 * further processing. Other IV sizes cause the GHASH() operation
133 * to be applied to the IV, which is more costly.
135 * The NIST SP 800-38D also recommends using a 96 bit IV for the same
136 * reasons. We actually follow the deterministic construction
137 * recommended in NIST SP 800-38D with a 64 bit invocation field as an
138 * integer counter and a random, session-specific fixed field.
140 * This means that we can essentially use the same session key and
141 * IV fixed field for up to 2^64 invocations of the authenticated
142 * encryption or decryption.
144 * With a chunk size of 64 bytes, this adds up to 1 zettabyte of
147 ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_SET_IVLEN,
148 DMSG_CRYPTO_GCM_IV_SIZE, NULL);
152 memset(ioq->iv, 0, DMSG_CRYPTO_GCM_IV_SIZE);
153 memcpy(ioq->iv, iv_fixed, DMSG_CRYPTO_GCM_IV_FIXED_SIZE);
156 * Strictly speaking, padding is irrelevant with a counter mode
159 * However, setting padding to 0, even if using a counter mode such
160 * as GCM, will cause an error in _finish if the pt/ct size is not
161 * a multiple of the cipher block size.
163 EVP_CIPHER_CTX_set_padding(&ioq->ctx, 0);
169 fprintf(stderr, "Error during _gcm_init\n");
175 _gcm_iv_increment(char *iv)
178 * Deterministic construction according to NIST SP 800-38D, with
179 * 64 bit invocation field as integer counter.
181 * In other words, our 96 bit IV consists of a 32 bit fixed field
182 * unique to the session and a 64 bit integer counter.
185 uint64_t *c = (uint64_t *)(&iv[DMSG_CRYPTO_GCM_IV_FIXED_SIZE]);
187 /* Increment invocation field integer counter */
188 *c = htobe64(be64toh(*c)+1);
191 * Detect wrap-around, which means it is time to renegotiate
192 * the session to get a new key and/or fixed field.
194 return (*c == 0) ? 0 : 1;
199 dmsg_crypto_gcm_encrypt_chunk(dmsg_ioq_t *ioq, char *ct, char *pt,
200 int in_size, int *out_size)
207 /* Re-initialize with new IV (but without redoing the key schedule) */
208 ok = EVP_EncryptInit_ex(&ioq->ctx, NULL, NULL, NULL, ioq->iv);
212 u_len = 0; /* safety */
213 ok = EVP_EncryptUpdate(&ioq->ctx, ct, &u_len, pt, in_size);
217 f_len = 0; /* safety */
218 ok = EVP_EncryptFinal(&ioq->ctx, ct + u_len, &f_len);
222 /* Retrieve auth tag */
223 ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_GET_TAG,
224 DMSG_CRYPTO_GCM_TAG_SIZE,
229 ok = _gcm_iv_increment(ioq->iv);
231 ioq->error = DMSG_IOQ_ERROR_IVWRAP;
235 *out_size = u_len + f_len + DMSG_CRYPTO_GCM_TAG_SIZE;
240 ioq->error = DMSG_IOQ_ERROR_ALGO;
243 fprintf(stderr, "error during encrypt_chunk\n");
249 dmsg_crypto_gcm_decrypt_chunk(dmsg_ioq_t *ioq, char *ct, char *pt,
250 int out_size, int *consume_size)
257 /* Re-initialize with new IV (but without redoing the key schedule) */
258 ok = EVP_DecryptInit_ex(&ioq->ctx, NULL, NULL, NULL, ioq->iv);
260 ioq->error = DMSG_IOQ_ERROR_ALGO;
264 ok = EVP_CIPHER_CTX_ctrl(&ioq->ctx, EVP_CTRL_GCM_SET_TAG,
265 DMSG_CRYPTO_GCM_TAG_SIZE,
268 ioq->error = DMSG_IOQ_ERROR_ALGO;
272 ok = EVP_DecryptUpdate(&ioq->ctx, pt, &u_len, ct, out_size);
276 ok = EVP_DecryptFinal(&ioq->ctx, pt + u_len, &f_len);
280 ok = _gcm_iv_increment(ioq->iv);
282 ioq->error = DMSG_IOQ_ERROR_IVWRAP;
286 *consume_size = u_len + f_len + DMSG_CRYPTO_GCM_TAG_SIZE;
291 ioq->error = DMSG_IOQ_ERROR_MACFAIL;
294 fprintf(stderr, "error during decrypt_chunk (likely authentication error)\n");
299 * Synchronously negotiate crypto for a new session. This must occur
300 * within 10 seconds or the connection is error'd out.
302 * We work off the IP address and/or reverse DNS. The IP address is
303 * checked first, followed by the IP address at various levels of granularity,
304 * followed by the full domain name and domain names at various levels of
307 * /etc/hammer2/remote/<name>.pub - Contains a public key
308 * /etc/hammer2/remote/<name>.none - Indicates no encryption (empty file)
309 * (e.g. localhost.none).
311 * We first attempt to locate a public key file based on the peer address or
314 * <name>.none - No further negotiation is needed. We simply return.
315 * All communication proceeds without encryption.
316 * No public key handshake occurs in this situation.
317 * (both ends must match).
319 * <name>.pub - We have located the public key for the peer. Both
320 * sides transmit a block encrypted with their private
321 * keys and the peer's public key.
323 * Both sides receive a block and decrypt it.
325 * Both sides formulate a reply using the decrypted
326 * block and transmit it.
328 * communication proceeds with the negotiated session
329 * key (typically AES-256-CBC).
331 * If we fail to locate the appropriate file and no floating.db exists the
332 * connection is terminated without further action.
334 * If floating.db exists the connection proceeds with a floating negotiation.
338 struct sockaddr_in sa_in;
339 struct sockaddr_in6 sa_in6;
343 dmsg_crypto_negotiate(dmsg_iocom_t *iocom)
346 socklen_t salen = sizeof(sa);
349 dmsg_handshake_t handtx;
350 dmsg_handshake_t handrx;
351 char buf1[sizeof(handtx)];
352 char buf2[sizeof(handtx)];
357 RSA *keys[3] = { NULL, NULL, NULL };
366 * Get the peer IP address for the connection as a string.
368 if (getpeername(iocom->sock_fd, &sa.sa, &salen) < 0) {
369 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOPEER;
370 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
372 fprintf(stderr, "accept: getpeername() failed\n");
375 if (getnameinfo(&sa.sa, salen, peername, sizeof(peername),
376 NULL, 0, NI_NUMERICHOST) < 0) {
377 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOPEER;
378 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
380 fprintf(stderr, "accept: cannot decode sockaddr\n");
384 if (realhostname_sa(realname, sizeof(realname),
385 &sa.sa, salen) == HOSTNAME_FOUND) {
386 fprintf(stderr, "accept from %s (%s)\n",
389 fprintf(stderr, "accept from %s\n", peername);
394 * Find the remote host's public key
396 * If the link is not to be encrypted (<ip>.none located) we shortcut
397 * the handshake entirely. No buffers are exchanged.
399 asprintf(&path, "%s/%s.pub", DMSG_PATH_REMOTE, peername);
400 if ((fp = fopen(path, "r")) == NULL) {
402 asprintf(&path, "%s/%s.none",
403 DMSG_PATH_REMOTE, peername);
404 if (stat(path, &st) < 0) {
405 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NORKEY;
406 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
408 fprintf(stderr, "auth failure: unknown host\n");
412 fprintf(stderr, "auth succeeded, unencrypted link\n");
416 keys[0] = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
418 if (keys[0] == NULL) {
419 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
420 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
423 "auth failure: bad key format\n");
429 * Get our public and private keys
432 asprintf(&path, DMSG_DEFAULT_DIR "/rsa.pub");
433 if ((fp = fopen(path, "r")) == NULL) {
434 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOLKEY;
435 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
438 keys[1] = PEM_read_RSA_PUBKEY(fp, NULL, NULL, NULL);
440 if (keys[1] == NULL) {
441 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
442 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
444 fprintf(stderr, "auth failure: bad host key format\n");
449 asprintf(&path, DMSG_DEFAULT_DIR "/rsa.prv");
450 if ((fp = fopen(path, "r")) == NULL) {
451 iocom->ioq_rx.error = DMSG_IOQ_ERROR_NOLKEY;
452 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
454 fprintf(stderr, "auth failure: bad host key format\n");
457 keys[2] = PEM_read_RSAPrivateKey(fp, NULL, NULL, NULL);
459 if (keys[2] == NULL) {
460 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
461 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
463 fprintf(stderr, "auth failure: bad host key format\n");
470 * public key encrypt/decrypt block size.
473 blksize = (size_t)RSA_size(keys[0]);
474 if (blksize != (size_t)RSA_size(keys[1]) ||
475 blksize != (size_t)RSA_size(keys[2]) ||
476 sizeof(handtx) % blksize != 0) {
477 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYFMT;
478 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
480 fprintf(stderr, "auth failure: "
481 "key size mismatch\n");
485 blksize = sizeof(handtx);
487 blkmask = blksize - 1;
489 bzero(&handrx, sizeof(handrx));
490 bzero(&handtx, sizeof(handtx));
493 * Fill all unused fields (particular all junk fields) with random
494 * data, and also set the session key.
496 fd = open("/dev/urandom", O_RDONLY);
498 fstat(fd, &st) < 0 || /* something wrong */
499 S_ISREG(st.st_mode) || /* supposed to be a RNG dev! */
500 read(fd, &handtx, sizeof(handtx)) != sizeof(handtx)) {
504 iocom->ioq_rx.error = DMSG_IOQ_ERROR_BADURANDOM;
505 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
507 fprintf(stderr, "auth failure: bad rng\n");
510 if (bcmp(&handrx, &handtx, sizeof(handtx)) == 0)
511 goto urandfail; /* read all zeros */
513 /* ERR_load_crypto_strings(); openssl debugging */
516 * Handshake with the remote.
518 * Encrypt with my private and remote's public
519 * Decrypt with my private and remote's public
521 * When encrypting we have to make sure our buffer fits within the
522 * modulus, which typically requires bit 7 o the first byte to be
523 * zero. To be safe make sure that bit 7 and bit 6 is zero.
525 snprintf(handtx.quickmsg, sizeof(handtx.quickmsg), "Testing 1 2 3");
526 handtx.magic = DMSG_HDR_MAGIC;
529 assert(sizeof(handtx.verf) * 4 == sizeof(handtx.sess));
530 bzero(handtx.verf, sizeof(handtx.verf));
532 handtx.pad1[0] &= 0x3f; /* message must fit within modulus */
533 handtx.pad2[0] &= 0x3f; /* message must fit within modulus */
535 for (i = 0; i < sizeof(handtx.sess); ++i)
536 handtx.verf[i / 4] ^= handtx.sess[i];
539 * Write handshake buffer to remote
541 for (i = 0; i < sizeof(handtx); i += blksize) {
542 ptr = (char *)&handtx + i;
545 * Since we are double-encrypting we have to make
546 * sure that the result of the first stage does
547 * not blow out the modulus for the second stage.
549 * The pointer is pointing to the pad*[] area so
550 * we can mess with that until the first stage
555 if (RSA_private_encrypt(blksize, ptr, buf1,
556 keys[2], RSA_NO_PADDING) < 0) {
557 iocom->ioq_rx.error =
558 DMSG_IOQ_ERROR_KEYXCHGFAIL;
560 } while (buf1[0] & 0xC0);
562 if (RSA_public_encrypt(blksize, buf1, buf2,
563 keys[0], RSA_NO_PADDING) < 0) {
564 iocom->ioq_rx.error =
565 DMSG_IOQ_ERROR_KEYXCHGFAIL;
568 if (write(iocom->sock_fd, buf2, blksize) != (ssize_t)blksize) {
569 fprintf(stderr, "WRITE ERROR\n");
572 if (iocom->ioq_rx.error) {
573 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
575 fprintf(stderr, "auth failure: key exchange failure "
576 "during encryption\n");
581 * Read handshake buffer from remote
584 while (i < sizeof(handrx)) {
585 ptr = (char *)&handrx + i;
586 n = read(iocom->sock_fd, ptr, blksize - (i & blkmask));
589 ptr -= (i & blkmask);
591 if (keys[0] && (i & blkmask) == 0) {
592 if (RSA_private_decrypt(blksize, ptr, buf1,
593 keys[2], RSA_NO_PADDING) < 0)
594 iocom->ioq_rx.error =
595 DMSG_IOQ_ERROR_KEYXCHGFAIL;
596 if (RSA_public_decrypt(blksize, buf1, ptr,
597 keys[0], RSA_NO_PADDING) < 0)
598 iocom->ioq_rx.error =
599 DMSG_IOQ_ERROR_KEYXCHGFAIL;
602 if (iocom->ioq_rx.error) {
603 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
605 fprintf(stderr, "auth failure: key exchange failure "
606 "during decryption\n");
611 * Validate the received data. Try to make this a constant-time
614 if (i != sizeof(handrx)) {
616 iocom->ioq_rx.error = DMSG_IOQ_ERROR_KEYXCHGFAIL;
617 atomic_set_int(&iocom->flags, DMSG_IOCOMF_EOF);
619 fprintf(stderr, "auth failure: key exchange failure\n");
623 if (handrx.magic == DMSG_HDR_MAGIC_REV) {
624 handrx.version = bswap16(handrx.version);
625 handrx.flags = bswap32(handrx.flags);
627 for (i = 0; i < sizeof(handrx.sess); ++i)
628 handrx.verf[i / 4] ^= handrx.sess[i];
630 for (i = 0; i < sizeof(handrx.verf); ++i)
632 if (handrx.version != 1)
638 * Use separate session keys and session fixed IVs for receive and
641 error = crypto_algos[DMSG_CRYPTO_ALGO].init(&iocom->ioq_rx, handrx.sess,
642 crypto_algos[DMSG_CRYPTO_ALGO].keylen,
643 handrx.sess + crypto_algos[DMSG_CRYPTO_ALGO].keylen,
644 sizeof(handrx.sess) - crypto_algos[DMSG_CRYPTO_ALGO].keylen,
649 error = crypto_algos[DMSG_CRYPTO_ALGO].init(&iocom->ioq_tx, handtx.sess,
650 crypto_algos[DMSG_CRYPTO_ALGO].keylen,
651 handtx.sess + crypto_algos[DMSG_CRYPTO_ALGO].keylen,
652 sizeof(handtx.sess) - crypto_algos[DMSG_CRYPTO_ALGO].keylen,
657 atomic_set_int(&iocom->flags, DMSG_IOCOMF_CRYPTED);
660 fprintf(stderr, "auth success: %s\n", handrx.quickmsg);
673 * Decrypt pending data in the ioq's fifo. The data is decrypted in-place.
676 dmsg_crypto_decrypt(dmsg_iocom_t *iocom __unused, dmsg_ioq_t *ioq)
680 __unused int error; /* XXX */
684 * fifo_beg to fifo_cdx is data already decrypted.
685 * fifo_cdn to fifo_end is data not yet decrypted.
687 p_len = ioq->fifo_end - ioq->fifo_cdn; /* data not yet decrypted */
692 while (p_len >= crypto_algos[DMSG_CRYPTO_ALGO].taglen +
693 DMSG_CRYPTO_CHUNK_SIZE) {
694 bcopy(ioq->buf + ioq->fifo_cdn, buf,
695 crypto_algos[DMSG_CRYPTO_ALGO].taglen +
696 DMSG_CRYPTO_CHUNK_SIZE);
697 error = crypto_algos[DMSG_CRYPTO_ALGO].dec_chunk(
699 ioq->buf + ioq->fifo_cdx,
700 DMSG_CRYPTO_CHUNK_SIZE,
703 printf("dec: p_len: %d, used: %d, fifo_cdn: %ju, fifo_cdx: %ju\n",
704 p_len, used, ioq->fifo_cdn, ioq->fifo_cdx);
707 ioq->fifo_cdn += used;
708 ioq->fifo_cdx += DMSG_CRYPTO_CHUNK_SIZE;
710 printf("dec: p_len: %d, used: %d, fifo_cdn: %ju, fifo_cdx: %ju\n",
711 p_len, used, ioq->fifo_cdn, ioq->fifo_cdx);
717 * *nactp is set to the number of ORIGINAL bytes consumed by the encrypter.
718 * The FIFO may contain more data.
721 dmsg_crypto_encrypt(dmsg_iocom_t *iocom __unused, dmsg_ioq_t *ioq,
722 struct iovec *iov, int n, size_t *nactp)
724 int p_len, used, ct_used;
726 __unused int error; /* XXX */
729 nmax = sizeof(ioq->buf) - ioq->fifo_end; /* max new bytes */
732 for (i = 0; i < n && nmax; ++i) {
734 p_len = iov[i].iov_len;
735 assert((p_len & DMSG_ALIGNMASK) == 0);
737 while (p_len >= DMSG_CRYPTO_CHUNK_SIZE &&
738 nmax >= DMSG_CRYPTO_CHUNK_SIZE +
739 (size_t)crypto_algos[DMSG_CRYPTO_ALGO].taglen) {
740 error = crypto_algos[DMSG_CRYPTO_ALGO].enc_chunk(
742 ioq->buf + ioq->fifo_cdx,
743 (char *)iov[i].iov_base + used,
744 DMSG_CRYPTO_CHUNK_SIZE, &ct_used);
746 printf("nactp: %ju, p_len: %d, ct_used: %d, used: %d, nmax: %ju\n",
747 *nactp, p_len, ct_used, used, nmax);
750 *nactp += (size_t)DMSG_CRYPTO_CHUNK_SIZE; /* plaintext count */
751 used += DMSG_CRYPTO_CHUNK_SIZE;
752 p_len -= DMSG_CRYPTO_CHUNK_SIZE;
755 * NOTE: crypted count will eventually differ from
756 * nmax, but for now we have not yet introduced
759 ioq->fifo_cdx += (size_t)ct_used;
760 ioq->fifo_cdn += (size_t)ct_used;
761 ioq->fifo_end += (size_t)ct_used;
762 nmax -= (size_t)ct_used;
764 printf("nactp: %ju, p_len: %d, ct_used: %d, used: %d, nmax: %ju\n",
765 *nactp, p_len, ct_used, used, nmax);
769 iov[0].iov_base = ioq->buf + ioq->fifo_beg;
770 iov[0].iov_len = ioq->fifo_cdx - ioq->fifo_beg;