opencrypto - Add AES GCM/GMAC support
[dragonfly.git] / sys / opencrypto / cryptosoft.c
CommitLineData
42ee1e6b 1/*-
984263bc 2 * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
42ee1e6b 3 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
984263bc
MD
4 *
5 * This code was written by Angelos D. Keromytis in Athens, Greece, in
6 * February 2000. Network Security Technologies Inc. (NSTI) kindly
7 * supported the development of this code.
8 *
9 * Copyright (c) 2000, 2001 Angelos D. Keromytis
10 *
a0419b33
MD
11 * SMP modifications by Matthew Dillon for the DragonFlyBSD Project
12 *
984263bc
MD
13 * Permission to use, copy, and modify this software with or without fee
14 * is hereby granted, provided that this entire notice is included in
15 * all source code copies of any software which is or includes a copy or
16 * modification of this software.
17 *
18 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
19 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
20 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
21 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
22 * PURPOSE.
a0419b33
MD
23 *
24 * $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $
25 * $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $
984263bc
MD
26 */
27
28#include <sys/param.h>
29#include <sys/systm.h>
30#include <sys/malloc.h>
31#include <sys/mbuf.h>
42ee1e6b 32#include <sys/module.h>
984263bc
MD
33#include <sys/sysctl.h>
34#include <sys/errno.h>
aface142 35#include <sys/endian.h>
984263bc
MD
36#include <sys/random.h>
37#include <sys/kernel.h>
38#include <sys/uio.h>
a0419b33 39#include <sys/spinlock2.h>
984263bc
MD
40
41#include <crypto/blowfish/blowfish.h>
984263bc
MD
42#include <crypto/sha1.h>
43#include <opencrypto/rmd160.h>
42ee1e6b 44#include <opencrypto/cast.h>
984263bc
MD
45#include <opencrypto/skipjack.h>
46#include <sys/md5.h>
47
48#include <opencrypto/cryptodev.h>
49#include <opencrypto/cryptosoft.h>
50#include <opencrypto/xform.h>
51
42ee1e6b
SW
52#include <sys/kobj.h>
53#include <sys/bus.h>
54#include "cryptodev_if.h"
984263bc 55
42ee1e6b
SW
56static int32_t swcr_id;
57static struct swcr_data **swcr_sessions = NULL;
58static u_int32_t swcr_sesnum;
a0419b33
MD
59static u_int32_t swcr_minsesnum = 1;
60
61static struct spinlock swcr_spin = SPINLOCK_INITIALIZER(swcr_spin);
984263bc 62
42ee1e6b
SW
63u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
64u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
984263bc
MD
65
66static int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
42ee1e6b 67static int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
aface142 68static int swcr_combined(struct cryptop *);
984263bc 69static int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
42ee1e6b 70static int swcr_freesession(device_t dev, u_int64_t tid);
a0419b33 71static int swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid);
984263bc
MD
72
73/*
74 * Apply a symmetric encryption/decryption algorithm.
75 */
76static int
77swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
42ee1e6b 78 int flags)
984263bc
MD
79{
80 unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
d6715a19 81 unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN];
a0419b33
MD
82 u_int8_t *kschedule;
83 u_int8_t *okschedule;
984263bc 84 struct enc_xform *exf;
da41e4e5 85 int i, k, j, blks, ivlen;
a0419b33
MD
86 int error;
87 int explicit_kschedule;
984263bc
MD
88
89 exf = sw->sw_exf;
90 blks = exf->blocksize;
da41e4e5 91 ivlen = exf->ivsize;
984263bc
MD
92
93 /* Check for non-padded data */
94 if (crd->crd_len % blks)
95 return EINVAL;
96
97 /* Initialize the IV */
98 if (crd->crd_flags & CRD_F_ENCRYPT) {
99 /* IV explicitly provided ? */
100 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
da41e4e5 101 bcopy(crd->crd_iv, iv, ivlen);
42ee1e6b 102 else
da41e4e5 103 karc4rand(iv, ivlen);
984263bc
MD
104
105 /* Do we need to write the IV */
42ee1e6b 106 if (!(crd->crd_flags & CRD_F_IV_PRESENT))
da41e4e5 107 crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
984263bc
MD
108
109 } else { /* Decryption */
110 /* IV explicitly provided ? */
111 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
da41e4e5 112 bcopy(crd->crd_iv, iv, ivlen);
984263bc
MD
113 else {
114 /* Get IV off buf */
da41e4e5 115 crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
984263bc
MD
116 }
117 }
118
da41e4e5
AH
119 ivp = iv;
120
121 /*
a0419b33
MD
122 * The semantics are seriously broken because the session key
123 * storage was never designed for concurrent ops.
124 */
dd6a2a13 125 if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
a0419b33
MD
126 kschedule = NULL;
127 explicit_kschedule = 1;
128 error = exf->setkey(&kschedule,
129 crd->crd_key, crd->crd_klen / 8);
dd6a2a13 130 if (error)
a0419b33
MD
131 goto done;
132 } else {
287a8577 133 spin_lock(&swcr_spin);
a0419b33
MD
134 kschedule = sw->sw_kschedule;
135 ++sw->sw_kschedule_refs;
287a8577 136 spin_unlock(&swcr_spin);
a0419b33 137 explicit_kschedule = 0;
dd6a2a13 138 }
a0419b33 139
bc1ea2f2
AH
140 /*
141 * xforms that provide a reinit method perform all IV
142 * handling themselves.
143 */
144 if (exf->reinit)
145 exf->reinit(kschedule, iv);
146
42ee1e6b 147 if (flags & CRYPTO_F_IMBUF) {
984263bc
MD
148 struct mbuf *m = (struct mbuf *) buf;
149
150 /* Find beginning of data */
151 m = m_getptr(m, crd->crd_skip, &k);
a0419b33
MD
152 if (m == NULL) {
153 error = EINVAL;
154 goto done;
155 }
984263bc
MD
156
157 i = crd->crd_len;
158
159 while (i > 0) {
160 /*
161 * If there's insufficient data at the end of
162 * an mbuf, we have to do some copying.
163 */
164 if (m->m_len < k + blks && m->m_len != k) {
165 m_copydata(m, k, blks, blk);
166
167 /* Actual encryption/decryption */
da41e4e5
AH
168 if (exf->reinit) {
169 if (crd->crd_flags & CRD_F_ENCRYPT) {
170 exf->encrypt(kschedule,
400158c1 171 blk, iv);
da41e4e5
AH
172 } else {
173 exf->decrypt(kschedule,
400158c1 174 blk, iv);
da41e4e5
AH
175 }
176 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
984263bc
MD
177 /* XOR with previous block */
178 for (j = 0; j < blks; j++)
179 blk[j] ^= ivp[j];
180
400158c1 181 exf->encrypt(kschedule, blk, iv);
984263bc
MD
182
183 /*
184 * Keep encrypted block for XOR'ing
185 * with next block
186 */
187 bcopy(blk, iv, blks);
188 ivp = iv;
189 } else { /* decrypt */
190 /*
191 * Keep encrypted block for XOR'ing
192 * with next block
193 */
d6715a19
AH
194 nivp = (ivp == iv) ? iv2 : iv;
195 bcopy(blk, nivp, blks);
984263bc 196
400158c1 197 exf->decrypt(kschedule, blk, iv);
984263bc
MD
198
199 /* XOR with previous block */
200 for (j = 0; j < blks; j++)
201 blk[j] ^= ivp[j];
202
d6715a19 203 ivp = nivp;
984263bc
MD
204 }
205
206 /* Copy back decrypted block */
207 m_copyback(m, k, blks, blk);
208
209 /* Advance pointer */
210 m = m_getptr(m, k + blks, &k);
a0419b33
MD
211 if (m == NULL) {
212 error = EINVAL;
213 goto done;
214 }
984263bc
MD
215
216 i -= blks;
217
218 /* Could be done... */
219 if (i == 0)
220 break;
221 }
222
223 /* Skip possibly empty mbufs */
224 if (k == m->m_len) {
225 for (m = m->m_next; m && m->m_len == 0;
226 m = m->m_next)
227 ;
228 k = 0;
229 }
230
231 /* Sanity check */
a0419b33
MD
232 if (m == NULL) {
233 error = EINVAL;
234 goto done;
235 }
984263bc
MD
236
237 /*
238 * Warning: idat may point to garbage here, but
239 * we only use it in the while() loop, only if
240 * there are indeed enough data.
241 */
242 idat = mtod(m, unsigned char *) + k;
243
244 while (m->m_len >= k + blks && i > 0) {
da41e4e5
AH
245 if (exf->reinit) {
246 if (crd->crd_flags & CRD_F_ENCRYPT) {
247 exf->encrypt(kschedule,
400158c1 248 idat, iv);
da41e4e5
AH
249 } else {
250 exf->decrypt(kschedule,
400158c1 251 idat, iv);
da41e4e5
AH
252 }
253 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
984263bc
MD
254 /* XOR with previous block/IV */
255 for (j = 0; j < blks; j++)
256 idat[j] ^= ivp[j];
257
400158c1 258 exf->encrypt(kschedule, idat, iv);
984263bc
MD
259 ivp = idat;
260 } else { /* decrypt */
261 /*
262 * Keep encrypted block to be used
263 * in next block's processing.
264 */
d6715a19
AH
265 nivp = (ivp == iv) ? iv2 : iv;
266 bcopy(idat, nivp, blks);
984263bc 267
400158c1 268 exf->decrypt(kschedule, idat, iv);
984263bc
MD
269
270 /* XOR with previous block/IV */
271 for (j = 0; j < blks; j++)
272 idat[j] ^= ivp[j];
273
d6715a19 274 ivp = nivp;
984263bc
MD
275 }
276
277 idat += blks;
278 k += blks;
279 i -= blks;
280 }
281 }
a0419b33 282 error = 0; /* Done with mbuf encryption/decryption */
42ee1e6b 283 } else if (flags & CRYPTO_F_IOV) {
984263bc
MD
284 struct uio *uio = (struct uio *) buf;
285 struct iovec *iov;
286
287 /* Find beginning of data */
288 iov = cuio_getptr(uio, crd->crd_skip, &k);
a0419b33
MD
289 if (iov == NULL) {
290 error = EINVAL;
291 goto done;
292 }
984263bc
MD
293
294 i = crd->crd_len;
295
296 while (i > 0) {
297 /*
298 * If there's insufficient data at the end of
299 * an iovec, we have to do some copying.
300 */
301 if (iov->iov_len < k + blks && iov->iov_len != k) {
302 cuio_copydata(uio, k, blks, blk);
303
304 /* Actual encryption/decryption */
da41e4e5
AH
305 if (exf->reinit) {
306 if (crd->crd_flags & CRD_F_ENCRYPT) {
307 exf->encrypt(kschedule,
400158c1 308 blk, iv);
da41e4e5
AH
309 } else {
310 exf->decrypt(kschedule,
400158c1 311 blk, iv);
da41e4e5
AH
312 }
313 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
984263bc
MD
314 /* XOR with previous block */
315 for (j = 0; j < blks; j++)
316 blk[j] ^= ivp[j];
317
400158c1 318 exf->encrypt(kschedule, blk, iv);
984263bc
MD
319
320 /*
321 * Keep encrypted block for XOR'ing
322 * with next block
323 */
324 bcopy(blk, iv, blks);
325 ivp = iv;
326 } else { /* decrypt */
327 /*
328 * Keep encrypted block for XOR'ing
329 * with next block
330 */
d6715a19
AH
331 nivp = (ivp == iv) ? iv2 : iv;
332 bcopy(blk, nivp, blks);
984263bc 333
400158c1 334 exf->decrypt(kschedule, blk, iv);
984263bc
MD
335
336 /* XOR with previous block */
337 for (j = 0; j < blks; j++)
338 blk[j] ^= ivp[j];
339
d6715a19 340 ivp = nivp;
984263bc
MD
341 }
342
343 /* Copy back decrypted block */
344 cuio_copyback(uio, k, blks, blk);
345
346 /* Advance pointer */
347 iov = cuio_getptr(uio, k + blks, &k);
a0419b33
MD
348 if (iov == NULL) {
349 error = EINVAL;
350 goto done;
351 }
984263bc
MD
352
353 i -= blks;
354
355 /* Could be done... */
356 if (i == 0)
357 break;
358 }
359
360 /*
361 * Warning: idat may point to garbage here, but
362 * we only use it in the while() loop, only if
363 * there are indeed enough data.
364 */
365 idat = (char *)iov->iov_base + k;
366
367 while (iov->iov_len >= k + blks && i > 0) {
da41e4e5
AH
368 if (exf->reinit) {
369 if (crd->crd_flags & CRD_F_ENCRYPT) {
370 exf->encrypt(kschedule,
400158c1 371 idat, iv);
da41e4e5
AH
372 } else {
373 exf->decrypt(kschedule,
400158c1 374 idat, iv);
da41e4e5
AH
375 }
376 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
984263bc
MD
377 /* XOR with previous block/IV */
378 for (j = 0; j < blks; j++)
379 idat[j] ^= ivp[j];
380
400158c1 381 exf->encrypt(kschedule, idat, iv);
984263bc
MD
382 ivp = idat;
383 } else { /* decrypt */
384 /*
385 * Keep encrypted block to be used
386 * in next block's processing.
387 */
d6715a19
AH
388 nivp = (ivp == iv) ? iv2 : iv;
389 bcopy(idat, nivp, blks);
984263bc 390
400158c1 391 exf->decrypt(kschedule, idat, iv);
984263bc
MD
392
393 /* XOR with previous block/IV */
394 for (j = 0; j < blks; j++)
395 idat[j] ^= ivp[j];
396
d6715a19 397 ivp = nivp;
984263bc
MD
398 }
399
400 idat += blks;
401 k += blks;
402 i -= blks;
403 }
42ee1e6b
SW
404 if (k == iov->iov_len) {
405 iov++;
406 k = 0;
407 }
984263bc 408 }
a0419b33
MD
409 error = 0; /* Done with iovec encryption/decryption */
410 } else {
411 /*
412 * contiguous buffer
413 */
bc1ea2f2
AH
414 if (exf->reinit) {
415 for(i = crd->crd_skip;
416 i < crd->crd_skip + crd->crd_len; i += blks) {
417 if (crd->crd_flags & CRD_F_ENCRYPT) {
400158c1 418 exf->encrypt(kschedule, buf + i, iv);
bc1ea2f2 419 } else {
400158c1 420 exf->decrypt(kschedule, buf + i, iv);
bc1ea2f2
AH
421 }
422 }
423 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
42ee1e6b
SW
424 for (i = crd->crd_skip;
425 i < crd->crd_skip + crd->crd_len; i += blks) {
426 /* XOR with the IV/previous block, as appropriate. */
427 if (i == crd->crd_skip)
428 for (k = 0; k < blks; k++)
429 buf[i + k] ^= ivp[k];
430 else
431 for (k = 0; k < blks; k++)
432 buf[i + k] ^= buf[i + k - blks];
400158c1 433 exf->encrypt(kschedule, buf + i, iv);
42ee1e6b
SW
434 }
435 } else { /* Decrypt */
436 /*
a0419b33
MD
437 * Start at the end, so we don't need to keep the
438 * encrypted block as the IV for the next block.
42ee1e6b
SW
439 */
440 for (i = crd->crd_skip + crd->crd_len - blks;
441 i >= crd->crd_skip; i -= blks) {
400158c1 442 exf->decrypt(kschedule, buf + i, iv);
42ee1e6b
SW
443
444 /* XOR with the IV/previous block, as appropriate */
445 if (i == crd->crd_skip)
446 for (k = 0; k < blks; k++)
447 buf[i + k] ^= ivp[k];
448 else
449 for (k = 0; k < blks; k++)
450 buf[i + k] ^= buf[i + k - blks];
451 }
452 }
a0419b33 453 error = 0; /* Done w/contiguous buffer encrypt/decrypt */
984263bc 454 }
a0419b33
MD
455done:
456 /*
457 * Cleanup - explicitly replace the session key if requested
458 * (horrible semantics for concurrent operation)
459 */
460 if (explicit_kschedule) {
287a8577 461 spin_lock(&swcr_spin);
a0419b33
MD
462 if (sw->sw_kschedule && sw->sw_kschedule_refs == 0) {
463 okschedule = sw->sw_kschedule;
464 sw->sw_kschedule = kschedule;
465 } else {
466 okschedule = NULL;
467 }
287a8577 468 spin_unlock(&swcr_spin);
a0419b33
MD
469 if (okschedule)
470 exf->zerokey(&okschedule);
471 } else {
287a8577 472 spin_lock(&swcr_spin);
a0419b33 473 --sw->sw_kschedule_refs;
287a8577 474 spin_unlock(&swcr_spin);
a0419b33
MD
475 }
476 return error;
984263bc
MD
477}
478
42ee1e6b
SW
479static void
480swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
481 int klen)
482{
483 int k;
484
485 klen /= 8;
486
487 switch (axf->type) {
488 case CRYPTO_MD5_HMAC:
489 case CRYPTO_SHA1_HMAC:
490 case CRYPTO_SHA2_256_HMAC:
491 case CRYPTO_SHA2_384_HMAC:
492 case CRYPTO_SHA2_512_HMAC:
493 case CRYPTO_NULL_HMAC:
494 case CRYPTO_RIPEMD160_HMAC:
495 for (k = 0; k < klen; k++)
496 key[k] ^= HMAC_IPAD_VAL;
497
498 axf->Init(sw->sw_ictx);
499 axf->Update(sw->sw_ictx, key, klen);
500 axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
501
502 for (k = 0; k < klen; k++)
503 key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
504
505 axf->Init(sw->sw_octx);
506 axf->Update(sw->sw_octx, key, klen);
507 axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
508
509 for (k = 0; k < klen; k++)
510 key[k] ^= HMAC_OPAD_VAL;
511 break;
512 case CRYPTO_MD5_KPDK:
513 case CRYPTO_SHA1_KPDK:
514 {
515 /* We need a buffer that can hold an md5 and a sha1 result. */
516 u_char buf[SHA1_RESULTLEN];
517
518 sw->sw_klen = klen;
519 bcopy(key, sw->sw_octx, klen);
520 axf->Init(sw->sw_ictx);
521 axf->Update(sw->sw_ictx, key, klen);
522 axf->Final(buf, sw->sw_ictx);
523 break;
524 }
525 default:
54734da1 526 kprintf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
42ee1e6b
SW
527 "doesn't use keys.\n", __func__, axf->type);
528 }
529}
530
984263bc
MD
531/*
532 * Compute keyed-hash authenticator.
533 */
534static int
42ee1e6b
SW
535swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
536 int flags)
984263bc 537{
42ee1e6b 538 unsigned char aalg[HASH_MAX_LEN];
984263bc
MD
539 struct auth_hash *axf;
540 union authctx ctx;
541 int err;
542
543 if (sw->sw_ictx == 0)
544 return EINVAL;
545
546 axf = sw->sw_axf;
547
42ee1e6b
SW
548 if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
549 swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
550
984263bc
MD
551 bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
552
42ee1e6b
SW
553 err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
554 (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
555 if (err)
556 return err;
984263bc
MD
557
558 switch (sw->sw_alg) {
559 case CRYPTO_MD5_HMAC:
560 case CRYPTO_SHA1_HMAC:
42ee1e6b
SW
561 case CRYPTO_SHA2_256_HMAC:
562 case CRYPTO_SHA2_384_HMAC:
563 case CRYPTO_SHA2_512_HMAC:
984263bc
MD
564 case CRYPTO_RIPEMD160_HMAC:
565 if (sw->sw_octx == NULL)
566 return EINVAL;
567
568 axf->Final(aalg, &ctx);
569 bcopy(sw->sw_octx, &ctx, axf->ctxsize);
570 axf->Update(&ctx, aalg, axf->hashsize);
571 axf->Final(aalg, &ctx);
572 break;
573
574 case CRYPTO_MD5_KPDK:
575 case CRYPTO_SHA1_KPDK:
576 if (sw->sw_octx == NULL)
577 return EINVAL;
578
579 axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
580 axf->Final(aalg, &ctx);
581 break;
582
583 case CRYPTO_NULL_HMAC:
584 axf->Final(aalg, &ctx);
585 break;
586 }
587
588 /* Inject the authentication data */
42ee1e6b
SW
589 crypto_copyback(flags, buf, crd->crd_inject,
590 sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
984263bc
MD
591 return 0;
592}
593
594/*
aface142
AH
595 * Apply a combined encryption-authentication transformation
596 */
597static int
598swcr_combined(struct cryptop *crp)
599{
600 uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
601 u_char *blk = (u_char *)blkbuf;
602 u_char aalg[HASH_MAX_LEN];
603 u_char iv[EALG_MAX_BLOCK_LEN];
604 uint8_t *kschedule;
605 union authctx ctx;
606 struct cryptodesc *crd, *crda = NULL, *crde = NULL;
607 struct swcr_data *sw, *swa, *swe;
608 struct auth_hash *axf = NULL;
609 struct enc_xform *exf = NULL;
610 struct mbuf *m = NULL;
611 struct uio *uio = NULL;
612 caddr_t buf = (caddr_t)crp->crp_buf;
613 uint32_t *blkp;
614 int i, blksz, ivlen, outtype, len;
615
616 blksz = 0;
617 ivlen = 0;
618
619 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
620 for (sw = swcr_sessions[crp->crp_sid & 0xffffffff];
621 sw && sw->sw_alg != crd->crd_alg;
622 sw = sw->sw_next)
623 ;
624 if (sw == NULL)
625 return (EINVAL);
626
627 switch (sw->sw_alg) {
628 case CRYPTO_AES_GCM_16:
629 case CRYPTO_AES_GMAC:
630 swe = sw;
631 crde = crd;
632 exf = swe->sw_exf;
633 ivlen = exf->ivsize;
634 break;
635 case CRYPTO_AES_128_GMAC:
636 case CRYPTO_AES_192_GMAC:
637 case CRYPTO_AES_256_GMAC:
638 swa = sw;
639 crda = crd;
640 axf = swa->sw_axf;
641 if (swa->sw_ictx == 0)
642 return (EINVAL);
643 bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
644 blksz = axf->blocksize;
645 break;
646 default:
647 return (EINVAL);
648 }
649 }
650 if (crde == NULL || crda == NULL)
651 return (EINVAL);
652
653 if (crp->crp_flags & CRYPTO_F_IMBUF) {
654 outtype = CRYPTO_BUF_MBUF;
655 m = (struct mbuf *)buf;
656 } else {
657 outtype = CRYPTO_BUF_IOV;
658 uio = (struct uio *)buf;
659 }
660
661 /* Initialize the IV */
662 if (crde->crd_flags & CRD_F_ENCRYPT) {
663 /* IV explicitly provided ? */
664 if (crde->crd_flags & CRD_F_IV_EXPLICIT)
665 bcopy(crde->crd_iv, iv, ivlen);
666 else
667 karc4rand(iv, ivlen);
668
669 /* Do we need to write the IV */
670 if (!(crde->crd_flags & CRD_F_IV_PRESENT))
671 crypto_copyback(crde->crd_flags, buf, crde->crd_inject,
672 ivlen, iv);
673
674 } else { /* Decryption */
675 /* IV explicitly provided ? */
676 if (crde->crd_flags & CRD_F_IV_EXPLICIT)
677 bcopy(crde->crd_iv, iv, ivlen);
678 else
679 /* Get IV off buf */
680 crypto_copydata(crde->crd_flags, buf, crde->crd_inject,
681 ivlen, iv);
682 }
683
684 /* Supply MAC with IV */
685 if (axf->Reinit)
686 axf->Reinit(&ctx, iv, ivlen);
687
688 /* Supply MAC with AAD */
689 for (i = 0; i < crda->crd_len; i += blksz) {
690 len = MIN(crda->crd_len - i, blksz);
691 crypto_copydata(crde->crd_flags, buf, crda->crd_skip + i, len,
692 blk);
693 axf->Update(&ctx, blk, len);
694 }
695
696 spin_lock(&swcr_spin);
697 kschedule = sw->sw_kschedule;
698 ++sw->sw_kschedule_refs;
699 spin_unlock(&swcr_spin);
700
701 if (exf->reinit)
702 exf->reinit(kschedule, iv);
703
704 /* Do encryption/decryption with MAC */
705 for (i = 0; i < crde->crd_len; i += blksz) {
706 len = MIN(crde->crd_len - i, blksz);
707 if (len < blksz)
708 bzero(blk, blksz);
709 crypto_copydata(crde->crd_flags, buf, crde->crd_skip + i, len,
710 blk);
711 if (crde->crd_flags & CRD_F_ENCRYPT) {
712 exf->encrypt(kschedule, blk, iv);
713 axf->Update(&ctx, blk, len);
714 } else {
715 axf->Update(&ctx, blk, len);
716 exf->decrypt(kschedule, blk, iv);
717 }
718 crypto_copyback(crde->crd_flags, buf, crde->crd_skip + i, len,
719 blk);
720 }
721
722 /* Do any required special finalization */
723 switch (crda->crd_alg) {
724 case CRYPTO_AES_128_GMAC:
725 case CRYPTO_AES_192_GMAC:
726 case CRYPTO_AES_256_GMAC:
727 /* length block */
728 bzero(blk, blksz);
729 blkp = (uint32_t *)blk + 1;
730 *blkp = htobe32(crda->crd_len * 8);
731 blkp = (uint32_t *)blk + 3;
732 *blkp = htobe32(crde->crd_len * 8);
733 axf->Update(&ctx, blk, blksz);
734 break;
735 }
736
737 /* Finalize MAC */
738 axf->Final(aalg, &ctx);
739
740 /* Inject the authentication data */
741 crypto_copyback(crda->crd_flags, crp->crp_buf, crda->crd_inject,
742 axf->blocksize, aalg);
743
744 spin_lock(&swcr_spin);
745 --sw->sw_kschedule_refs;
746 spin_unlock(&swcr_spin);
747
748 return (0);
749}
750
751/*
984263bc
MD
752 * Apply a compression/decompression algorithm
753 */
754static int
755swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
a0419b33 756 caddr_t buf, int flags)
984263bc
MD
757{
758 u_int8_t *data, *out;
759 struct comp_algo *cxf;
760 int adj;
761 u_int32_t result;
762
763 cxf = sw->sw_cxf;
764
a0419b33
MD
765 /*
766 * We must handle the whole buffer of data in one time
984263bc
MD
767 * then if there is not all the data in the mbuf, we must
768 * copy in a buffer.
769 */
a0419b33 770 data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_INTWAIT);
984263bc
MD
771 if (data == NULL)
772 return (EINVAL);
42ee1e6b 773 crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
984263bc
MD
774
775 if (crd->crd_flags & CRD_F_COMP)
776 result = cxf->compress(data, crd->crd_len, &out);
777 else
778 result = cxf->decompress(data, crd->crd_len, &out);
779
42ee1e6b 780 kfree(data, M_CRYPTO_DATA);
984263bc
MD
781 if (result == 0)
782 return EINVAL;
783
784 /* Copy back the (de)compressed data. m_copyback is
785 * extending the mbuf as necessary.
786 */
787 sw->sw_size = result;
788 /* Check the compressed size when doing compression */
789 if (crd->crd_flags & CRD_F_COMP) {
1f26af6e 790 if (result >= crd->crd_len) {
984263bc 791 /* Compression was useless, we lost time */
42ee1e6b 792 kfree(out, M_CRYPTO_DATA);
984263bc
MD
793 return 0;
794 }
795 }
796
42ee1e6b 797 crypto_copyback(flags, buf, crd->crd_skip, result, out);
984263bc
MD
798 if (result < crd->crd_len) {
799 adj = result - crd->crd_len;
42ee1e6b 800 if (flags & CRYPTO_F_IMBUF) {
984263bc
MD
801 adj = result - crd->crd_len;
802 m_adj((struct mbuf *)buf, adj);
42ee1e6b 803 } else if (flags & CRYPTO_F_IOV) {
984263bc
MD
804 struct uio *uio = (struct uio *)buf;
805 int ind;
806
807 adj = crd->crd_len - result;
808 ind = uio->uio_iovcnt - 1;
809
810 while (adj > 0 && ind >= 0) {
811 if (adj < uio->uio_iov[ind].iov_len) {
812 uio->uio_iov[ind].iov_len -= adj;
813 break;
814 }
815
816 adj -= uio->uio_iov[ind].iov_len;
817 uio->uio_iov[ind].iov_len = 0;
818 ind--;
819 uio->uio_iovcnt--;
820 }
821 }
822 }
42ee1e6b 823 kfree(out, M_CRYPTO_DATA);
984263bc
MD
824 return 0;
825}
826
827/*
828 * Generate a new software session.
829 */
830static int
42ee1e6b 831swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
984263bc 832{
a0419b33 833 struct swcr_data *swd_base;
984263bc 834 struct swcr_data **swd;
a0419b33 835 struct swcr_data **oswd;
984263bc
MD
836 struct auth_hash *axf;
837 struct enc_xform *txf;
838 struct comp_algo *cxf;
839 u_int32_t i;
a0419b33 840 u_int32_t n;
42ee1e6b 841 int error;
984263bc
MD
842
843 if (sid == NULL || cri == NULL)
844 return EINVAL;
845
a0419b33
MD
846 swd_base = NULL;
847 swd = &swd_base;
984263bc
MD
848
849 while (cri) {
42ee1e6b 850 *swd = kmalloc(sizeof(struct swcr_data),
a0419b33 851 M_CRYPTO_DATA, M_WAITOK | M_ZERO);
984263bc
MD
852
853 switch (cri->cri_alg) {
854 case CRYPTO_DES_CBC:
855 txf = &enc_xform_des;
856 goto enccommon;
857 case CRYPTO_3DES_CBC:
858 txf = &enc_xform_3des;
859 goto enccommon;
860 case CRYPTO_BLF_CBC:
861 txf = &enc_xform_blf;
862 goto enccommon;
863 case CRYPTO_CAST_CBC:
864 txf = &enc_xform_cast5;
865 goto enccommon;
866 case CRYPTO_SKIPJACK_CBC:
867 txf = &enc_xform_skipjack;
868 goto enccommon;
869 case CRYPTO_RIJNDAEL128_CBC:
870 txf = &enc_xform_rijndael128;
871 goto enccommon;
da41e4e5
AH
872 case CRYPTO_AES_XTS:
873 txf = &enc_xform_aes_xts;
874 goto enccommon;
875 case CRYPTO_AES_CTR:
876 txf = &enc_xform_aes_ctr;
877 goto enccommon;
aface142
AH
878 case CRYPTO_AES_GCM_16:
879 txf = &enc_xform_aes_gcm;
880 goto enccommon;
881 case CRYPTO_AES_GMAC:
882 txf = &enc_xform_aes_gmac;
883 (*swd)->sw_exf = txf;
884 break;
42ee1e6b
SW
885 case CRYPTO_CAMELLIA_CBC:
886 txf = &enc_xform_camellia;
887 goto enccommon;
984263bc
MD
888 case CRYPTO_NULL_CBC:
889 txf = &enc_xform_null;
890 goto enccommon;
891 enccommon:
42ee1e6b
SW
892 if (cri->cri_key != NULL) {
893 error = txf->setkey(&((*swd)->sw_kschedule),
a0419b33
MD
894 cri->cri_key,
895 cri->cri_klen / 8);
42ee1e6b 896 if (error) {
a0419b33 897 swcr_freesession_slot(&swd_base, 0);
42ee1e6b
SW
898 return error;
899 }
984263bc
MD
900 }
901 (*swd)->sw_exf = txf;
902 break;
a0419b33 903
984263bc 904 case CRYPTO_MD5_HMAC:
42ee1e6b 905 axf = &auth_hash_hmac_md5;
984263bc
MD
906 goto authcommon;
907 case CRYPTO_SHA1_HMAC:
42ee1e6b 908 axf = &auth_hash_hmac_sha1;
984263bc 909 goto authcommon;
42ee1e6b
SW
910 case CRYPTO_SHA2_256_HMAC:
911 axf = &auth_hash_hmac_sha2_256;
912 goto authcommon;
913 case CRYPTO_SHA2_384_HMAC:
914 axf = &auth_hash_hmac_sha2_384;
915 goto authcommon;
916 case CRYPTO_SHA2_512_HMAC:
917 axf = &auth_hash_hmac_sha2_512;
984263bc
MD
918 goto authcommon;
919 case CRYPTO_NULL_HMAC:
920 axf = &auth_hash_null;
921 goto authcommon;
922 case CRYPTO_RIPEMD160_HMAC:
42ee1e6b 923 axf = &auth_hash_hmac_ripemd_160;
984263bc 924 authcommon:
efda3bd0 925 (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
a0419b33 926 M_WAITOK);
984263bc 927 if ((*swd)->sw_ictx == NULL) {
a0419b33 928 swcr_freesession_slot(&swd_base, 0);
984263bc
MD
929 return ENOBUFS;
930 }
931
efda3bd0 932 (*swd)->sw_octx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
a0419b33 933 M_WAITOK);
984263bc 934 if ((*swd)->sw_octx == NULL) {
a0419b33 935 swcr_freesession_slot(&swd_base, 0);
984263bc
MD
936 return ENOBUFS;
937 }
938
42ee1e6b
SW
939 if (cri->cri_key != NULL) {
940 swcr_authprepare(axf, *swd, cri->cri_key,
941 cri->cri_klen);
942 }
984263bc 943
42ee1e6b 944 (*swd)->sw_mlen = cri->cri_mlen;
984263bc
MD
945 (*swd)->sw_axf = axf;
946 break;
947
948 case CRYPTO_MD5_KPDK:
949 axf = &auth_hash_key_md5;
950 goto auth2common;
951
952 case CRYPTO_SHA1_KPDK:
953 axf = &auth_hash_key_sha1;
954 auth2common:
efda3bd0 955 (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
a0419b33 956 M_WAITOK);
984263bc 957 if ((*swd)->sw_ictx == NULL) {
a0419b33 958 swcr_freesession_slot(&swd_base, 0);
984263bc
MD
959 return ENOBUFS;
960 }
961
42ee1e6b 962 (*swd)->sw_octx = kmalloc(cri->cri_klen / 8,
a0419b33 963 M_CRYPTO_DATA, M_WAITOK);
984263bc 964 if ((*swd)->sw_octx == NULL) {
a0419b33 965 swcr_freesession_slot(&swd_base, 0);
984263bc
MD
966 return ENOBUFS;
967 }
968
42ee1e6b
SW
969 /* Store the key so we can "append" it to the payload */
970 if (cri->cri_key != NULL) {
971 swcr_authprepare(axf, *swd, cri->cri_key,
972 cri->cri_klen);
973 }
974
975 (*swd)->sw_mlen = cri->cri_mlen;
984263bc
MD
976 (*swd)->sw_axf = axf;
977 break;
978#ifdef notdef
979 case CRYPTO_MD5:
980 axf = &auth_hash_md5;
981 goto auth3common;
982
983 case CRYPTO_SHA1:
984 axf = &auth_hash_sha1;
985 auth3common:
efda3bd0 986 (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
a0419b33 987 M_WAITOK);
984263bc 988 if ((*swd)->sw_ictx == NULL) {
a0419b33 989 swcr_freesession_slot(&swd_base, 0);
984263bc
MD
990 return ENOBUFS;
991 }
992
993 axf->Init((*swd)->sw_ictx);
42ee1e6b 994 (*swd)->sw_mlen = cri->cri_mlen;
984263bc
MD
995 (*swd)->sw_axf = axf;
996 break;
997#endif
aface142
AH
998 case CRYPTO_AES_128_GMAC:
999 axf = &auth_hash_gmac_aes_128;
1000 goto auth4common;
1001
1002 case CRYPTO_AES_192_GMAC:
1003 axf = &auth_hash_gmac_aes_192;
1004 goto auth4common;
1005
1006 case CRYPTO_AES_256_GMAC:
1007 axf = &auth_hash_gmac_aes_256;
1008 auth4common:
1009 (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
1010 M_NOWAIT);
1011 if ((*swd)->sw_ictx == NULL) {
1012 swcr_freesession_slot(&swd_base, 0);
1013 return ENOBUFS;
1014 }
1015
1016 axf->Init((*swd)->sw_ictx);
1017 axf->Setkey((*swd)->sw_ictx, cri->cri_key,
1018 cri->cri_klen / 8);
1019 (*swd)->sw_axf = axf;
1020 break;
1021
984263bc
MD
1022 case CRYPTO_DEFLATE_COMP:
1023 cxf = &comp_algo_deflate;
1024 (*swd)->sw_cxf = cxf;
1025 break;
1026 default:
a0419b33 1027 swcr_freesession_slot(&swd_base, 0);
984263bc
MD
1028 return EINVAL;
1029 }
1030
1031 (*swd)->sw_alg = cri->cri_alg;
1032 cri = cri->cri_next;
1033 swd = &((*swd)->sw_next);
1034 }
a0419b33
MD
1035
1036 for (;;) {
1037 /*
1038 * Atomically allocate a session
1039 */
287a8577 1040 spin_lock(&swcr_spin);
a0419b33
MD
1041 for (i = swcr_minsesnum; i < swcr_sesnum; ++i) {
1042 if (swcr_sessions[i] == NULL)
1043 break;
1044 }
1045 if (i < swcr_sesnum) {
1046 swcr_sessions[i] = swd_base;
1047 swcr_minsesnum = i + 1;
287a8577 1048 spin_unlock(&swcr_spin);
a0419b33
MD
1049 break;
1050 }
1051 n = swcr_sesnum;
287a8577 1052 spin_unlock(&swcr_spin);
a0419b33
MD
1053
1054 /*
1055 * A larger allocation is required, reallocate the array
1056 * and replace, checking for SMP races.
1057 */
1058 if (n < CRYPTO_SW_SESSIONS)
1059 n = CRYPTO_SW_SESSIONS;
1060 else
1061 n = n * 3 / 2;
1062 swd = kmalloc(n * sizeof(struct swcr_data *),
1063 M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1064
287a8577 1065 spin_lock(&swcr_spin);
a0419b33 1066 if (swcr_sesnum >= n) {
287a8577 1067 spin_unlock(&swcr_spin);
a0419b33
MD
1068 kfree(swd, M_CRYPTO_DATA);
1069 } else if (swcr_sesnum) {
1070 bcopy(swcr_sessions, swd,
1071 swcr_sesnum * sizeof(struct swcr_data *));
1072 oswd = swcr_sessions;
1073 swcr_sessions = swd;
1074 swcr_sesnum = n;
287a8577 1075 spin_unlock(&swcr_spin);
a0419b33
MD
1076 kfree(oswd, M_CRYPTO_DATA);
1077 } else {
1078 swcr_sessions = swd;
1079 swcr_sesnum = n;
287a8577 1080 spin_unlock(&swcr_spin);
a0419b33
MD
1081 }
1082 }
1083
1084 *sid = i;
984263bc
MD
1085 return 0;
1086}
1087
1088/*
1089 * Free a session.
1090 */
1091static int
42ee1e6b 1092swcr_freesession(device_t dev, u_int64_t tid)
984263bc 1093{
42ee1e6b 1094 u_int32_t sid = CRYPTO_SESID2LID(tid);
984263bc
MD
1095
1096 if (sid > swcr_sesnum || swcr_sessions == NULL ||
a0419b33 1097 swcr_sessions[sid] == NULL) {
984263bc 1098 return EINVAL;
a0419b33 1099 }
984263bc
MD
1100
1101 /* Silently accept and return */
1102 if (sid == 0)
1103 return 0;
1104
a0419b33
MD
1105 return(swcr_freesession_slot(&swcr_sessions[sid], sid));
1106}
1107
1108static
1109int
1110swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid)
1111{
1112 struct enc_xform *txf;
1113 struct auth_hash *axf;
1114 struct comp_algo *cxf;
1115 struct swcr_data *swd;
1116 struct swcr_data *swnext;
1117
1118 /*
1119 * Protect session detachment with the spinlock.
1120 */
287a8577 1121 spin_lock(&swcr_spin);
a0419b33
MD
1122 swnext = *swdp;
1123 *swdp = NULL;
1124 if (sid && swcr_minsesnum > sid)
1125 swcr_minsesnum = sid;
287a8577 1126 spin_unlock(&swcr_spin);
a0419b33
MD
1127
1128 /*
1129 * Clean up at our leisure.
1130 */
1131 while ((swd = swnext) != NULL) {
1132 swnext = swd->sw_next;
1133
1134 swd->sw_next = NULL;
984263bc
MD
1135
1136 switch (swd->sw_alg) {
1137 case CRYPTO_DES_CBC:
1138 case CRYPTO_3DES_CBC:
1139 case CRYPTO_BLF_CBC:
1140 case CRYPTO_CAST_CBC:
1141 case CRYPTO_SKIPJACK_CBC:
1142 case CRYPTO_RIJNDAEL128_CBC:
da41e4e5
AH
1143 case CRYPTO_AES_XTS:
1144 case CRYPTO_AES_CTR:
aface142
AH
1145 case CRYPTO_AES_GCM_16:
1146 case CRYPTO_AES_GMAC:
42ee1e6b 1147 case CRYPTO_CAMELLIA_CBC:
984263bc
MD
1148 case CRYPTO_NULL_CBC:
1149 txf = swd->sw_exf;
1150
1151 if (swd->sw_kschedule)
1152 txf->zerokey(&(swd->sw_kschedule));
1153 break;
1154
1155 case CRYPTO_MD5_HMAC:
1156 case CRYPTO_SHA1_HMAC:
42ee1e6b
SW
1157 case CRYPTO_SHA2_256_HMAC:
1158 case CRYPTO_SHA2_384_HMAC:
1159 case CRYPTO_SHA2_512_HMAC:
984263bc
MD
1160 case CRYPTO_RIPEMD160_HMAC:
1161 case CRYPTO_NULL_HMAC:
1162 axf = swd->sw_axf;
1163
1164 if (swd->sw_ictx) {
1165 bzero(swd->sw_ictx, axf->ctxsize);
efda3bd0 1166 kfree(swd->sw_ictx, M_CRYPTO_DATA);
984263bc
MD
1167 }
1168 if (swd->sw_octx) {
1169 bzero(swd->sw_octx, axf->ctxsize);
efda3bd0 1170 kfree(swd->sw_octx, M_CRYPTO_DATA);
984263bc
MD
1171 }
1172 break;
1173
1174 case CRYPTO_MD5_KPDK:
1175 case CRYPTO_SHA1_KPDK:
1176 axf = swd->sw_axf;
1177
1178 if (swd->sw_ictx) {
1179 bzero(swd->sw_ictx, axf->ctxsize);
efda3bd0 1180 kfree(swd->sw_ictx, M_CRYPTO_DATA);
984263bc
MD
1181 }
1182 if (swd->sw_octx) {
1183 bzero(swd->sw_octx, swd->sw_klen);
efda3bd0 1184 kfree(swd->sw_octx, M_CRYPTO_DATA);
984263bc
MD
1185 }
1186 break;
1187
aface142
AH
1188 case CRYPTO_AES_128_GMAC:
1189 case CRYPTO_AES_192_GMAC:
1190 case CRYPTO_AES_256_GMAC:
984263bc
MD
1191 case CRYPTO_MD5:
1192 case CRYPTO_SHA1:
1193 axf = swd->sw_axf;
1194
66abcdda
AH
1195 if (swd->sw_ictx) {
1196 bzero(swd->sw_ictx, axf->ctxsize);
efda3bd0 1197 kfree(swd->sw_ictx, M_CRYPTO_DATA);
66abcdda 1198 }
984263bc
MD
1199 break;
1200
1201 case CRYPTO_DEFLATE_COMP:
1202 cxf = swd->sw_cxf;
1203 break;
1204 }
1205
54734da1
AH
1206 //FREE(swd, M_CRYPTO_DATA);
1207 kfree(swd, M_CRYPTO_DATA);
984263bc
MD
1208 }
1209 return 0;
1210}
1211
1212/*
1213 * Process a software request.
1214 */
1215static int
42ee1e6b 1216swcr_process(device_t dev, struct cryptop *crp, int hint)
984263bc
MD
1217{
1218 struct cryptodesc *crd;
1219 struct swcr_data *sw;
1220 u_int32_t lid;
984263bc
MD
1221
1222 /* Sanity check */
1223 if (crp == NULL)
1224 return EINVAL;
1225
1226 if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
1227 crp->crp_etype = EINVAL;
1228 goto done;
1229 }
1230
1231 lid = crp->crp_sid & 0xffffffff;
1232 if (lid >= swcr_sesnum || lid == 0 || swcr_sessions[lid] == NULL) {
1233 crp->crp_etype = ENOENT;
1234 goto done;
1235 }
1236
984263bc
MD
1237 /* Go through crypto descriptors, processing as we go */
1238 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1239 /*
1240 * Find the crypto context.
1241 *
1242 * XXX Note that the logic here prevents us from having
1243 * XXX the same algorithm multiple times in a session
1244 * XXX (or rather, we can but it won't give us the right
1245 * XXX results). To do that, we'd need some way of differentiating
1246 * XXX between the various instances of an algorithm (so we can
1247 * XXX locate the correct crypto context).
1248 */
1249 for (sw = swcr_sessions[lid];
1250 sw && sw->sw_alg != crd->crd_alg;
1251 sw = sw->sw_next)
1252 ;
1253
1254 /* No such context ? */
1255 if (sw == NULL) {
1256 crp->crp_etype = EINVAL;
1257 goto done;
1258 }
1259 switch (sw->sw_alg) {
1260 case CRYPTO_DES_CBC:
1261 case CRYPTO_3DES_CBC:
1262 case CRYPTO_BLF_CBC:
1263 case CRYPTO_CAST_CBC:
1264 case CRYPTO_SKIPJACK_CBC:
1265 case CRYPTO_RIJNDAEL128_CBC:
da41e4e5
AH
1266 case CRYPTO_AES_XTS:
1267 case CRYPTO_AES_CTR:
42ee1e6b 1268 case CRYPTO_CAMELLIA_CBC:
984263bc 1269 if ((crp->crp_etype = swcr_encdec(crd, sw,
42ee1e6b 1270 crp->crp_buf, crp->crp_flags)) != 0)
984263bc
MD
1271 goto done;
1272 break;
1273 case CRYPTO_NULL_CBC:
1274 crp->crp_etype = 0;
1275 break;
1276 case CRYPTO_MD5_HMAC:
1277 case CRYPTO_SHA1_HMAC:
42ee1e6b
SW
1278 case CRYPTO_SHA2_256_HMAC:
1279 case CRYPTO_SHA2_384_HMAC:
1280 case CRYPTO_SHA2_512_HMAC:
984263bc
MD
1281 case CRYPTO_RIPEMD160_HMAC:
1282 case CRYPTO_NULL_HMAC:
1283 case CRYPTO_MD5_KPDK:
1284 case CRYPTO_SHA1_KPDK:
1285 case CRYPTO_MD5:
1286 case CRYPTO_SHA1:
42ee1e6b
SW
1287 if ((crp->crp_etype = swcr_authcompute(crd, sw,
1288 crp->crp_buf, crp->crp_flags)) != 0)
984263bc
MD
1289 goto done;
1290 break;
1291
aface142
AH
1292 case CRYPTO_AES_GCM_16:
1293 case CRYPTO_AES_GMAC:
1294 case CRYPTO_AES_128_GMAC:
1295 case CRYPTO_AES_192_GMAC:
1296 case CRYPTO_AES_256_GMAC:
1297 crp->crp_etype = swcr_combined(crp);
1298 goto done;
1299
984263bc
MD
1300 case CRYPTO_DEFLATE_COMP:
1301 if ((crp->crp_etype = swcr_compdec(crd, sw,
42ee1e6b 1302 crp->crp_buf, crp->crp_flags)) != 0)
984263bc
MD
1303 goto done;
1304 else
1305 crp->crp_olen = (int)sw->sw_size;
1306 break;
1307
1308 default:
1309 /* Unknown/unsupported algorithm */
1310 crp->crp_etype = EINVAL;
1311 goto done;
1312 }
1313 }
1314
1315done:
1316 crypto_done(crp);
c5fcbac7 1317 lwkt_yield();
984263bc
MD
1318 return 0;
1319}
1320
984263bc 1321static void
42ee1e6b
SW
1322swcr_identify(driver_t *drv, device_t parent)
1323{
1324 /* NB: order 10 is so we get attached after h/w devices */
54734da1 1325 /* XXX: wouldn't bet about this BUS_ADD_CHILD correctness */
42ee1e6b 1326 if (device_find_child(parent, "cryptosoft", -1) == NULL &&
54734da1 1327 BUS_ADD_CHILD(parent, parent, 10, "cryptosoft", -1) == 0)
42ee1e6b
SW
1328 panic("cryptosoft: could not attach");
1329}
1330
1331static int
1332swcr_probe(device_t dev)
984263bc 1333{
42ee1e6b
SW
1334 device_set_desc(dev, "software crypto");
1335 return (0);
1336}
1337
1338static int
1339swcr_attach(device_t dev)
1340{
1341 memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
1342 memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
1343
a0419b33
MD
1344 swcr_id = crypto_get_driverid(dev, CRYPTOCAP_F_SOFTWARE |
1345 CRYPTOCAP_F_SYNC |
1346 CRYPTOCAP_F_SMP);
42ee1e6b
SW
1347 if (swcr_id < 0) {
1348 device_printf(dev, "cannot initialize!");
1349 return ENOMEM;
1350 }
984263bc 1351#define REGISTER(alg) \
42ee1e6b
SW
1352 crypto_register(swcr_id, alg, 0,0)
1353 REGISTER(CRYPTO_DES_CBC);
984263bc
MD
1354 REGISTER(CRYPTO_3DES_CBC);
1355 REGISTER(CRYPTO_BLF_CBC);
1356 REGISTER(CRYPTO_CAST_CBC);
1357 REGISTER(CRYPTO_SKIPJACK_CBC);
1358 REGISTER(CRYPTO_NULL_CBC);
1359 REGISTER(CRYPTO_MD5_HMAC);
1360 REGISTER(CRYPTO_SHA1_HMAC);
42ee1e6b
SW
1361 REGISTER(CRYPTO_SHA2_256_HMAC);
1362 REGISTER(CRYPTO_SHA2_384_HMAC);
1363 REGISTER(CRYPTO_SHA2_512_HMAC);
984263bc
MD
1364 REGISTER(CRYPTO_RIPEMD160_HMAC);
1365 REGISTER(CRYPTO_NULL_HMAC);
1366 REGISTER(CRYPTO_MD5_KPDK);
1367 REGISTER(CRYPTO_SHA1_KPDK);
1368 REGISTER(CRYPTO_MD5);
1369 REGISTER(CRYPTO_SHA1);
1370 REGISTER(CRYPTO_RIJNDAEL128_CBC);
da41e4e5
AH
1371 REGISTER(CRYPTO_AES_XTS);
1372 REGISTER(CRYPTO_AES_CTR);
aface142
AH
1373 REGISTER(CRYPTO_AES_GCM_16);
1374 REGISTER(CRYPTO_AES_GMAC);
1375 REGISTER(CRYPTO_AES_128_GMAC);
1376 REGISTER(CRYPTO_AES_192_GMAC);
1377 REGISTER(CRYPTO_AES_256_GMAC);
42ee1e6b 1378 REGISTER(CRYPTO_CAMELLIA_CBC);
984263bc
MD
1379 REGISTER(CRYPTO_DEFLATE_COMP);
1380#undef REGISTER
42ee1e6b
SW
1381
1382 return 0;
1383}
1384
1385static int
1386swcr_detach(device_t dev)
1387{
1388 crypto_unregister_all(swcr_id);
1389 if (swcr_sessions != NULL)
54734da1 1390 kfree(swcr_sessions, M_CRYPTO_DATA);
42ee1e6b 1391 return 0;
984263bc 1392}
42ee1e6b
SW
1393
1394static device_method_t swcr_methods[] = {
1395 DEVMETHOD(device_identify, swcr_identify),
1396 DEVMETHOD(device_probe, swcr_probe),
1397 DEVMETHOD(device_attach, swcr_attach),
1398 DEVMETHOD(device_detach, swcr_detach),
1399
1400 DEVMETHOD(cryptodev_newsession, swcr_newsession),
1401 DEVMETHOD(cryptodev_freesession,swcr_freesession),
1402 DEVMETHOD(cryptodev_process, swcr_process),
1403
1404 {0, 0},
1405};
1406
1407static driver_t swcr_driver = {
1408 "cryptosoft",
1409 swcr_methods,
1410 0, /* NB: no softc */
1411};
1412static devclass_t swcr_devclass;
1413
1414/*
1415 * NB: We explicitly reference the crypto module so we
1416 * get the necessary ordering when built as a loadable
1417 * module. This is required because we bundle the crypto
1418 * module code together with the cryptosoft driver (otherwise
1419 * normal module dependencies would handle things).
1420 */
1421extern int crypto_modevent(struct module *, int, void *);
1422/* XXX where to attach */
aa2b9d05 1423DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,NULL);
42ee1e6b
SW
1424MODULE_VERSION(cryptosoft, 1);
1425MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);