15e3ba10101b6e47a9bfd40c08bb638e30fd7e1b
[dragonfly.git] / sys / opencrypto / cryptosoft.c
1 /*-
2  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
3  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
4  *
5  * This code was written by Angelos D. Keromytis in Athens, Greece, in
6  * February 2000. Network Security Technologies Inc. (NSTI) kindly
7  * supported the development of this code.
8  *
9  * Copyright (c) 2000, 2001 Angelos D. Keromytis
10  *
11  * SMP modifications by Matthew Dillon for the DragonFlyBSD Project
12  *
13  * Permission to use, copy, and modify this software with or without fee
14  * is hereby granted, provided that this entire notice is included in
15  * all source code copies of any software which is or includes a copy or
16  * modification of this software.
17  *
18  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
19  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
20  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
21  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
22  * PURPOSE.
23  *
24  * $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $
25  * $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $
26  */
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/malloc.h>
31 #include <sys/mbuf.h>
32 #include <sys/module.h>
33 #include <sys/sysctl.h>
34 #include <sys/errno.h>
35 #include <sys/random.h>
36 #include <sys/kernel.h>
37 #include <sys/uio.h>
38 #include <sys/spinlock2.h>
39
40 #include <crypto/blowfish/blowfish.h>
41 #include <crypto/sha1.h>
42 #include <opencrypto/rmd160.h>
43 #include <opencrypto/cast.h>
44 #include <opencrypto/skipjack.h>
45 #include <sys/md5.h>
46
47 #include <opencrypto/cryptodev.h>
48 #include <opencrypto/cryptosoft.h>
49 #include <opencrypto/xform.h>
50
51 #include <sys/kobj.h>
52 #include <sys/bus.h>
53 #include "cryptodev_if.h"
54
55 static  int32_t swcr_id;
56 static  struct swcr_data **swcr_sessions = NULL;
57 static  u_int32_t swcr_sesnum;
58 static  u_int32_t swcr_minsesnum = 1;
59
60 static struct spinlock swcr_spin = SPINLOCK_INITIALIZER(swcr_spin);
61
62 u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
63 u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
64
65 static  int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
66 static  int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
67 static  int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
68 static  int swcr_freesession(device_t dev, u_int64_t tid);
69 static  int swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid);
70
71 /*
72  * Apply a symmetric encryption/decryption algorithm.
73  */
74 static int
75 swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
76     int flags)
77 {
78         unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
79         unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN];
80         u_int8_t *kschedule;
81         u_int8_t *okschedule;
82         struct enc_xform *exf;
83         int i, k, j, blks, ivlen;
84         int error;
85         int explicit_kschedule;
86
87         exf = sw->sw_exf;
88         blks = exf->blocksize;
89         ivlen = exf->ivsize;
90
91         /* Check for non-padded data */
92         if (crd->crd_len % blks)
93                 return EINVAL;
94
95         /* Initialize the IV */
96         if (crd->crd_flags & CRD_F_ENCRYPT) {
97                 /* IV explicitly provided ? */
98                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
99                         bcopy(crd->crd_iv, iv, ivlen);
100                 else
101                         karc4rand(iv, ivlen);
102
103                 /* Do we need to write the IV */
104                 if (!(crd->crd_flags & CRD_F_IV_PRESENT))
105                         crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
106
107         } else {        /* Decryption */
108                         /* IV explicitly provided ? */
109                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
110                         bcopy(crd->crd_iv, iv, ivlen);
111                 else {
112                         /* Get IV off buf */
113                         crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
114                 }
115         }
116
117         ivp = iv;
118
119         /*
120          * The semantics are seriously broken because the session key
121          * storage was never designed for concurrent ops.
122          */
123         if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
124                 kschedule = NULL;
125                 explicit_kschedule = 1;
126                 error = exf->setkey(&kschedule,
127                                     crd->crd_key, crd->crd_klen / 8);
128                 if (error)
129                         goto done;
130         } else {
131                 spin_lock(&swcr_spin);
132                 kschedule = sw->sw_kschedule;
133                 ++sw->sw_kschedule_refs;
134                 spin_unlock(&swcr_spin);
135                 explicit_kschedule = 0;
136         }
137
138         /*
139          * xforms that provide a reinit method perform all IV
140          * handling themselves.
141          */
142         if (exf->reinit)
143                 exf->reinit(kschedule, iv);
144
145         if (flags & CRYPTO_F_IMBUF) {
146                 struct mbuf *m = (struct mbuf *) buf;
147
148                 /* Find beginning of data */
149                 m = m_getptr(m, crd->crd_skip, &k);
150                 if (m == NULL) {
151                         error = EINVAL;
152                         goto done;
153                 }
154
155                 i = crd->crd_len;
156
157                 while (i > 0) {
158                         /*
159                          * If there's insufficient data at the end of
160                          * an mbuf, we have to do some copying.
161                          */
162                         if (m->m_len < k + blks && m->m_len != k) {
163                                 m_copydata(m, k, blks, blk);
164
165                                 /* Actual encryption/decryption */
166                                 if (exf->reinit) {
167                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
168                                                 exf->encrypt(kschedule,
169                                                     blk, iv);
170                                         } else {
171                                                 exf->decrypt(kschedule,
172                                                     blk, iv);
173                                         }
174                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
175                                         /* XOR with previous block */
176                                         for (j = 0; j < blks; j++)
177                                                 blk[j] ^= ivp[j];
178
179                                         exf->encrypt(kschedule, blk, iv);
180
181                                         /*
182                                          * Keep encrypted block for XOR'ing
183                                          * with next block
184                                          */
185                                         bcopy(blk, iv, blks);
186                                         ivp = iv;
187                                 } else {        /* decrypt */
188                                         /*      
189                                          * Keep encrypted block for XOR'ing
190                                          * with next block
191                                          */
192                                         nivp = (ivp == iv) ? iv2 : iv;
193                                         bcopy(blk, nivp, blks);
194
195                                         exf->decrypt(kschedule, blk, iv);
196
197                                         /* XOR with previous block */
198                                         for (j = 0; j < blks; j++)
199                                                 blk[j] ^= ivp[j];
200
201                                         ivp = nivp;
202                                 }
203
204                                 /* Copy back decrypted block */
205                                 m_copyback(m, k, blks, blk);
206
207                                 /* Advance pointer */
208                                 m = m_getptr(m, k + blks, &k);
209                                 if (m == NULL) {
210                                         error = EINVAL;
211                                         goto done;
212                                 }
213
214                                 i -= blks;
215
216                                 /* Could be done... */
217                                 if (i == 0)
218                                         break;
219                         }
220
221                         /* Skip possibly empty mbufs */
222                         if (k == m->m_len) {
223                                 for (m = m->m_next; m && m->m_len == 0;
224                                     m = m->m_next)
225                                         ;
226                                 k = 0;
227                         }
228
229                         /* Sanity check */
230                         if (m == NULL) {
231                                 error = EINVAL;
232                                 goto done;
233                         }
234
235                         /*
236                          * Warning: idat may point to garbage here, but
237                          * we only use it in the while() loop, only if
238                          * there are indeed enough data.
239                          */
240                         idat = mtod(m, unsigned char *) + k;
241
242                         while (m->m_len >= k + blks && i > 0) {
243                                 if (exf->reinit) {
244                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
245                                                 exf->encrypt(kschedule,
246                                                     idat, iv);
247                                         } else {
248                                                 exf->decrypt(kschedule,
249                                                     idat, iv);
250                                         }
251                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
252                                         /* XOR with previous block/IV */
253                                         for (j = 0; j < blks; j++)
254                                                 idat[j] ^= ivp[j];
255
256                                         exf->encrypt(kschedule, idat, iv);
257                                         ivp = idat;
258                                 } else {        /* decrypt */
259                                         /*
260                                          * Keep encrypted block to be used
261                                          * in next block's processing.
262                                          */
263                                         nivp = (ivp == iv) ? iv2 : iv;
264                                         bcopy(idat, nivp, blks);
265
266                                         exf->decrypt(kschedule, idat, iv);
267
268                                         /* XOR with previous block/IV */
269                                         for (j = 0; j < blks; j++)
270                                                 idat[j] ^= ivp[j];
271
272                                         ivp = nivp;
273                                 }
274
275                                 idat += blks;
276                                 k += blks;
277                                 i -= blks;
278                         }
279                 }
280                 error = 0;      /* Done with mbuf encryption/decryption */
281         } else if (flags & CRYPTO_F_IOV) {
282                 struct uio *uio = (struct uio *) buf;
283                 struct iovec *iov;
284
285                 /* Find beginning of data */
286                 iov = cuio_getptr(uio, crd->crd_skip, &k);
287                 if (iov == NULL) {
288                         error = EINVAL;
289                         goto done;
290                 }
291
292                 i = crd->crd_len;
293
294                 while (i > 0) {
295                         /*
296                          * If there's insufficient data at the end of
297                          * an iovec, we have to do some copying.
298                          */
299                         if (iov->iov_len < k + blks && iov->iov_len != k) {
300                                 cuio_copydata(uio, k, blks, blk);
301
302                                 /* Actual encryption/decryption */
303                                 if (exf->reinit) {
304                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
305                                                 exf->encrypt(kschedule,
306                                                     blk, iv);
307                                         } else {
308                                                 exf->decrypt(kschedule,
309                                                     blk, iv);
310                                         }
311                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
312                                         /* XOR with previous block */
313                                         for (j = 0; j < blks; j++)
314                                                 blk[j] ^= ivp[j];
315
316                                         exf->encrypt(kschedule, blk, iv);
317
318                                         /*
319                                          * Keep encrypted block for XOR'ing
320                                          * with next block
321                                          */
322                                         bcopy(blk, iv, blks);
323                                         ivp = iv;
324                                 } else {        /* decrypt */
325                                         /*      
326                                          * Keep encrypted block for XOR'ing
327                                          * with next block
328                                          */
329                                         nivp = (ivp == iv) ? iv2 : iv;
330                                         bcopy(blk, nivp, blks);
331
332                                         exf->decrypt(kschedule, blk, iv);
333
334                                         /* XOR with previous block */
335                                         for (j = 0; j < blks; j++)
336                                                 blk[j] ^= ivp[j];
337
338                                         ivp = nivp;
339                                 }
340
341                                 /* Copy back decrypted block */
342                                 cuio_copyback(uio, k, blks, blk);
343
344                                 /* Advance pointer */
345                                 iov = cuio_getptr(uio, k + blks, &k);
346                                 if (iov == NULL) {
347                                         error = EINVAL;
348                                         goto done;
349                                 }
350
351                                 i -= blks;
352
353                                 /* Could be done... */
354                                 if (i == 0)
355                                         break;
356                         }
357
358                         /*
359                          * Warning: idat may point to garbage here, but
360                          * we only use it in the while() loop, only if
361                          * there are indeed enough data.
362                          */
363                         idat = (char *)iov->iov_base + k;
364
365                         while (iov->iov_len >= k + blks && i > 0) {
366                                 if (exf->reinit) {
367                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
368                                                 exf->encrypt(kschedule,
369                                                     idat, iv);
370                                         } else {
371                                                 exf->decrypt(kschedule,
372                                                     idat, iv);
373                                         }
374                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
375                                         /* XOR with previous block/IV */
376                                         for (j = 0; j < blks; j++)
377                                                 idat[j] ^= ivp[j];
378
379                                         exf->encrypt(kschedule, idat, iv);
380                                         ivp = idat;
381                                 } else {        /* decrypt */
382                                         /*
383                                          * Keep encrypted block to be used
384                                          * in next block's processing.
385                                          */
386                                         nivp = (ivp == iv) ? iv2 : iv;
387                                         bcopy(idat, nivp, blks);
388
389                                         exf->decrypt(kschedule, idat, iv);
390
391                                         /* XOR with previous block/IV */
392                                         for (j = 0; j < blks; j++)
393                                                 idat[j] ^= ivp[j];
394
395                                         ivp = nivp;
396                                 }
397
398                                 idat += blks;
399                                 k += blks;
400                                 i -= blks;
401                         }
402                         if (k == iov->iov_len) {
403                                 iov++;
404                                 k = 0;
405                         }
406                 }
407                 error = 0;      /* Done with iovec encryption/decryption */
408         } else {
409                 /*
410                  * contiguous buffer
411                  */
412                 if (exf->reinit) {
413                         for(i = crd->crd_skip;
414                             i < crd->crd_skip + crd->crd_len; i += blks) {
415                                 if (crd->crd_flags & CRD_F_ENCRYPT) {
416                                         exf->encrypt(kschedule, buf + i, iv);
417                                 } else {
418                                         exf->decrypt(kschedule, buf + i, iv);
419                                 }
420                         }
421                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
422                         for (i = crd->crd_skip;
423                             i < crd->crd_skip + crd->crd_len; i += blks) {
424                                 /* XOR with the IV/previous block, as appropriate. */
425                                 if (i == crd->crd_skip)
426                                         for (k = 0; k < blks; k++)
427                                                 buf[i + k] ^= ivp[k];
428                                 else
429                                         for (k = 0; k < blks; k++)
430                                                 buf[i + k] ^= buf[i + k - blks];
431                                 exf->encrypt(kschedule, buf + i, iv);
432                         }
433                 } else {                /* Decrypt */
434                         /*
435                          * Start at the end, so we don't need to keep the
436                          * encrypted block as the IV for the next block.
437                          */
438                         for (i = crd->crd_skip + crd->crd_len - blks;
439                             i >= crd->crd_skip; i -= blks) {
440                                 exf->decrypt(kschedule, buf + i, iv);
441
442                                 /* XOR with the IV/previous block, as appropriate */
443                                 if (i == crd->crd_skip)
444                                         for (k = 0; k < blks; k++)
445                                                 buf[i + k] ^= ivp[k];
446                                 else
447                                         for (k = 0; k < blks; k++)
448                                                 buf[i + k] ^= buf[i + k - blks];
449                         }
450                 }
451                 error = 0; /* Done w/contiguous buffer encrypt/decrypt */
452         }
453 done:
454         /*
455          * Cleanup - explicitly replace the session key if requested
456          *           (horrible semantics for concurrent operation)
457          */
458         if (explicit_kschedule) {
459                 spin_lock(&swcr_spin);
460                 if (sw->sw_kschedule && sw->sw_kschedule_refs == 0) {
461                         okschedule = sw->sw_kschedule;
462                         sw->sw_kschedule = kschedule;
463                 } else {
464                         okschedule = NULL;
465                 }
466                 spin_unlock(&swcr_spin);
467                 if (okschedule)
468                         exf->zerokey(&okschedule);
469         } else {
470                 spin_lock(&swcr_spin);
471                 --sw->sw_kschedule_refs;
472                 spin_unlock(&swcr_spin);
473         }
474         return error;
475 }
476
477 static void
478 swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
479     int klen)
480 {
481         int k;
482
483         klen /= 8;
484
485         switch (axf->type) {
486         case CRYPTO_MD5_HMAC:
487         case CRYPTO_SHA1_HMAC:
488         case CRYPTO_SHA2_256_HMAC:
489         case CRYPTO_SHA2_384_HMAC:
490         case CRYPTO_SHA2_512_HMAC:
491         case CRYPTO_NULL_HMAC:
492         case CRYPTO_RIPEMD160_HMAC:
493                 for (k = 0; k < klen; k++)
494                         key[k] ^= HMAC_IPAD_VAL;
495
496                 axf->Init(sw->sw_ictx);
497                 axf->Update(sw->sw_ictx, key, klen);
498                 axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
499
500                 for (k = 0; k < klen; k++)
501                         key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
502
503                 axf->Init(sw->sw_octx);
504                 axf->Update(sw->sw_octx, key, klen);
505                 axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
506
507                 for (k = 0; k < klen; k++)
508                         key[k] ^= HMAC_OPAD_VAL;
509                 break;
510         case CRYPTO_MD5_KPDK:
511         case CRYPTO_SHA1_KPDK:
512         {
513                 /* We need a buffer that can hold an md5 and a sha1 result. */
514                 u_char buf[SHA1_RESULTLEN];
515
516                 sw->sw_klen = klen;
517                 bcopy(key, sw->sw_octx, klen);
518                 axf->Init(sw->sw_ictx);
519                 axf->Update(sw->sw_ictx, key, klen);
520                 axf->Final(buf, sw->sw_ictx);
521                 break;
522         }
523         default:
524                 kprintf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
525                     "doesn't use keys.\n", __func__, axf->type);
526         }
527 }
528
529 /*
530  * Compute keyed-hash authenticator.
531  */
532 static int
533 swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
534     int flags)
535 {
536         unsigned char aalg[HASH_MAX_LEN];
537         struct auth_hash *axf;
538         union authctx ctx;
539         int err;
540
541         if (sw->sw_ictx == 0)
542                 return EINVAL;
543
544         axf = sw->sw_axf;
545
546         if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
547                 swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
548
549         bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
550
551         err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
552             (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
553         if (err)
554                 return err;
555
556         switch (sw->sw_alg) {
557         case CRYPTO_MD5_HMAC:
558         case CRYPTO_SHA1_HMAC:
559         case CRYPTO_SHA2_256_HMAC:
560         case CRYPTO_SHA2_384_HMAC:
561         case CRYPTO_SHA2_512_HMAC:
562         case CRYPTO_RIPEMD160_HMAC:
563                 if (sw->sw_octx == NULL)
564                         return EINVAL;
565
566                 axf->Final(aalg, &ctx);
567                 bcopy(sw->sw_octx, &ctx, axf->ctxsize);
568                 axf->Update(&ctx, aalg, axf->hashsize);
569                 axf->Final(aalg, &ctx);
570                 break;
571
572         case CRYPTO_MD5_KPDK:
573         case CRYPTO_SHA1_KPDK:
574                 if (sw->sw_octx == NULL)
575                         return EINVAL;
576
577                 axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
578                 axf->Final(aalg, &ctx);
579                 break;
580
581         case CRYPTO_NULL_HMAC:
582                 axf->Final(aalg, &ctx);
583                 break;
584         }
585
586         /* Inject the authentication data */
587         crypto_copyback(flags, buf, crd->crd_inject,
588             sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
589         return 0;
590 }
591
592 /*
593  * Apply a compression/decompression algorithm
594  */
595 static int
596 swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
597              caddr_t buf, int flags)
598 {
599         u_int8_t *data, *out;
600         struct comp_algo *cxf;
601         int adj;
602         u_int32_t result;
603
604         cxf = sw->sw_cxf;
605
606         /*
607          * We must handle the whole buffer of data in one time
608          * then if there is not all the data in the mbuf, we must
609          * copy in a buffer.
610          */
611         data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_INTWAIT);
612         if (data == NULL)
613                 return (EINVAL);
614         crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
615
616         if (crd->crd_flags & CRD_F_COMP)
617                 result = cxf->compress(data, crd->crd_len, &out);
618         else
619                 result = cxf->decompress(data, crd->crd_len, &out);
620
621         kfree(data, M_CRYPTO_DATA);
622         if (result == 0)
623                 return EINVAL;
624
625         /* Copy back the (de)compressed data. m_copyback is
626          * extending the mbuf as necessary.
627          */
628         sw->sw_size = result;
629         /* Check the compressed size when doing compression */
630         if (crd->crd_flags & CRD_F_COMP) {
631                 if (result >= crd->crd_len) {
632                         /* Compression was useless, we lost time */
633                         kfree(out, M_CRYPTO_DATA);
634                         return 0;
635                 }
636         }
637
638         crypto_copyback(flags, buf, crd->crd_skip, result, out);
639         if (result < crd->crd_len) {
640                 adj = result - crd->crd_len;
641                 if (flags & CRYPTO_F_IMBUF) {
642                         adj = result - crd->crd_len;
643                         m_adj((struct mbuf *)buf, adj);
644                 } else if (flags & CRYPTO_F_IOV) {
645                         struct uio *uio = (struct uio *)buf;
646                         int ind;
647
648                         adj = crd->crd_len - result;
649                         ind = uio->uio_iovcnt - 1;
650
651                         while (adj > 0 && ind >= 0) {
652                                 if (adj < uio->uio_iov[ind].iov_len) {
653                                         uio->uio_iov[ind].iov_len -= adj;
654                                         break;
655                                 }
656
657                                 adj -= uio->uio_iov[ind].iov_len;
658                                 uio->uio_iov[ind].iov_len = 0;
659                                 ind--;
660                                 uio->uio_iovcnt--;
661                         }
662                 }
663         }
664         kfree(out, M_CRYPTO_DATA);
665         return 0;
666 }
667
668 /*
669  * Generate a new software session.
670  */
671 static int
672 swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
673 {
674         struct swcr_data *swd_base;
675         struct swcr_data **swd;
676         struct swcr_data **oswd;
677         struct auth_hash *axf;
678         struct enc_xform *txf;
679         struct comp_algo *cxf;
680         u_int32_t i;
681         u_int32_t n;
682         int error;
683
684         if (sid == NULL || cri == NULL)
685                 return EINVAL;
686
687         swd_base = NULL;
688         swd = &swd_base;
689
690         while (cri) {
691                 *swd = kmalloc(sizeof(struct swcr_data),
692                                M_CRYPTO_DATA, M_WAITOK | M_ZERO);
693
694                 switch (cri->cri_alg) {
695                 case CRYPTO_DES_CBC:
696                         txf = &enc_xform_des;
697                         goto enccommon;
698                 case CRYPTO_3DES_CBC:
699                         txf = &enc_xform_3des;
700                         goto enccommon;
701                 case CRYPTO_BLF_CBC:
702                         txf = &enc_xform_blf;
703                         goto enccommon;
704                 case CRYPTO_CAST_CBC:
705                         txf = &enc_xform_cast5;
706                         goto enccommon;
707                 case CRYPTO_SKIPJACK_CBC:
708                         txf = &enc_xform_skipjack;
709                         goto enccommon;
710                 case CRYPTO_RIJNDAEL128_CBC:
711                         txf = &enc_xform_rijndael128;
712                         goto enccommon;
713                 case CRYPTO_AES_XTS:
714                         txf = &enc_xform_aes_xts;
715                         goto enccommon;
716                 case CRYPTO_AES_CTR:
717                         txf = &enc_xform_aes_ctr;
718                         goto enccommon;
719                 case CRYPTO_CAMELLIA_CBC:
720                         txf = &enc_xform_camellia;
721                         goto enccommon;
722                 case CRYPTO_NULL_CBC:
723                         txf = &enc_xform_null;
724                         goto enccommon;
725                 enccommon:
726                         if (cri->cri_key != NULL) {
727                                 error = txf->setkey(&((*swd)->sw_kschedule),
728                                                     cri->cri_key,
729                                                     cri->cri_klen / 8);
730                                 if (error) {
731                                         swcr_freesession_slot(&swd_base, 0);
732                                         return error;
733                                 }
734                         }
735                         (*swd)->sw_exf = txf;
736                         break;
737
738                 case CRYPTO_MD5_HMAC:
739                         axf = &auth_hash_hmac_md5;
740                         goto authcommon;
741                 case CRYPTO_SHA1_HMAC:
742                         axf = &auth_hash_hmac_sha1;
743                         goto authcommon;
744                 case CRYPTO_SHA2_256_HMAC:
745                         axf = &auth_hash_hmac_sha2_256;
746                         goto authcommon;
747                 case CRYPTO_SHA2_384_HMAC:
748                         axf = &auth_hash_hmac_sha2_384;
749                         goto authcommon;
750                 case CRYPTO_SHA2_512_HMAC:
751                         axf = &auth_hash_hmac_sha2_512;
752                         goto authcommon;
753                 case CRYPTO_NULL_HMAC:
754                         axf = &auth_hash_null;
755                         goto authcommon;
756                 case CRYPTO_RIPEMD160_HMAC:
757                         axf = &auth_hash_hmac_ripemd_160;
758                 authcommon:
759                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
760                                                   M_WAITOK);
761                         if ((*swd)->sw_ictx == NULL) {
762                                 swcr_freesession_slot(&swd_base, 0);
763                                 return ENOBUFS;
764                         }
765         
766                         (*swd)->sw_octx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
767                                                   M_WAITOK);
768                         if ((*swd)->sw_octx == NULL) {
769                                 swcr_freesession_slot(&swd_base, 0);
770                                 return ENOBUFS;
771                         }
772         
773                         if (cri->cri_key != NULL) {
774                                 swcr_authprepare(axf, *swd, cri->cri_key,
775                                     cri->cri_klen);
776                         }
777         
778                         (*swd)->sw_mlen = cri->cri_mlen;
779                         (*swd)->sw_axf = axf;
780                         break;
781         
782                 case CRYPTO_MD5_KPDK:
783                         axf = &auth_hash_key_md5;
784                         goto auth2common;
785         
786                 case CRYPTO_SHA1_KPDK:
787                         axf = &auth_hash_key_sha1;
788                 auth2common:
789                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
790                                                   M_WAITOK);
791                         if ((*swd)->sw_ictx == NULL) {
792                                 swcr_freesession_slot(&swd_base, 0);
793                                 return ENOBUFS;
794                         }
795         
796                         (*swd)->sw_octx = kmalloc(cri->cri_klen / 8,
797                                                   M_CRYPTO_DATA, M_WAITOK);
798                         if ((*swd)->sw_octx == NULL) {
799                                 swcr_freesession_slot(&swd_base, 0);
800                                 return ENOBUFS;
801                         }
802         
803                         /* Store the key so we can "append" it to the payload */
804                         if (cri->cri_key != NULL) {
805                                 swcr_authprepare(axf, *swd, cri->cri_key,
806                                     cri->cri_klen);
807                         }
808
809                         (*swd)->sw_mlen = cri->cri_mlen;
810                         (*swd)->sw_axf = axf;
811                         break;
812 #ifdef notdef
813                 case CRYPTO_MD5:
814                         axf = &auth_hash_md5;
815                         goto auth3common;
816
817                 case CRYPTO_SHA1:
818                         axf = &auth_hash_sha1;
819                 auth3common:
820                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
821                                                   M_WAITOK);
822                         if ((*swd)->sw_ictx == NULL) {
823                                 swcr_freesession_slot(&swd_base, 0);
824                                 return ENOBUFS;
825                         }
826
827                         axf->Init((*swd)->sw_ictx);
828                         (*swd)->sw_mlen = cri->cri_mlen;
829                         (*swd)->sw_axf = axf;
830                         break;
831 #endif
832                 case CRYPTO_DEFLATE_COMP:
833                         cxf = &comp_algo_deflate;
834                         (*swd)->sw_cxf = cxf;
835                         break;
836                 default:
837                         swcr_freesession_slot(&swd_base, 0);
838                         return EINVAL;
839                 }
840         
841                 (*swd)->sw_alg = cri->cri_alg;
842                 cri = cri->cri_next;
843                 swd = &((*swd)->sw_next);
844         }
845
846         for (;;) {
847                 /*
848                  * Atomically allocate a session
849                  */
850                 spin_lock(&swcr_spin);
851                 for (i = swcr_minsesnum; i < swcr_sesnum; ++i) {
852                         if (swcr_sessions[i] == NULL)
853                                 break;
854                 }
855                 if (i < swcr_sesnum) {
856                         swcr_sessions[i] = swd_base;
857                         swcr_minsesnum = i + 1;
858                         spin_unlock(&swcr_spin);
859                         break;
860                 }
861                 n = swcr_sesnum;
862                 spin_unlock(&swcr_spin);
863
864                 /*
865                  * A larger allocation is required, reallocate the array
866                  * and replace, checking for SMP races.
867                  */
868                 if (n < CRYPTO_SW_SESSIONS)
869                         n = CRYPTO_SW_SESSIONS;
870                 else
871                         n = n * 3 / 2;
872                 swd = kmalloc(n * sizeof(struct swcr_data *),
873                               M_CRYPTO_DATA, M_WAITOK | M_ZERO);
874
875                 spin_lock(&swcr_spin);
876                 if (swcr_sesnum >= n) {
877                         spin_unlock(&swcr_spin);
878                         kfree(swd, M_CRYPTO_DATA);
879                 } else if (swcr_sesnum) {
880                         bcopy(swcr_sessions, swd,
881                               swcr_sesnum * sizeof(struct swcr_data *));
882                         oswd = swcr_sessions;
883                         swcr_sessions = swd;
884                         swcr_sesnum = n;
885                         spin_unlock(&swcr_spin);
886                         kfree(oswd, M_CRYPTO_DATA);
887                 } else {
888                         swcr_sessions = swd;
889                         swcr_sesnum = n;
890                         spin_unlock(&swcr_spin);
891                 }
892         }
893
894         *sid = i;
895         return 0;
896 }
897
898 /*
899  * Free a session.
900  */
901 static int
902 swcr_freesession(device_t dev, u_int64_t tid)
903 {
904         u_int32_t sid = CRYPTO_SESID2LID(tid);
905
906         if (sid > swcr_sesnum || swcr_sessions == NULL ||
907             swcr_sessions[sid] == NULL) {
908                 return EINVAL;
909         }
910
911         /* Silently accept and return */
912         if (sid == 0)
913                 return 0;
914
915         return(swcr_freesession_slot(&swcr_sessions[sid], sid));
916 }
917
918 static
919 int
920 swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid)
921 {
922         struct enc_xform *txf;
923         struct auth_hash *axf;
924         struct comp_algo *cxf;
925         struct swcr_data *swd;
926         struct swcr_data *swnext;
927
928         /*
929          * Protect session detachment with the spinlock.
930          */
931         spin_lock(&swcr_spin);
932         swnext = *swdp;
933         *swdp = NULL;
934         if (sid && swcr_minsesnum > sid)
935                 swcr_minsesnum = sid;
936         spin_unlock(&swcr_spin);
937
938         /*
939          * Clean up at our leisure.
940          */
941         while ((swd = swnext) != NULL) {
942                 swnext = swd->sw_next;
943
944                 swd->sw_next = NULL;
945
946                 switch (swd->sw_alg) {
947                 case CRYPTO_DES_CBC:
948                 case CRYPTO_3DES_CBC:
949                 case CRYPTO_BLF_CBC:
950                 case CRYPTO_CAST_CBC:
951                 case CRYPTO_SKIPJACK_CBC:
952                 case CRYPTO_RIJNDAEL128_CBC:
953                 case CRYPTO_AES_XTS:
954                 case CRYPTO_AES_CTR:
955                 case CRYPTO_CAMELLIA_CBC:
956                 case CRYPTO_NULL_CBC:
957                         txf = swd->sw_exf;
958
959                         if (swd->sw_kschedule)
960                                 txf->zerokey(&(swd->sw_kschedule));
961                         break;
962
963                 case CRYPTO_MD5_HMAC:
964                 case CRYPTO_SHA1_HMAC:
965                 case CRYPTO_SHA2_256_HMAC:
966                 case CRYPTO_SHA2_384_HMAC:
967                 case CRYPTO_SHA2_512_HMAC:
968                 case CRYPTO_RIPEMD160_HMAC:
969                 case CRYPTO_NULL_HMAC:
970                         axf = swd->sw_axf;
971
972                         if (swd->sw_ictx) {
973                                 bzero(swd->sw_ictx, axf->ctxsize);
974                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
975                         }
976                         if (swd->sw_octx) {
977                                 bzero(swd->sw_octx, axf->ctxsize);
978                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
979                         }
980                         break;
981
982                 case CRYPTO_MD5_KPDK:
983                 case CRYPTO_SHA1_KPDK:
984                         axf = swd->sw_axf;
985
986                         if (swd->sw_ictx) {
987                                 bzero(swd->sw_ictx, axf->ctxsize);
988                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
989                         }
990                         if (swd->sw_octx) {
991                                 bzero(swd->sw_octx, swd->sw_klen);
992                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
993                         }
994                         break;
995
996                 case CRYPTO_MD5:
997                 case CRYPTO_SHA1:
998                         axf = swd->sw_axf;
999
1000                         if (swd->sw_ictx) {
1001                                 bzero(swd->sw_ictx, axf->ctxsize);
1002                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
1003                         }
1004                         break;
1005
1006                 case CRYPTO_DEFLATE_COMP:
1007                         cxf = swd->sw_cxf;
1008                         break;
1009                 }
1010
1011                 //FREE(swd, M_CRYPTO_DATA);
1012                 kfree(swd, M_CRYPTO_DATA);
1013         }
1014         return 0;
1015 }
1016
1017 /*
1018  * Process a software request.
1019  */
1020 static int
1021 swcr_process(device_t dev, struct cryptop *crp, int hint)
1022 {
1023         struct cryptodesc *crd;
1024         struct swcr_data *sw;
1025         u_int32_t lid;
1026
1027         /* Sanity check */
1028         if (crp == NULL)
1029                 return EINVAL;
1030
1031         if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
1032                 crp->crp_etype = EINVAL;
1033                 goto done;
1034         }
1035
1036         lid = crp->crp_sid & 0xffffffff;
1037         if (lid >= swcr_sesnum || lid == 0 || swcr_sessions[lid] == NULL) {
1038                 crp->crp_etype = ENOENT;
1039                 goto done;
1040         }
1041
1042         /* Go through crypto descriptors, processing as we go */
1043         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1044                 /*
1045                  * Find the crypto context.
1046                  *
1047                  * XXX Note that the logic here prevents us from having
1048                  * XXX the same algorithm multiple times in a session
1049                  * XXX (or rather, we can but it won't give us the right
1050                  * XXX results). To do that, we'd need some way of differentiating
1051                  * XXX between the various instances of an algorithm (so we can
1052                  * XXX locate the correct crypto context).
1053                  */
1054                 for (sw = swcr_sessions[lid];
1055                     sw && sw->sw_alg != crd->crd_alg;
1056                     sw = sw->sw_next)
1057                         ;
1058
1059                 /* No such context ? */
1060                 if (sw == NULL) {
1061                         crp->crp_etype = EINVAL;
1062                         goto done;
1063                 }
1064                 switch (sw->sw_alg) {
1065                 case CRYPTO_DES_CBC:
1066                 case CRYPTO_3DES_CBC:
1067                 case CRYPTO_BLF_CBC:
1068                 case CRYPTO_CAST_CBC:
1069                 case CRYPTO_SKIPJACK_CBC:
1070                 case CRYPTO_RIJNDAEL128_CBC:
1071                 case CRYPTO_AES_XTS:
1072                 case CRYPTO_AES_CTR:
1073                 case CRYPTO_CAMELLIA_CBC:
1074                         if ((crp->crp_etype = swcr_encdec(crd, sw,
1075                             crp->crp_buf, crp->crp_flags)) != 0)
1076                                 goto done;
1077                         break;
1078                 case CRYPTO_NULL_CBC:
1079                         crp->crp_etype = 0;
1080                         break;
1081                 case CRYPTO_MD5_HMAC:
1082                 case CRYPTO_SHA1_HMAC:
1083                 case CRYPTO_SHA2_256_HMAC:
1084                 case CRYPTO_SHA2_384_HMAC:
1085                 case CRYPTO_SHA2_512_HMAC:
1086                 case CRYPTO_RIPEMD160_HMAC:
1087                 case CRYPTO_NULL_HMAC:
1088                 case CRYPTO_MD5_KPDK:
1089                 case CRYPTO_SHA1_KPDK:
1090                 case CRYPTO_MD5:
1091                 case CRYPTO_SHA1:
1092                         if ((crp->crp_etype = swcr_authcompute(crd, sw,
1093                             crp->crp_buf, crp->crp_flags)) != 0)
1094                                 goto done;
1095                         break;
1096
1097                 case CRYPTO_DEFLATE_COMP:
1098                         if ((crp->crp_etype = swcr_compdec(crd, sw, 
1099                             crp->crp_buf, crp->crp_flags)) != 0)
1100                                 goto done;
1101                         else
1102                                 crp->crp_olen = (int)sw->sw_size;
1103                         break;
1104
1105                 default:
1106                         /* Unknown/unsupported algorithm */
1107                         crp->crp_etype = EINVAL;
1108                         goto done;
1109                 }
1110         }
1111
1112 done:
1113         crypto_done(crp);
1114         lwkt_yield();
1115         return 0;
1116 }
1117
1118 static void
1119 swcr_identify(driver_t *drv, device_t parent)
1120 {
1121         /* NB: order 10 is so we get attached after h/w devices */
1122         /* XXX: wouldn't bet about this BUS_ADD_CHILD correctness */
1123         if (device_find_child(parent, "cryptosoft", -1) == NULL &&
1124             BUS_ADD_CHILD(parent, parent, 10, "cryptosoft", -1) == 0)
1125                 panic("cryptosoft: could not attach");
1126 }
1127
1128 static int
1129 swcr_probe(device_t dev)
1130 {
1131         device_set_desc(dev, "software crypto");
1132         return (0);
1133 }
1134
1135 static int
1136 swcr_attach(device_t dev)
1137 {
1138         memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
1139         memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
1140
1141         swcr_id = crypto_get_driverid(dev, CRYPTOCAP_F_SOFTWARE |
1142                                            CRYPTOCAP_F_SYNC |
1143                                            CRYPTOCAP_F_SMP);
1144         if (swcr_id < 0) {
1145                 device_printf(dev, "cannot initialize!");
1146                 return ENOMEM;
1147         }
1148 #define REGISTER(alg) \
1149         crypto_register(swcr_id, alg, 0,0)
1150         REGISTER(CRYPTO_DES_CBC);
1151         REGISTER(CRYPTO_3DES_CBC);
1152         REGISTER(CRYPTO_BLF_CBC);
1153         REGISTER(CRYPTO_CAST_CBC);
1154         REGISTER(CRYPTO_SKIPJACK_CBC);
1155         REGISTER(CRYPTO_NULL_CBC);
1156         REGISTER(CRYPTO_MD5_HMAC);
1157         REGISTER(CRYPTO_SHA1_HMAC);
1158         REGISTER(CRYPTO_SHA2_256_HMAC);
1159         REGISTER(CRYPTO_SHA2_384_HMAC);
1160         REGISTER(CRYPTO_SHA2_512_HMAC);
1161         REGISTER(CRYPTO_RIPEMD160_HMAC);
1162         REGISTER(CRYPTO_NULL_HMAC);
1163         REGISTER(CRYPTO_MD5_KPDK);
1164         REGISTER(CRYPTO_SHA1_KPDK);
1165         REGISTER(CRYPTO_MD5);
1166         REGISTER(CRYPTO_SHA1);
1167         REGISTER(CRYPTO_RIJNDAEL128_CBC);
1168         REGISTER(CRYPTO_AES_XTS);
1169         REGISTER(CRYPTO_AES_CTR);
1170         REGISTER(CRYPTO_CAMELLIA_CBC);
1171         REGISTER(CRYPTO_DEFLATE_COMP);
1172 #undef REGISTER
1173
1174         return 0;
1175 }
1176
1177 static int
1178 swcr_detach(device_t dev)
1179 {
1180         crypto_unregister_all(swcr_id);
1181         if (swcr_sessions != NULL)
1182                 kfree(swcr_sessions, M_CRYPTO_DATA);
1183         return 0;
1184 }
1185
1186 static device_method_t swcr_methods[] = {
1187         DEVMETHOD(device_identify,      swcr_identify),
1188         DEVMETHOD(device_probe,         swcr_probe),
1189         DEVMETHOD(device_attach,        swcr_attach),
1190         DEVMETHOD(device_detach,        swcr_detach),
1191
1192         DEVMETHOD(cryptodev_newsession, swcr_newsession),
1193         DEVMETHOD(cryptodev_freesession,swcr_freesession),
1194         DEVMETHOD(cryptodev_process,    swcr_process),
1195
1196         {0, 0},
1197 };
1198
1199 static driver_t swcr_driver = {
1200         "cryptosoft",
1201         swcr_methods,
1202         0,              /* NB: no softc */
1203 };
1204 static devclass_t swcr_devclass;
1205
1206 /*
1207  * NB: We explicitly reference the crypto module so we
1208  * get the necessary ordering when built as a loadable
1209  * module.  This is required because we bundle the crypto
1210  * module code together with the cryptosoft driver (otherwise
1211  * normal module dependencies would handle things).
1212  */
1213 extern int crypto_modevent(struct module *, int, void *);
1214 /* XXX where to attach */
1215 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,NULL);
1216 MODULE_VERSION(cryptosoft, 1);
1217 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);