sys/opencrypto/cryptosoft.c

   1 /*-
   2  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
   3  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
   4  *
   5  * This code was written by Angelos D. Keromytis in Athens, Greece, in
   6  * February 2000. Network Security Technologies Inc. (NSTI) kindly
   7  * supported the development of this code.
   8  *
   9  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  10  *
  11  * SMP modifications by Matthew Dillon for the DragonFlyBSD Project
  12  *
  13  * Permission to use, copy, and modify this software with or without fee
  14  * is hereby granted, provided that this entire notice is included in
  15  * all source code copies of any software which is or includes a copy or
  16  * modification of this software.
  17  *
  18  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  19  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  20  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  21  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  22  * PURPOSE.
  23  *
  24  * $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $
  25  * $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $
  26  */
  27
  28 #include <sys/param.h>
  29 #include <sys/systm.h>
  30 #include <sys/malloc.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/module.h>
  33 #include <sys/sysctl.h>
  34 #include <sys/errno.h>
  35 #include <sys/endian.h>
  36 #include <sys/random.h>
  37 #include <sys/kernel.h>
  38 #include <sys/uio.h>
  39 #include <sys/spinlock2.h>
  40
  41 #include <crypto/blowfish/blowfish.h>
  42 #include <crypto/sha1.h>
  43 #include <opencrypto/rmd160.h>
  44 #include <opencrypto/cast.h>
  45 #include <opencrypto/skipjack.h>
  46 #include <sys/md5.h>
  47
  48 #include <opencrypto/cryptodev.h>
  49 #include <opencrypto/cryptosoft.h>
  50 #include <opencrypto/xform.h>
  51
  52 #include <sys/kobj.h>
  53 #include <sys/bus.h>
  54 #include "cryptodev_if.h"
  55
  56 static  int32_t swcr_id;
  57 static  struct swcr_data **swcr_sessions = NULL;
  58 static  u_int32_t swcr_sesnum;
  59 static  u_int32_t swcr_minsesnum = 1;
  60
  61 static struct spinlock swcr_spin = SPINLOCK_INITIALIZER(swcr_spin);
  62
  63 u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
  64 u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
  65
  66 static  int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  67 static  int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  68 static  int swcr_combined(struct cryptop *);
  69 static  int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  70 static  int swcr_freesession(device_t dev, u_int64_t tid);
  71 static  int swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid);
  72
  73 /*
  74  * Apply a symmetric encryption/decryption algorithm.
  75  */
  76 static int
  77 swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
  78     int flags)
  79 {
  80         unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
  81         unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN];
  82         u_int8_t *kschedule;
  83         u_int8_t *okschedule;
  84         struct enc_xform *exf;
  85         int i, k, j, blks, ivlen;
  86         int error;
  87         int explicit_kschedule;
  88
  89         exf = sw->sw_exf;
  90         blks = exf->blocksize;
  91         ivlen = exf->ivsize;
  92
  93         /* Check for non-padded data */
  94         if (crd->crd_len % blks)
  95                 return EINVAL;
  96
  97         /* Initialize the IV */
  98         if (crd->crd_flags & CRD_F_ENCRYPT) {
  99                 /* IV explicitly provided ? */
 100                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
 101                         bcopy(crd->crd_iv, iv, ivlen);
 102                 else
 103                         karc4rand(iv, ivlen);
 104
 105                 /* Do we need to write the IV */
 106                 if (!(crd->crd_flags & CRD_F_IV_PRESENT))
 107                         crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
 108
 109         } else {        /* Decryption */
 110                         /* IV explicitly provided ? */
 111                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
 112                         bcopy(crd->crd_iv, iv, ivlen);
 113                 else {
 114                         /* Get IV off buf */
 115                         crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
 116                 }
 117         }
 118
 119         ivp = iv;
 120
 121         /*
 122          * The semantics are seriously broken because the session key
 123          * storage was never designed for concurrent ops.
 124          */
 125         if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
 126                 kschedule = NULL;
 127                 explicit_kschedule = 1;
 128                 error = exf->setkey(&kschedule,
 129                                     crd->crd_key, crd->crd_klen / 8);
 130                 if (error)
 131                         goto done;
 132         } else {
 133                 spin_lock(&swcr_spin);
 134                 kschedule = sw->sw_kschedule;
 135                 ++sw->sw_kschedule_refs;
 136                 spin_unlock(&swcr_spin);
 137                 explicit_kschedule = 0;
 138         }
 139
 140         /*
 141          * xforms that provide a reinit method perform all IV
 142          * handling themselves.
 143          */
 144         if (exf->reinit)
 145                 exf->reinit(kschedule, iv);
 146
 147         if (flags & CRYPTO_F_IMBUF) {
 148                 struct mbuf *m = (struct mbuf *) buf;
 149
 150                 /* Find beginning of data */
 151                 m = m_getptr(m, crd->crd_skip, &k);
 152                 if (m == NULL) {
 153                         error = EINVAL;
 154                         goto done;
 155                 }
 156
 157                 i = crd->crd_len;
 158
 159                 while (i > 0) {
 160                         /*
 161                          * If there's insufficient data at the end of
 162                          * an mbuf, we have to do some copying.
 163                          */
 164                         if (m->m_len < k + blks && m->m_len != k) {
 165                                 m_copydata(m, k, blks, blk);
 166
 167                                 /* Actual encryption/decryption */
 168                                 if (exf->reinit) {
 169                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 170                                                 exf->encrypt(kschedule,
 171                                                     blk, iv);
 172                                         } else {
 173                                                 exf->decrypt(kschedule,
 174                                                     blk, iv);
 175                                         }
 176                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 177                                         /* XOR with previous block */
 178                                         for (j = 0; j < blks; j++)
 179                                                 blk[j] ^= ivp[j];
 180
 181                                         exf->encrypt(kschedule, blk, iv);
 182
 183                                         /*
 184                                          * Keep encrypted block for XOR'ing
 185                                          * with next block
 186                                          */
 187                                         bcopy(blk, iv, blks);
 188                                         ivp = iv;
 189                                 } else {        /* decrypt */
 190                                         /*
 191                                          * Keep encrypted block for XOR'ing
 192                                          * with next block
 193                                          */
 194                                         nivp = (ivp == iv) ? iv2 : iv;
 195                                         bcopy(blk, nivp, blks);
 196
 197                                         exf->decrypt(kschedule, blk, iv);
 198
 199                                         /* XOR with previous block */
 200                                         for (j = 0; j < blks; j++)
 201                                                 blk[j] ^= ivp[j];
 202
 203                                         ivp = nivp;
 204                                 }
 205
 206                                 /* Copy back decrypted block */
 207                                 m_copyback(m, k, blks, blk);
 208
 209                                 /* Advance pointer */
 210                                 m = m_getptr(m, k + blks, &k);
 211                                 if (m == NULL) {
 212                                         error = EINVAL;
 213                                         goto done;
 214                                 }
 215
 216                                 i -= blks;
 217
 218                                 /* Could be done... */
 219                                 if (i == 0)
 220                                         break;
 221                         }
 222
 223                         /* Skip possibly empty mbufs */
 224                         if (k == m->m_len) {
 225                                 for (m = m->m_next; m && m->m_len == 0;
 226                                     m = m->m_next)
 227                                         ;
 228                                 k = 0;
 229                         }
 230
 231                         /* Sanity check */
 232                         if (m == NULL) {
 233                                 error = EINVAL;
 234                                 goto done;
 235                         }
 236
 237                         /*
 238                          * Warning: idat may point to garbage here, but
 239                          * we only use it in the while() loop, only if
 240                          * there are indeed enough data.
 241                          */
 242                         idat = mtod(m, unsigned char *) + k;
 243
 244                         while (m->m_len >= k + blks && i > 0) {
 245                                 if (exf->reinit) {
 246                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 247                                                 exf->encrypt(kschedule,
 248                                                     idat, iv);
 249                                         } else {
 250                                                 exf->decrypt(kschedule,
 251                                                     idat, iv);
 252                                         }
 253                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 254                                         /* XOR with previous block/IV */
 255                                         for (j = 0; j < blks; j++)
 256                                                 idat[j] ^= ivp[j];
 257
 258                                         exf->encrypt(kschedule, idat, iv);
 259                                         ivp = idat;
 260                                 } else {        /* decrypt */
 261                                         /*
 262                                          * Keep encrypted block to be used
 263                                          * in next block's processing.
 264                                          */
 265                                         nivp = (ivp == iv) ? iv2 : iv;
 266                                         bcopy(idat, nivp, blks);
 267
 268                                         exf->decrypt(kschedule, idat, iv);
 269
 270                                         /* XOR with previous block/IV */
 271                                         for (j = 0; j < blks; j++)
 272                                                 idat[j] ^= ivp[j];
 273
 274                                         ivp = nivp;
 275                                 }
 276
 277                                 idat += blks;
 278                                 k += blks;
 279                                 i -= blks;
 280                         }
 281                 }
 282                 error = 0;      /* Done with mbuf encryption/decryption */
 283         } else if (flags & CRYPTO_F_IOV) {
 284                 struct uio *uio = (struct uio *) buf;
 285                 struct iovec *iov;
 286
 287                 /* Find beginning of data */
 288                 iov = cuio_getptr(uio, crd->crd_skip, &k);
 289                 if (iov == NULL) {
 290                         error = EINVAL;
 291                         goto done;
 292                 }
 293
 294                 i = crd->crd_len;
 295
 296                 while (i > 0) {
 297                         /*
 298                          * If there's insufficient data at the end of
 299                          * an iovec, we have to do some copying.
 300                          */
 301                         if (iov->iov_len < k + blks && iov->iov_len != k) {
 302                                 cuio_copydata(uio, k, blks, blk);
 303
 304                                 /* Actual encryption/decryption */
 305                                 if (exf->reinit) {
 306                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 307                                                 exf->encrypt(kschedule,
 308                                                     blk, iv);
 309                                         } else {
 310                                                 exf->decrypt(kschedule,
 311                                                     blk, iv);
 312                                         }
 313                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 314                                         /* XOR with previous block */
 315                                         for (j = 0; j < blks; j++)
 316                                                 blk[j] ^= ivp[j];
 317
 318                                         exf->encrypt(kschedule, blk, iv);
 319
 320                                         /*
 321                                          * Keep encrypted block for XOR'ing
 322                                          * with next block
 323                                          */
 324                                         bcopy(blk, iv, blks);
 325                                         ivp = iv;
 326                                 } else {        /* decrypt */
 327                                         /*
 328                                          * Keep encrypted block for XOR'ing
 329                                          * with next block
 330                                          */
 331                                         nivp = (ivp == iv) ? iv2 : iv;
 332                                         bcopy(blk, nivp, blks);
 333
 334                                         exf->decrypt(kschedule, blk, iv);
 335
 336                                         /* XOR with previous block */
 337                                         for (j = 0; j < blks; j++)
 338                                                 blk[j] ^= ivp[j];
 339
 340                                         ivp = nivp;
 341                                 }
 342
 343                                 /* Copy back decrypted block */
 344                                 cuio_copyback(uio, k, blks, blk);
 345
 346                                 /* Advance pointer */
 347                                 iov = cuio_getptr(uio, k + blks, &k);
 348                                 if (iov == NULL) {
 349                                         error = EINVAL;
 350                                         goto done;
 351                                 }
 352
 353                                 i -= blks;
 354
 355                                 /* Could be done... */
 356                                 if (i == 0)
 357                                         break;
 358                         }
 359
 360                         /*
 361                          * Warning: idat may point to garbage here, but
 362                          * we only use it in the while() loop, only if
 363                          * there are indeed enough data.
 364                          */
 365                         idat = (char *)iov->iov_base + k;
 366
 367                         while (iov->iov_len >= k + blks && i > 0) {
 368                                 if (exf->reinit) {
 369                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 370                                                 exf->encrypt(kschedule,
 371                                                     idat, iv);
 372                                         } else {
 373                                                 exf->decrypt(kschedule,
 374                                                     idat, iv);
 375                                         }
 376                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 377                                         /* XOR with previous block/IV */
 378                                         for (j = 0; j < blks; j++)
 379                                                 idat[j] ^= ivp[j];
 380
 381                                         exf->encrypt(kschedule, idat, iv);
 382                                         ivp = idat;
 383                                 } else {        /* decrypt */
 384                                         /*
 385                                          * Keep encrypted block to be used
 386                                          * in next block's processing.
 387                                          */
 388                                         nivp = (ivp == iv) ? iv2 : iv;
 389                                         bcopy(idat, nivp, blks);
 390
 391                                         exf->decrypt(kschedule, idat, iv);
 392
 393                                         /* XOR with previous block/IV */
 394                                         for (j = 0; j < blks; j++)
 395                                                 idat[j] ^= ivp[j];
 396
 397                                         ivp = nivp;
 398                                 }
 399
 400                                 idat += blks;
 401                                 k += blks;
 402                                 i -= blks;
 403                         }
 404                         if (k == iov->iov_len) {
 405                                 iov++;
 406                                 k = 0;
 407                         }
 408                 }
 409                 error = 0;      /* Done with iovec encryption/decryption */
 410         } else {
 411                 /*
 412                  * contiguous buffer
 413                  */
 414                 if (exf->reinit) {
 415                         for(i = crd->crd_skip;
 416                             i < crd->crd_skip + crd->crd_len; i += blks) {
 417                                 if (crd->crd_flags & CRD_F_ENCRYPT) {
 418                                         exf->encrypt(kschedule, buf + i, iv);
 419                                 } else {
 420                                         exf->decrypt(kschedule, buf + i, iv);
 421                                 }
 422                         }
 423                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 424                         for (i = crd->crd_skip;
 425                             i < crd->crd_skip + crd->crd_len; i += blks) {
 426                                 /* XOR with the IV/previous block, as appropriate. */
 427                                 if (i == crd->crd_skip)
 428                                         for (k = 0; k < blks; k++)
 429                                                 buf[i + k] ^= ivp[k];
 430                                 else
 431                                         for (k = 0; k < blks; k++)
 432                                                 buf[i + k] ^= buf[i + k - blks];
 433                                 exf->encrypt(kschedule, buf + i, iv);
 434                         }
 435                 } else {                /* Decrypt */
 436                         /*
 437                          * Start at the end, so we don't need to keep the
 438                          * encrypted block as the IV for the next block.
 439                          */
 440                         for (i = crd->crd_skip + crd->crd_len - blks;
 441                             i >= crd->crd_skip; i -= blks) {
 442                                 exf->decrypt(kschedule, buf + i, iv);
 443
 444                                 /* XOR with the IV/previous block, as appropriate */
 445                                 if (i == crd->crd_skip)
 446                                         for (k = 0; k < blks; k++)
 447                                                 buf[i + k] ^= ivp[k];
 448                                 else
 449                                         for (k = 0; k < blks; k++)
 450                                                 buf[i + k] ^= buf[i + k - blks];
 451                         }
 452                 }
 453                 error = 0; /* Done w/contiguous buffer encrypt/decrypt */
 454         }
 455 done:
 456         /*
 457          * Cleanup - explicitly replace the session key if requested
 458          *           (horrible semantics for concurrent operation)
 459          */
 460         if (explicit_kschedule) {
 461                 spin_lock(&swcr_spin);
 462                 if (sw->sw_kschedule && sw->sw_kschedule_refs == 0) {
 463                         okschedule = sw->sw_kschedule;
 464                         sw->sw_kschedule = kschedule;
 465                 } else {
 466                         okschedule = NULL;
 467                 }
 468                 spin_unlock(&swcr_spin);
 469                 if (okschedule)
 470                         exf->zerokey(&okschedule);
 471         } else {
 472                 spin_lock(&swcr_spin);
 473                 --sw->sw_kschedule_refs;
 474                 spin_unlock(&swcr_spin);
 475         }
 476         return error;
 477 }
 478
 479 static void
 480 swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
 481     int klen)
 482 {
 483         int k;
 484
 485         klen /= 8;
 486
 487         switch (axf->type) {
 488         case CRYPTO_MD5_HMAC:
 489         case CRYPTO_SHA1_HMAC:
 490         case CRYPTO_SHA2_256_HMAC:
 491         case CRYPTO_SHA2_384_HMAC:
 492         case CRYPTO_SHA2_512_HMAC:
 493         case CRYPTO_NULL_HMAC:
 494         case CRYPTO_RIPEMD160_HMAC:
 495                 for (k = 0; k < klen; k++)
 496                         key[k] ^= HMAC_IPAD_VAL;
 497
 498                 axf->Init(sw->sw_ictx);
 499                 axf->Update(sw->sw_ictx, key, klen);
 500                 axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
 501
 502                 for (k = 0; k < klen; k++)
 503                         key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 504
 505                 axf->Init(sw->sw_octx);
 506                 axf->Update(sw->sw_octx, key, klen);
 507                 axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
 508
 509                 for (k = 0; k < klen; k++)
 510                         key[k] ^= HMAC_OPAD_VAL;
 511                 break;
 512         case CRYPTO_MD5_KPDK:
 513         case CRYPTO_SHA1_KPDK:
 514         {
 515                 /* We need a buffer that can hold an md5 and a sha1 result. */
 516                 u_char buf[SHA1_RESULTLEN];
 517
 518                 sw->sw_klen = klen;
 519                 bcopy(key, sw->sw_octx, klen);
 520                 axf->Init(sw->sw_ictx);
 521                 axf->Update(sw->sw_ictx, key, klen);
 522                 axf->Final(buf, sw->sw_ictx);
 523                 break;
 524         }
 525         default:
 526                 kprintf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
 527                     "doesn't use keys.\n", __func__, axf->type);
 528         }
 529 }
 530
 531 /*
 532  * Compute keyed-hash authenticator.
 533  */
 534 static int
 535 swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
 536     int flags)
 537 {
 538         unsigned char aalg[HASH_MAX_LEN];
 539         struct auth_hash *axf;
 540         union authctx ctx;
 541         int err;
 542
 543         if (sw->sw_ictx == 0)
 544                 return EINVAL;
 545
 546         axf = sw->sw_axf;
 547
 548         if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
 549                 swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
 550
 551         bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
 552
 553         err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
 554             (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
 555         if (err)
 556                 return err;
 557
 558         switch (sw->sw_alg) {
 559         case CRYPTO_MD5_HMAC:
 560         case CRYPTO_SHA1_HMAC:
 561         case CRYPTO_SHA2_256_HMAC:
 562         case CRYPTO_SHA2_384_HMAC:
 563         case CRYPTO_SHA2_512_HMAC:
 564         case CRYPTO_RIPEMD160_HMAC:
 565                 if (sw->sw_octx == NULL)
 566                         return EINVAL;
 567
 568                 axf->Final(aalg, &ctx);
 569                 bcopy(sw->sw_octx, &ctx, axf->ctxsize);
 570                 axf->Update(&ctx, aalg, axf->hashsize);
 571                 axf->Final(aalg, &ctx);
 572                 break;
 573
 574         case CRYPTO_MD5_KPDK:
 575         case CRYPTO_SHA1_KPDK:
 576                 if (sw->sw_octx == NULL)
 577                         return EINVAL;
 578
 579                 axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
 580                 axf->Final(aalg, &ctx);
 581                 break;
 582
 583         case CRYPTO_NULL_HMAC:
 584                 axf->Final(aalg, &ctx);
 585                 break;
 586         }
 587
 588         /* Inject the authentication data */
 589         crypto_copyback(flags, buf, crd->crd_inject,
 590             sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
 591         return 0;
 592 }
 593
 594 /*
 595  * Apply a combined encryption-authentication transformation
 596  */
 597 static int
 598 swcr_combined(struct cryptop *crp)
 599 {
 600         uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
 601         u_char *blk = (u_char *)blkbuf;
 602         u_char aalg[HASH_MAX_LEN];
 603         u_char iv[EALG_MAX_BLOCK_LEN];
 604         uint8_t *kschedule;
 605         union authctx ctx;
 606         struct cryptodesc *crd, *crda = NULL, *crde = NULL;
 607         struct swcr_data *sw, *swa, *swe;
 608         struct auth_hash *axf = NULL;
 609         struct enc_xform *exf = NULL;
 610         struct mbuf *m = NULL;
 611         struct uio *uio = NULL;
 612         caddr_t buf = (caddr_t)crp->crp_buf;
 613         uint32_t *blkp;
 614         int i, blksz, ivlen, outtype, len;
 615
 616         blksz = 0;
 617         ivlen = 0;
 618
 619         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 620                 for (sw = swcr_sessions[crp->crp_sid & 0xffffffff];
 621                      sw && sw->sw_alg != crd->crd_alg;
 622                      sw = sw->sw_next)
 623                         ;
 624                 if (sw == NULL)
 625                         return (EINVAL);
 626
 627                 switch (sw->sw_alg) {
 628                 case CRYPTO_AES_GCM_16:
 629                 case CRYPTO_AES_GMAC:
 630                         swe = sw;
 631                         crde = crd;
 632                         exf = swe->sw_exf;
 633                         ivlen = exf->ivsize;
 634                         break;
 635                 case CRYPTO_AES_128_GMAC:
 636                 case CRYPTO_AES_192_GMAC:
 637                 case CRYPTO_AES_256_GMAC:
 638                         swa = sw;
 639                         crda = crd;
 640                         axf = swa->sw_axf;
 641                         if (swa->sw_ictx == 0)
 642                                 return (EINVAL);
 643                         bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
 644                         blksz = axf->blocksize;
 645                         break;
 646                 default:
 647                         return (EINVAL);
 648                 }
 649         }
 650         if (crde == NULL || crda == NULL)
 651                 return (EINVAL);
 652
 653         if (crp->crp_flags & CRYPTO_F_IMBUF) {
 654                 outtype = CRYPTO_BUF_MBUF;
 655                 m = (struct mbuf *)buf;
 656         } else {
 657                 outtype = CRYPTO_BUF_IOV;
 658                 uio = (struct uio *)buf;
 659         }
 660
 661         /* Initialize the IV */
 662         if (crde->crd_flags & CRD_F_ENCRYPT) {
 663                 /* IV explicitly provided ? */
 664                 if (crde->crd_flags & CRD_F_IV_EXPLICIT)
 665                         bcopy(crde->crd_iv, iv, ivlen);
 666                 else
 667                         karc4rand(iv, ivlen);
 668
 669                 /* Do we need to write the IV */
 670                 if (!(crde->crd_flags & CRD_F_IV_PRESENT))
 671                         crypto_copyback(crde->crd_flags, buf, crde->crd_inject,
 672                             ivlen, iv);
 673
 674         } else {        /* Decryption */
 675                         /* IV explicitly provided ? */
 676                 if (crde->crd_flags & CRD_F_IV_EXPLICIT)
 677                         bcopy(crde->crd_iv, iv, ivlen);
 678                 else
 679                         /* Get IV off buf */
 680                         crypto_copydata(crde->crd_flags, buf, crde->crd_inject,
 681                             ivlen, iv);
 682         }
 683
 684         /* Supply MAC with IV */
 685         if (axf->Reinit)
 686                 axf->Reinit(&ctx, iv, ivlen);
 687
 688         /* Supply MAC with AAD */
 689         for (i = 0; i < crda->crd_len; i += blksz) {
 690                 len = MIN(crda->crd_len - i, blksz);
 691                 crypto_copydata(crde->crd_flags, buf, crda->crd_skip + i, len,
 692                     blk);
 693                 axf->Update(&ctx, blk, len);
 694         }
 695
 696         spin_lock(&swcr_spin);
 697         kschedule = sw->sw_kschedule;
 698         ++sw->sw_kschedule_refs;
 699         spin_unlock(&swcr_spin);
 700
 701         if (exf->reinit)
 702                 exf->reinit(kschedule, iv);
 703
 704         /* Do encryption/decryption with MAC */
 705         for (i = 0; i < crde->crd_len; i += blksz) {
 706                 len = MIN(crde->crd_len - i, blksz);
 707                 if (len < blksz)
 708                         bzero(blk, blksz);
 709                 crypto_copydata(crde->crd_flags, buf, crde->crd_skip + i, len,
 710                     blk);
 711                 if (crde->crd_flags & CRD_F_ENCRYPT) {
 712                         exf->encrypt(kschedule, blk, iv);
 713                         axf->Update(&ctx, blk, len);
 714                 } else {
 715                         axf->Update(&ctx, blk, len);
 716                         exf->decrypt(kschedule, blk, iv);
 717                 }
 718                 crypto_copyback(crde->crd_flags, buf, crde->crd_skip + i, len,
 719                     blk);
 720         }
 721
 722         /* Do any required special finalization */
 723         switch (crda->crd_alg) {
 724                 case CRYPTO_AES_128_GMAC:
 725                 case CRYPTO_AES_192_GMAC:
 726                 case CRYPTO_AES_256_GMAC:
 727                         /* length block */
 728                         bzero(blk, blksz);
 729                         blkp = (uint32_t *)blk + 1;
 730                         *blkp = htobe32(crda->crd_len * 8);
 731                         blkp = (uint32_t *)blk + 3;
 732                         *blkp = htobe32(crde->crd_len * 8);
 733                         axf->Update(&ctx, blk, blksz);
 734                         break;
 735         }
 736
 737         /* Finalize MAC */
 738         axf->Final(aalg, &ctx);
 739
 740         /* Inject the authentication data */
 741         crypto_copyback(crda->crd_flags, crp->crp_buf, crda->crd_inject,
 742             axf->blocksize, aalg);
 743
 744         spin_lock(&swcr_spin);
 745         --sw->sw_kschedule_refs;
 746         spin_unlock(&swcr_spin);
 747
 748         return (0);
 749 }
 750
 751 /*
 752  * Apply a compression/decompression algorithm
 753  */
 754 static int
 755 swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
 756              caddr_t buf, int flags)
 757 {
 758         u_int8_t *data, *out;
 759         struct comp_algo *cxf;
 760         int adj;
 761         u_int32_t result;
 762
 763         cxf = sw->sw_cxf;
 764
 765         /*
 766          * We must handle the whole buffer of data in one time
 767          * then if there is not all the data in the mbuf, we must
 768          * copy in a buffer.
 769          */
 770         data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_INTWAIT);
 771         if (data == NULL)
 772                 return (EINVAL);
 773         crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
 774
 775         if (crd->crd_flags & CRD_F_COMP)
 776                 result = cxf->compress(data, crd->crd_len, &out);
 777         else
 778                 result = cxf->decompress(data, crd->crd_len, &out);
 779
 780         kfree(data, M_CRYPTO_DATA);
 781         if (result == 0)
 782                 return EINVAL;
 783
 784         /* Copy back the (de)compressed data. m_copyback is
 785          * extending the mbuf as necessary.
 786          */
 787         sw->sw_size = result;
 788         /* Check the compressed size when doing compression */
 789         if (crd->crd_flags & CRD_F_COMP) {
 790                 if (result >= crd->crd_len) {
 791                         /* Compression was useless, we lost time */
 792                         kfree(out, M_CRYPTO_DATA);
 793                         return 0;
 794                 }
 795         }
 796
 797         crypto_copyback(flags, buf, crd->crd_skip, result, out);
 798         if (result < crd->crd_len) {
 799                 adj = result - crd->crd_len;
 800                 if (flags & CRYPTO_F_IMBUF) {
 801                         adj = result - crd->crd_len;
 802                         m_adj((struct mbuf *)buf, adj);
 803                 } else if (flags & CRYPTO_F_IOV) {
 804                         struct uio *uio = (struct uio *)buf;
 805                         int ind;
 806
 807                         adj = crd->crd_len - result;
 808                         ind = uio->uio_iovcnt - 1;
 809
 810                         while (adj > 0 && ind >= 0) {
 811                                 if (adj < uio->uio_iov[ind].iov_len) {
 812                                         uio->uio_iov[ind].iov_len -= adj;
 813                                         break;
 814                                 }
 815
 816                                 adj -= uio->uio_iov[ind].iov_len;
 817                                 uio->uio_iov[ind].iov_len = 0;
 818                                 ind--;
 819                                 uio->uio_iovcnt--;
 820                         }
 821                 }
 822         }
 823         kfree(out, M_CRYPTO_DATA);
 824         return 0;
 825 }
 826
 827 /*
 828  * Generate a new software session.
 829  */
 830 static int
 831 swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
 832 {
 833         struct swcr_data *swd_base;
 834         struct swcr_data **swd;
 835         struct swcr_data **oswd;
 836         struct auth_hash *axf;
 837         struct enc_xform *txf;
 838         struct comp_algo *cxf;
 839         u_int32_t i;
 840         u_int32_t n;
 841         int error;
 842
 843         if (sid == NULL || cri == NULL)
 844                 return EINVAL;
 845
 846         swd_base = NULL;
 847         swd = &swd_base;
 848
 849         while (cri) {
 850                 *swd = kmalloc(sizeof(struct swcr_data),
 851                                M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 852
 853                 switch (cri->cri_alg) {
 854                 case CRYPTO_DES_CBC:
 855                         txf = &enc_xform_des;
 856                         goto enccommon;
 857                 case CRYPTO_3DES_CBC:
 858                         txf = &enc_xform_3des;
 859                         goto enccommon;
 860                 case CRYPTO_BLF_CBC:
 861                         txf = &enc_xform_blf;
 862                         goto enccommon;
 863                 case CRYPTO_CAST_CBC:
 864                         txf = &enc_xform_cast5;
 865                         goto enccommon;
 866                 case CRYPTO_SKIPJACK_CBC:
 867                         txf = &enc_xform_skipjack;
 868                         goto enccommon;
 869                 case CRYPTO_RIJNDAEL128_CBC:
 870                         txf = &enc_xform_rijndael128;
 871                         goto enccommon;
 872                 case CRYPTO_AES_XTS:
 873                         txf = &enc_xform_aes_xts;
 874                         goto enccommon;
 875                 case CRYPTO_AES_CTR:
 876                         txf = &enc_xform_aes_ctr;
 877                         goto enccommon;
 878                 case CRYPTO_AES_GCM_16:
 879                         txf = &enc_xform_aes_gcm;
 880                         goto enccommon;
 881                 case CRYPTO_AES_GMAC:
 882                         txf = &enc_xform_aes_gmac;
 883                         (*swd)->sw_exf = txf;
 884                         break;
 885                 case CRYPTO_CAMELLIA_CBC:
 886                         txf = &enc_xform_camellia;
 887                         goto enccommon;
 888                 case CRYPTO_TWOFISH_CBC:
 889                         txf = &enc_xform_twofish;
 890                         goto enccommon;
 891                 case CRYPTO_NULL_CBC:
 892                         txf = &enc_xform_null;
 893                         goto enccommon;
 894                 enccommon:
 895                         if (cri->cri_key != NULL) {
 896                                 error = txf->setkey(&((*swd)->sw_kschedule),
 897                                                     cri->cri_key,
 898                                                     cri->cri_klen / 8);
 899                                 if (error) {
 900                                         swcr_freesession_slot(&swd_base, 0);
 901                                         return error;
 902                                 }
 903                         }
 904                         (*swd)->sw_exf = txf;
 905                         break;
 906
 907                 case CRYPTO_MD5_HMAC:
 908                         axf = &auth_hash_hmac_md5;
 909                         goto authcommon;
 910                 case CRYPTO_SHA1_HMAC:
 911                         axf = &auth_hash_hmac_sha1;
 912                         goto authcommon;
 913                 case CRYPTO_SHA2_256_HMAC:
 914                         axf = &auth_hash_hmac_sha2_256;
 915                         goto authcommon;
 916                 case CRYPTO_SHA2_384_HMAC:
 917                         axf = &auth_hash_hmac_sha2_384;
 918                         goto authcommon;
 919                 case CRYPTO_SHA2_512_HMAC:
 920                         axf = &auth_hash_hmac_sha2_512;
 921                         goto authcommon;
 922                 case CRYPTO_NULL_HMAC:
 923                         axf = &auth_hash_null;
 924                         goto authcommon;
 925                 case CRYPTO_RIPEMD160_HMAC:
 926                         axf = &auth_hash_hmac_ripemd_160;
 927                 authcommon:
 928                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 929                                                   M_WAITOK);
 930                         if ((*swd)->sw_ictx == NULL) {
 931                                 swcr_freesession_slot(&swd_base, 0);
 932                                 return ENOBUFS;
 933                         }
 934
 935                         (*swd)->sw_octx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 936                                                   M_WAITOK);
 937                         if ((*swd)->sw_octx == NULL) {
 938                                 swcr_freesession_slot(&swd_base, 0);
 939                                 return ENOBUFS;
 940                         }
 941
 942                         if (cri->cri_key != NULL) {
 943                                 swcr_authprepare(axf, *swd, cri->cri_key,
 944                                     cri->cri_klen);
 945                         }
 946
 947                         (*swd)->sw_mlen = cri->cri_mlen;
 948                         (*swd)->sw_axf = axf;
 949                         break;
 950
 951                 case CRYPTO_MD5_KPDK:
 952                         axf = &auth_hash_key_md5;
 953                         goto auth2common;
 954
 955                 case CRYPTO_SHA1_KPDK:
 956                         axf = &auth_hash_key_sha1;
 957                 auth2common:
 958                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 959                                                   M_WAITOK);
 960                         if ((*swd)->sw_ictx == NULL) {
 961                                 swcr_freesession_slot(&swd_base, 0);
 962                                 return ENOBUFS;
 963                         }
 964
 965                         (*swd)->sw_octx = kmalloc(cri->cri_klen / 8,
 966                                                   M_CRYPTO_DATA, M_WAITOK);
 967                         if ((*swd)->sw_octx == NULL) {
 968                                 swcr_freesession_slot(&swd_base, 0);
 969                                 return ENOBUFS;
 970                         }
 971
 972                         /* Store the key so we can "append" it to the payload */
 973                         if (cri->cri_key != NULL) {
 974                                 swcr_authprepare(axf, *swd, cri->cri_key,
 975                                     cri->cri_klen);
 976                         }
 977
 978                         (*swd)->sw_mlen = cri->cri_mlen;
 979                         (*swd)->sw_axf = axf;
 980                         break;
 981 #ifdef notdef
 982                 case CRYPTO_MD5:
 983                         axf = &auth_hash_md5;
 984                         goto auth3common;
 985
 986                 case CRYPTO_SHA1:
 987                         axf = &auth_hash_sha1;
 988                 auth3common:
 989                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 990                                                   M_WAITOK);
 991                         if ((*swd)->sw_ictx == NULL) {
 992                                 swcr_freesession_slot(&swd_base, 0);
 993                                 return ENOBUFS;
 994                         }
 995
 996                         axf->Init((*swd)->sw_ictx);
 997                         (*swd)->sw_mlen = cri->cri_mlen;
 998                         (*swd)->sw_axf = axf;
 999                         break;
1000 #endif
1001                 case CRYPTO_AES_128_GMAC:
1002                         axf = &auth_hash_gmac_aes_128;
1003                         goto auth4common;
1004
1005                 case CRYPTO_AES_192_GMAC:
1006                         axf = &auth_hash_gmac_aes_192;
1007                         goto auth4common;
1008
1009                 case CRYPTO_AES_256_GMAC:
1010                         axf = &auth_hash_gmac_aes_256;
1011                 auth4common:
1012                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
1013                             M_NOWAIT);
1014                         if ((*swd)->sw_ictx == NULL) {
1015                                 swcr_freesession_slot(&swd_base, 0);
1016                                 return ENOBUFS;
1017                         }
1018
1019                         axf->Init((*swd)->sw_ictx);
1020                         axf->Setkey((*swd)->sw_ictx, cri->cri_key,
1021                             cri->cri_klen / 8);
1022                         (*swd)->sw_axf = axf;
1023                         break;
1024
1025                 case CRYPTO_DEFLATE_COMP:
1026                         cxf = &comp_algo_deflate;
1027                         (*swd)->sw_cxf = cxf;
1028                         break;
1029                 default:
1030                         swcr_freesession_slot(&swd_base, 0);
1031                         return EINVAL;
1032                 }
1033
1034                 (*swd)->sw_alg = cri->cri_alg;
1035                 cri = cri->cri_next;
1036                 swd = &((*swd)->sw_next);
1037         }
1038
1039         for (;;) {
1040                 /*
1041                  * Atomically allocate a session
1042                  */
1043                 spin_lock(&swcr_spin);
1044                 for (i = swcr_minsesnum; i < swcr_sesnum; ++i) {
1045                         if (swcr_sessions[i] == NULL)
1046                                 break;
1047                 }
1048                 if (i < swcr_sesnum) {
1049                         swcr_sessions[i] = swd_base;
1050                         swcr_minsesnum = i + 1;
1051                         spin_unlock(&swcr_spin);
1052                         break;
1053                 }
1054                 n = swcr_sesnum;
1055                 spin_unlock(&swcr_spin);
1056
1057                 /*
1058                  * A larger allocation is required, reallocate the array
1059                  * and replace, checking for SMP races.
1060                  */
1061                 if (n < CRYPTO_SW_SESSIONS)
1062                         n = CRYPTO_SW_SESSIONS;
1063                 else
1064                         n = n * 3 / 2;
1065                 swd = kmalloc(n * sizeof(struct swcr_data *),
1066                               M_CRYPTO_DATA, M_WAITOK | M_ZERO);
1067
1068                 spin_lock(&swcr_spin);
1069                 if (swcr_sesnum >= n) {
1070                         spin_unlock(&swcr_spin);
1071                         kfree(swd, M_CRYPTO_DATA);
1072                 } else if (swcr_sesnum) {
1073                         bcopy(swcr_sessions, swd,
1074                               swcr_sesnum * sizeof(struct swcr_data *));
1075                         oswd = swcr_sessions;
1076                         swcr_sessions = swd;
1077                         swcr_sesnum = n;
1078                         spin_unlock(&swcr_spin);
1079                         kfree(oswd, M_CRYPTO_DATA);
1080                 } else {
1081                         swcr_sessions = swd;
1082                         swcr_sesnum = n;
1083                         spin_unlock(&swcr_spin);
1084                 }
1085         }
1086
1087         *sid = i;
1088         return 0;
1089 }
1090
1091 /*
1092  * Free a session.
1093  */
1094 static int
1095 swcr_freesession(device_t dev, u_int64_t tid)
1096 {
1097         u_int32_t sid = CRYPTO_SESID2LID(tid);
1098
1099         if (sid > swcr_sesnum || swcr_sessions == NULL ||
1100             swcr_sessions[sid] == NULL) {
1101                 return EINVAL;
1102         }
1103
1104         /* Silently accept and return */
1105         if (sid == 0)
1106                 return 0;
1107
1108         return(swcr_freesession_slot(&swcr_sessions[sid], sid));
1109 }
1110
1111 static
1112 int
1113 swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid)
1114 {
1115         struct enc_xform *txf;
1116         struct auth_hash *axf;
1117         struct comp_algo *cxf;
1118         struct swcr_data *swd;
1119         struct swcr_data *swnext;
1120
1121         /*
1122          * Protect session detachment with the spinlock.
1123          */
1124         spin_lock(&swcr_spin);
1125         swnext = *swdp;
1126         *swdp = NULL;
1127         if (sid && swcr_minsesnum > sid)
1128                 swcr_minsesnum = sid;
1129         spin_unlock(&swcr_spin);
1130
1131         /*
1132          * Clean up at our leisure.
1133          */
1134         while ((swd = swnext) != NULL) {
1135                 swnext = swd->sw_next;
1136
1137                 swd->sw_next = NULL;
1138
1139                 switch (swd->sw_alg) {
1140                 case CRYPTO_DES_CBC:
1141                 case CRYPTO_3DES_CBC:
1142                 case CRYPTO_BLF_CBC:
1143                 case CRYPTO_CAST_CBC:
1144                 case CRYPTO_SKIPJACK_CBC:
1145                 case CRYPTO_RIJNDAEL128_CBC:
1146                 case CRYPTO_AES_XTS:
1147                 case CRYPTO_AES_CTR:
1148                 case CRYPTO_AES_GCM_16:
1149                 case CRYPTO_AES_GMAC:
1150                 case CRYPTO_CAMELLIA_CBC:
1151                 case CRYPTO_TWOFISH_CBC:
1152                 case CRYPTO_NULL_CBC:
1153                         txf = swd->sw_exf;
1154
1155                         if (swd->sw_kschedule)
1156                                 txf->zerokey(&(swd->sw_kschedule));
1157                         break;
1158
1159                 case CRYPTO_MD5_HMAC:
1160                 case CRYPTO_SHA1_HMAC:
1161                 case CRYPTO_SHA2_256_HMAC:
1162                 case CRYPTO_SHA2_384_HMAC:
1163                 case CRYPTO_SHA2_512_HMAC:
1164                 case CRYPTO_RIPEMD160_HMAC:
1165                 case CRYPTO_NULL_HMAC:
1166                         axf = swd->sw_axf;
1167
1168                         if (swd->sw_ictx) {
1169                                 bzero(swd->sw_ictx, axf->ctxsize);
1170                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
1171                         }
1172                         if (swd->sw_octx) {
1173                                 bzero(swd->sw_octx, axf->ctxsize);
1174                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
1175                         }
1176                         break;
1177
1178                 case CRYPTO_MD5_KPDK:
1179                 case CRYPTO_SHA1_KPDK:
1180                         axf = swd->sw_axf;
1181
1182                         if (swd->sw_ictx) {
1183                                 bzero(swd->sw_ictx, axf->ctxsize);
1184                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
1185                         }
1186                         if (swd->sw_octx) {
1187                                 bzero(swd->sw_octx, swd->sw_klen);
1188                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
1189                         }
1190                         break;
1191
1192                 case CRYPTO_AES_128_GMAC:
1193                 case CRYPTO_AES_192_GMAC:
1194                 case CRYPTO_AES_256_GMAC:
1195                 case CRYPTO_MD5:
1196                 case CRYPTO_SHA1:
1197                         axf = swd->sw_axf;
1198
1199                         if (swd->sw_ictx) {
1200                                 bzero(swd->sw_ictx, axf->ctxsize);
1201                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
1202                         }
1203                         break;
1204
1205                 case CRYPTO_DEFLATE_COMP:
1206                         cxf = swd->sw_cxf;
1207                         break;
1208                 }
1209
1210                 //FREE(swd, M_CRYPTO_DATA);
1211                 kfree(swd, M_CRYPTO_DATA);
1212         }
1213         return 0;
1214 }
1215
1216 /*
1217  * Process a software request.
1218  */
1219 static int
1220 swcr_process(device_t dev, struct cryptop *crp, int hint)
1221 {
1222         struct cryptodesc *crd;
1223         struct swcr_data *sw;
1224         u_int32_t lid;
1225
1226         /* Sanity check */
1227         if (crp == NULL)
1228                 return EINVAL;
1229
1230         if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
1231                 crp->crp_etype = EINVAL;
1232                 goto done;
1233         }
1234
1235         lid = crp->crp_sid & 0xffffffff;
1236         if (lid >= swcr_sesnum || lid == 0 || swcr_sessions[lid] == NULL) {
1237                 crp->crp_etype = ENOENT;
1238                 goto done;
1239         }
1240
1241         /* Go through crypto descriptors, processing as we go */
1242         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1243                 /*
1244                  * Find the crypto context.
1245                  *
1246                  * XXX Note that the logic here prevents us from having
1247                  * XXX the same algorithm multiple times in a session
1248                  * XXX (or rather, we can but it won't give us the right
1249                  * XXX results). To do that, we'd need some way of differentiating
1250                  * XXX between the various instances of an algorithm (so we can
1251                  * XXX locate the correct crypto context).
1252                  */
1253                 for (sw = swcr_sessions[lid];
1254                     sw && sw->sw_alg != crd->crd_alg;
1255                     sw = sw->sw_next)
1256                         ;
1257
1258                 /* No such context ? */
1259                 if (sw == NULL) {
1260                         crp->crp_etype = EINVAL;
1261                         goto done;
1262                 }
1263                 switch (sw->sw_alg) {
1264                 case CRYPTO_DES_CBC:
1265                 case CRYPTO_3DES_CBC:
1266                 case CRYPTO_BLF_CBC:
1267                 case CRYPTO_CAST_CBC:
1268                 case CRYPTO_SKIPJACK_CBC:
1269                 case CRYPTO_RIJNDAEL128_CBC:
1270                 case CRYPTO_AES_XTS:
1271                 case CRYPTO_AES_CTR:
1272                 case CRYPTO_CAMELLIA_CBC:
1273                 case CRYPTO_TWOFISH_CBC:
1274                         if ((crp->crp_etype = swcr_encdec(crd, sw,
1275                             crp->crp_buf, crp->crp_flags)) != 0)
1276                                 goto done;
1277                         break;
1278                 case CRYPTO_NULL_CBC:
1279                         crp->crp_etype = 0;
1280                         break;
1281                 case CRYPTO_MD5_HMAC:
1282                 case CRYPTO_SHA1_HMAC:
1283                 case CRYPTO_SHA2_256_HMAC:
1284                 case CRYPTO_SHA2_384_HMAC:
1285                 case CRYPTO_SHA2_512_HMAC:
1286                 case CRYPTO_RIPEMD160_HMAC:
1287                 case CRYPTO_NULL_HMAC:
1288                 case CRYPTO_MD5_KPDK:
1289                 case CRYPTO_SHA1_KPDK:
1290                 case CRYPTO_MD5:
1291                 case CRYPTO_SHA1:
1292                         if ((crp->crp_etype = swcr_authcompute(crd, sw,
1293                             crp->crp_buf, crp->crp_flags)) != 0)
1294                                 goto done;
1295                         break;
1296
1297                 case CRYPTO_AES_GCM_16:
1298                 case CRYPTO_AES_GMAC:
1299                 case CRYPTO_AES_128_GMAC:
1300                 case CRYPTO_AES_192_GMAC:
1301                 case CRYPTO_AES_256_GMAC:
1302                         crp->crp_etype = swcr_combined(crp);
1303                         goto done;
1304
1305                 case CRYPTO_DEFLATE_COMP:
1306                         if ((crp->crp_etype = swcr_compdec(crd, sw,
1307                             crp->crp_buf, crp->crp_flags)) != 0)
1308                                 goto done;
1309                         else
1310                                 crp->crp_olen = (int)sw->sw_size;
1311                         break;
1312
1313                 default:
1314                         /* Unknown/unsupported algorithm */
1315                         crp->crp_etype = EINVAL;
1316                         goto done;
1317                 }
1318         }
1319
1320 done:
1321         crypto_done(crp);
1322         lwkt_yield();
1323         return 0;
1324 }
1325
1326 static void
1327 swcr_identify(driver_t *drv, device_t parent)
1328 {
1329         /* NB: order 10 is so we get attached after h/w devices */
1330         /* XXX: wouldn't bet about this BUS_ADD_CHILD correctness */
1331         if (device_find_child(parent, "cryptosoft", -1) == NULL &&
1332             BUS_ADD_CHILD(parent, parent, 10, "cryptosoft", -1) == 0)
1333                 panic("cryptosoft: could not attach");
1334 }
1335
1336 static int
1337 swcr_probe(device_t dev)
1338 {
1339         device_set_desc(dev, "software crypto");
1340         return (0);
1341 }
1342
1343 static int
1344 swcr_attach(device_t dev)
1345 {
1346         memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
1347         memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
1348
1349         swcr_id = crypto_get_driverid(dev, CRYPTOCAP_F_SOFTWARE |
1350                                            CRYPTOCAP_F_SYNC |
1351                                            CRYPTOCAP_F_SMP);
1352         if (swcr_id < 0) {
1353                 device_printf(dev, "cannot initialize!");
1354                 return ENOMEM;
1355         }
1356 #define REGISTER(alg) \
1357         crypto_register(swcr_id, alg, 0,0)
1358         REGISTER(CRYPTO_DES_CBC);
1359         REGISTER(CRYPTO_3DES_CBC);
1360         REGISTER(CRYPTO_BLF_CBC);
1361         REGISTER(CRYPTO_CAST_CBC);
1362         REGISTER(CRYPTO_SKIPJACK_CBC);
1363         REGISTER(CRYPTO_NULL_CBC);
1364         REGISTER(CRYPTO_MD5_HMAC);
1365         REGISTER(CRYPTO_SHA1_HMAC);
1366         REGISTER(CRYPTO_SHA2_256_HMAC);
1367         REGISTER(CRYPTO_SHA2_384_HMAC);
1368         REGISTER(CRYPTO_SHA2_512_HMAC);
1369         REGISTER(CRYPTO_RIPEMD160_HMAC);
1370         REGISTER(CRYPTO_NULL_HMAC);
1371         REGISTER(CRYPTO_MD5_KPDK);
1372         REGISTER(CRYPTO_SHA1_KPDK);
1373         REGISTER(CRYPTO_MD5);
1374         REGISTER(CRYPTO_SHA1);
1375         REGISTER(CRYPTO_RIJNDAEL128_CBC);
1376         REGISTER(CRYPTO_AES_XTS);
1377         REGISTER(CRYPTO_AES_CTR);
1378         REGISTER(CRYPTO_AES_GCM_16);
1379         REGISTER(CRYPTO_AES_GMAC);
1380         REGISTER(CRYPTO_AES_128_GMAC);
1381         REGISTER(CRYPTO_AES_192_GMAC);
1382         REGISTER(CRYPTO_AES_256_GMAC);
1383         REGISTER(CRYPTO_CAMELLIA_CBC);
1384         REGISTER(CRYPTO_TWOFISH_CBC);
1385         REGISTER(CRYPTO_DEFLATE_COMP);
1386 #undef REGISTER
1387
1388         return 0;
1389 }
1390
1391 static int
1392 swcr_detach(device_t dev)
1393 {
1394         crypto_unregister_all(swcr_id);
1395         if (swcr_sessions != NULL)
1396                 kfree(swcr_sessions, M_CRYPTO_DATA);
1397         return 0;
1398 }
1399
1400 static device_method_t swcr_methods[] = {
1401         DEVMETHOD(device_identify,      swcr_identify),
1402         DEVMETHOD(device_probe,         swcr_probe),
1403         DEVMETHOD(device_attach,        swcr_attach),
1404         DEVMETHOD(device_detach,        swcr_detach),
1405
1406         DEVMETHOD(cryptodev_newsession, swcr_newsession),
1407         DEVMETHOD(cryptodev_freesession,swcr_freesession),
1408         DEVMETHOD(cryptodev_process,    swcr_process),
1409
1410         {0, 0},
1411 };
1412
1413 static driver_t swcr_driver = {
1414         "cryptosoft",
1415         swcr_methods,
1416         0,              /* NB: no softc */
1417 };
1418 static devclass_t swcr_devclass;
1419
1420 /*
1421  * NB: We explicitly reference the crypto module so we
1422  * get the necessary ordering when built as a loadable
1423  * module.  This is required because we bundle the crypto
1424  * module code together with the cryptosoft driver (otherwise
1425  * normal module dependencies would handle things).
1426  */
1427 extern int crypto_modevent(struct module *, int, void *);
1428 /* XXX where to attach */
1429 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,NULL);
1430 MODULE_VERSION(cryptosoft, 1);
1431 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);