sys/opencrypto/cryptosoft.c

   1 /*-
   2  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
   3  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
   4  *
   5  * This code was written by Angelos D. Keromytis in Athens, Greece, in
   6  * February 2000. Network Security Technologies Inc. (NSTI) kindly
   7  * supported the development of this code.
   8  *
   9  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  10  *
  11  * SMP modifications by Matthew Dillon for the DragonFlyBSD Project
  12  *
  13  * Permission to use, copy, and modify this software with or without fee
  14  * is hereby granted, provided that this entire notice is included in
  15  * all source code copies of any software which is or includes a copy or
  16  * modification of this software.
  17  *
  18  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  19  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  20  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  21  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  22  * PURPOSE.
  23  *
  24  * $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $
  25  * $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $
  26  */
  27
  28 #include <sys/param.h>
  29 #include <sys/systm.h>
  30 #include <sys/malloc.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/module.h>
  33 #include <sys/sysctl.h>
  34 #include <sys/errno.h>
  35 #include <sys/random.h>
  36 #include <sys/kernel.h>
  37 #include <sys/uio.h>
  38 #include <sys/spinlock2.h>
  39
  40 #include <crypto/blowfish/blowfish.h>
  41 #include <crypto/sha1.h>
  42 #include <opencrypto/rmd160.h>
  43 #include <opencrypto/cast.h>
  44 #include <opencrypto/skipjack.h>
  45 #include <sys/md5.h>
  46
  47 #include <opencrypto/cryptodev.h>
  48 #include <opencrypto/cryptosoft.h>
  49 #include <opencrypto/xform.h>
  50
  51 #include <sys/kobj.h>
  52 #include <sys/bus.h>
  53 #include "cryptodev_if.h"
  54
  55 static  int32_t swcr_id;
  56 static  struct swcr_data **swcr_sessions = NULL;
  57 static  u_int32_t swcr_sesnum;
  58 static  u_int32_t swcr_minsesnum = 1;
  59
  60 static struct spinlock swcr_spin = SPINLOCK_INITIALIZER(swcr_spin);
  61
  62 u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
  63 u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
  64
  65 static  int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  66 static  int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  67 static  int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  68 static  int swcr_freesession(device_t dev, u_int64_t tid);
  69 static  int swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid);
  70
  71 /*
  72  * Apply a symmetric encryption/decryption algorithm.
  73  */
  74 static int
  75 swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
  76     int flags)
  77 {
  78         unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
  79         unsigned char *ivp, piv[EALG_MAX_BLOCK_LEN];
  80         u_int8_t *kschedule;
  81         u_int8_t *okschedule;
  82         struct enc_xform *exf;
  83         int i, k, j, blks, ivlen;
  84         int error;
  85         int explicit_kschedule;
  86
  87         exf = sw->sw_exf;
  88         blks = exf->blocksize;
  89         ivlen = exf->ivsize;
  90
  91         /* Check for non-padded data */
  92         if (crd->crd_len % blks)
  93                 return EINVAL;
  94
  95         /* Initialize the IV */
  96         if (crd->crd_flags & CRD_F_ENCRYPT) {
  97                 /* IV explicitly provided ? */
  98                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
  99                         bcopy(crd->crd_iv, iv, ivlen);
 100                 else
 101                         karc4rand(iv, ivlen);
 102
 103                 /* Do we need to write the IV */
 104                 if (!(crd->crd_flags & CRD_F_IV_PRESENT))
 105                         crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
 106
 107         } else {        /* Decryption */
 108                         /* IV explicitly provided ? */
 109                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
 110                         bcopy(crd->crd_iv, iv, ivlen);
 111                 else {
 112                         /* Get IV off buf */
 113                         crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
 114                 }
 115         }
 116
 117         ivp = iv;
 118
 119         /*
 120          * The semantics are seriously broken because the session key
 121          * storage was never designed for concurrent ops.
 122          */
 123         if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
 124                 kschedule = NULL;
 125                 explicit_kschedule = 1;
 126                 error = exf->setkey(&kschedule,
 127                                     crd->crd_key, crd->crd_klen / 8);
 128                 if (error)
 129                         goto done;
 130         } else {
 131                 spin_lock(&swcr_spin);
 132                 kschedule = sw->sw_kschedule;
 133                 ++sw->sw_kschedule_refs;
 134                 spin_unlock(&swcr_spin);
 135                 explicit_kschedule = 0;
 136         }
 137
 138         /*
 139          * xforms that provide a reinit method perform all IV
 140          * handling themselves.
 141          */
 142         if (exf->reinit)
 143                 exf->reinit(kschedule, iv);
 144
 145         if (flags & CRYPTO_F_IMBUF) {
 146                 struct mbuf *m = (struct mbuf *) buf;
 147
 148                 /* Find beginning of data */
 149                 m = m_getptr(m, crd->crd_skip, &k);
 150                 if (m == NULL) {
 151                         error = EINVAL;
 152                         goto done;
 153                 }
 154
 155                 i = crd->crd_len;
 156
 157                 while (i > 0) {
 158                         /*
 159                          * If there's insufficient data at the end of
 160                          * an mbuf, we have to do some copying.
 161                          */
 162                         if (m->m_len < k + blks && m->m_len != k) {
 163                                 m_copydata(m, k, blks, blk);
 164
 165                                 /* Actual encryption/decryption */
 166                                 if (exf->reinit) {
 167                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 168                                                 exf->encrypt(kschedule,
 169                                                     blk, iv);
 170                                         } else {
 171                                                 exf->decrypt(kschedule,
 172                                                     blk, iv);
 173                                         }
 174                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 175                                         /* XOR with previous block */
 176                                         for (j = 0; j < blks; j++)
 177                                                 blk[j] ^= ivp[j];
 178
 179                                         exf->encrypt(kschedule, blk, iv);
 180
 181                                         /*
 182                                          * Keep encrypted block for XOR'ing
 183                                          * with next block
 184                                          */
 185                                         bcopy(blk, iv, blks);
 186                                         ivp = iv;
 187                                 } else {        /* decrypt */
 188                                         /*
 189                                          * Keep encrypted block for XOR'ing
 190                                          * with next block
 191                                          */
 192                                         if (ivp == iv)
 193                                                 bcopy(blk, piv, blks);
 194                                         else
 195                                                 bcopy(blk, iv, blks);
 196
 197                                         exf->decrypt(kschedule, blk, iv);
 198
 199                                         /* XOR with previous block */
 200                                         for (j = 0; j < blks; j++)
 201                                                 blk[j] ^= ivp[j];
 202
 203                                         if (ivp == iv)
 204                                                 bcopy(piv, iv, blks);
 205                                         else
 206                                                 ivp = iv;
 207                                 }
 208
 209                                 /* Copy back decrypted block */
 210                                 m_copyback(m, k, blks, blk);
 211
 212                                 /* Advance pointer */
 213                                 m = m_getptr(m, k + blks, &k);
 214                                 if (m == NULL) {
 215                                         error = EINVAL;
 216                                         goto done;
 217                                 }
 218
 219                                 i -= blks;
 220
 221                                 /* Could be done... */
 222                                 if (i == 0)
 223                                         break;
 224                         }
 225
 226                         /* Skip possibly empty mbufs */
 227                         if (k == m->m_len) {
 228                                 for (m = m->m_next; m && m->m_len == 0;
 229                                     m = m->m_next)
 230                                         ;
 231                                 k = 0;
 232                         }
 233
 234                         /* Sanity check */
 235                         if (m == NULL) {
 236                                 error = EINVAL;
 237                                 goto done;
 238                         }
 239
 240                         /*
 241                          * Warning: idat may point to garbage here, but
 242                          * we only use it in the while() loop, only if
 243                          * there are indeed enough data.
 244                          */
 245                         idat = mtod(m, unsigned char *) + k;
 246
 247                         while (m->m_len >= k + blks && i > 0) {
 248                                 if (exf->reinit) {
 249                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 250                                                 exf->encrypt(kschedule,
 251                                                     idat, iv);
 252                                         } else {
 253                                                 exf->decrypt(kschedule,
 254                                                     idat, iv);
 255                                         }
 256                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 257                                         /* XOR with previous block/IV */
 258                                         for (j = 0; j < blks; j++)
 259                                                 idat[j] ^= ivp[j];
 260
 261                                         exf->encrypt(kschedule, idat, iv);
 262                                         ivp = idat;
 263                                 } else {        /* decrypt */
 264                                         /*
 265                                          * Keep encrypted block to be used
 266                                          * in next block's processing.
 267                                          */
 268                                         if (ivp == iv)
 269                                                 bcopy(idat, piv, blks);
 270                                         else
 271                                                 bcopy(idat, iv, blks);
 272
 273                                         exf->decrypt(kschedule, idat, iv);
 274
 275                                         /* XOR with previous block/IV */
 276                                         for (j = 0; j < blks; j++)
 277                                                 idat[j] ^= ivp[j];
 278
 279                                         if (ivp == iv)
 280                                                 bcopy(piv, iv, blks);
 281                                         else
 282                                                 ivp = iv;
 283                                 }
 284
 285                                 idat += blks;
 286                                 k += blks;
 287                                 i -= blks;
 288                         }
 289                 }
 290                 error = 0;      /* Done with mbuf encryption/decryption */
 291         } else if (flags & CRYPTO_F_IOV) {
 292                 struct uio *uio = (struct uio *) buf;
 293                 struct iovec *iov;
 294
 295                 /* Find beginning of data */
 296                 iov = cuio_getptr(uio, crd->crd_skip, &k);
 297                 if (iov == NULL) {
 298                         error = EINVAL;
 299                         goto done;
 300                 }
 301
 302                 i = crd->crd_len;
 303
 304                 while (i > 0) {
 305                         /*
 306                          * If there's insufficient data at the end of
 307                          * an iovec, we have to do some copying.
 308                          */
 309                         if (iov->iov_len < k + blks && iov->iov_len != k) {
 310                                 cuio_copydata(uio, k, blks, blk);
 311
 312                                 /* Actual encryption/decryption */
 313                                 if (exf->reinit) {
 314                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 315                                                 exf->encrypt(kschedule,
 316                                                     blk, iv);
 317                                         } else {
 318                                                 exf->decrypt(kschedule,
 319                                                     blk, iv);
 320                                         }
 321                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 322                                         /* XOR with previous block */
 323                                         for (j = 0; j < blks; j++)
 324                                                 blk[j] ^= ivp[j];
 325
 326                                         exf->encrypt(kschedule, blk, iv);
 327
 328                                         /*
 329                                          * Keep encrypted block for XOR'ing
 330                                          * with next block
 331                                          */
 332                                         bcopy(blk, iv, blks);
 333                                         ivp = iv;
 334                                 } else {        /* decrypt */
 335                                         /*
 336                                          * Keep encrypted block for XOR'ing
 337                                          * with next block
 338                                          */
 339                                         if (ivp == iv)
 340                                                 bcopy(blk, piv, blks);
 341                                         else
 342                                                 bcopy(blk, iv, blks);
 343
 344                                         exf->decrypt(kschedule, blk, iv);
 345
 346                                         /* XOR with previous block */
 347                                         for (j = 0; j < blks; j++)
 348                                                 blk[j] ^= ivp[j];
 349
 350                                         if (ivp == iv)
 351                                                 bcopy(piv, iv, blks);
 352                                         else
 353                                                 ivp = iv;
 354                                 }
 355
 356                                 /* Copy back decrypted block */
 357                                 cuio_copyback(uio, k, blks, blk);
 358
 359                                 /* Advance pointer */
 360                                 iov = cuio_getptr(uio, k + blks, &k);
 361                                 if (iov == NULL) {
 362                                         error = EINVAL;
 363                                         goto done;
 364                                 }
 365
 366                                 i -= blks;
 367
 368                                 /* Could be done... */
 369                                 if (i == 0)
 370                                         break;
 371                         }
 372
 373                         /*
 374                          * Warning: idat may point to garbage here, but
 375                          * we only use it in the while() loop, only if
 376                          * there are indeed enough data.
 377                          */
 378                         idat = (char *)iov->iov_base + k;
 379
 380                         while (iov->iov_len >= k + blks && i > 0) {
 381                                 if (exf->reinit) {
 382                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 383                                                 exf->encrypt(kschedule,
 384                                                     idat, iv);
 385                                         } else {
 386                                                 exf->decrypt(kschedule,
 387                                                     idat, iv);
 388                                         }
 389                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 390                                         /* XOR with previous block/IV */
 391                                         for (j = 0; j < blks; j++)
 392                                                 idat[j] ^= ivp[j];
 393
 394                                         exf->encrypt(kschedule, idat, iv);
 395                                         ivp = idat;
 396                                 } else {        /* decrypt */
 397                                         /*
 398                                          * Keep encrypted block to be used
 399                                          * in next block's processing.
 400                                          */
 401                                         if (ivp == iv)
 402                                                 bcopy(idat, piv, blks);
 403                                         else
 404                                                 bcopy(idat, iv, blks);
 405
 406                                         exf->decrypt(kschedule, idat, iv);
 407
 408                                         /* XOR with previous block/IV */
 409                                         for (j = 0; j < blks; j++)
 410                                                 idat[j] ^= ivp[j];
 411
 412                                         if (ivp == iv)
 413                                                 bcopy(piv, iv, blks);
 414                                         else
 415                                                 ivp = iv;
 416                                 }
 417
 418                                 idat += blks;
 419                                 k += blks;
 420                                 i -= blks;
 421                         }
 422                         if (k == iov->iov_len) {
 423                                 iov++;
 424                                 k = 0;
 425                         }
 426                 }
 427                 error = 0;      /* Done with iovec encryption/decryption */
 428         } else {
 429                 /*
 430                  * contiguous buffer
 431                  */
 432                 if (exf->reinit) {
 433                         for(i = crd->crd_skip;
 434                             i < crd->crd_skip + crd->crd_len; i += blks) {
 435                                 if (crd->crd_flags & CRD_F_ENCRYPT) {
 436                                         exf->encrypt(kschedule, buf + i, iv);
 437                                 } else {
 438                                         exf->decrypt(kschedule, buf + i, iv);
 439                                 }
 440                         }
 441                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 442                         for (i = crd->crd_skip;
 443                             i < crd->crd_skip + crd->crd_len; i += blks) {
 444                                 /* XOR with the IV/previous block, as appropriate. */
 445                                 if (i == crd->crd_skip)
 446                                         for (k = 0; k < blks; k++)
 447                                                 buf[i + k] ^= ivp[k];
 448                                 else
 449                                         for (k = 0; k < blks; k++)
 450                                                 buf[i + k] ^= buf[i + k - blks];
 451                                 exf->encrypt(kschedule, buf + i, iv);
 452                         }
 453                 } else {                /* Decrypt */
 454                         /*
 455                          * Start at the end, so we don't need to keep the
 456                          * encrypted block as the IV for the next block.
 457                          */
 458                         for (i = crd->crd_skip + crd->crd_len - blks;
 459                             i >= crd->crd_skip; i -= blks) {
 460                                 exf->decrypt(kschedule, buf + i, iv);
 461
 462                                 /* XOR with the IV/previous block, as appropriate */
 463                                 if (i == crd->crd_skip)
 464                                         for (k = 0; k < blks; k++)
 465                                                 buf[i + k] ^= ivp[k];
 466                                 else
 467                                         for (k = 0; k < blks; k++)
 468                                                 buf[i + k] ^= buf[i + k - blks];
 469                         }
 470                 }
 471                 error = 0; /* Done w/contiguous buffer encrypt/decrypt */
 472         }
 473 done:
 474         /*
 475          * Cleanup - explicitly replace the session key if requested
 476          *           (horrible semantics for concurrent operation)
 477          */
 478         if (explicit_kschedule) {
 479                 spin_lock(&swcr_spin);
 480                 if (sw->sw_kschedule && sw->sw_kschedule_refs == 0) {
 481                         okschedule = sw->sw_kschedule;
 482                         sw->sw_kschedule = kschedule;
 483                 } else {
 484                         okschedule = NULL;
 485                 }
 486                 spin_unlock(&swcr_spin);
 487                 if (okschedule)
 488                         exf->zerokey(&okschedule);
 489         } else {
 490                 spin_lock(&swcr_spin);
 491                 --sw->sw_kschedule_refs;
 492                 spin_unlock(&swcr_spin);
 493         }
 494         return error;
 495 }
 496
 497 static void
 498 swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
 499     int klen)
 500 {
 501         int k;
 502
 503         klen /= 8;
 504
 505         switch (axf->type) {
 506         case CRYPTO_MD5_HMAC:
 507         case CRYPTO_SHA1_HMAC:
 508         case CRYPTO_SHA2_256_HMAC:
 509         case CRYPTO_SHA2_384_HMAC:
 510         case CRYPTO_SHA2_512_HMAC:
 511         case CRYPTO_NULL_HMAC:
 512         case CRYPTO_RIPEMD160_HMAC:
 513                 for (k = 0; k < klen; k++)
 514                         key[k] ^= HMAC_IPAD_VAL;
 515
 516                 axf->Init(sw->sw_ictx);
 517                 axf->Update(sw->sw_ictx, key, klen);
 518                 axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
 519
 520                 for (k = 0; k < klen; k++)
 521                         key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 522
 523                 axf->Init(sw->sw_octx);
 524                 axf->Update(sw->sw_octx, key, klen);
 525                 axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
 526
 527                 for (k = 0; k < klen; k++)
 528                         key[k] ^= HMAC_OPAD_VAL;
 529                 break;
 530         case CRYPTO_MD5_KPDK:
 531         case CRYPTO_SHA1_KPDK:
 532         {
 533                 /* We need a buffer that can hold an md5 and a sha1 result. */
 534                 u_char buf[SHA1_RESULTLEN];
 535
 536                 sw->sw_klen = klen;
 537                 bcopy(key, sw->sw_octx, klen);
 538                 axf->Init(sw->sw_ictx);
 539                 axf->Update(sw->sw_ictx, key, klen);
 540                 axf->Final(buf, sw->sw_ictx);
 541                 break;
 542         }
 543         default:
 544                 kprintf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
 545                     "doesn't use keys.\n", __func__, axf->type);
 546         }
 547 }
 548
 549 /*
 550  * Compute keyed-hash authenticator.
 551  */
 552 static int
 553 swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
 554     int flags)
 555 {
 556         unsigned char aalg[HASH_MAX_LEN];
 557         struct auth_hash *axf;
 558         union authctx ctx;
 559         int err;
 560
 561         if (sw->sw_ictx == 0)
 562                 return EINVAL;
 563
 564         axf = sw->sw_axf;
 565
 566         if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
 567                 swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
 568
 569         bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
 570
 571         err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
 572             (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
 573         if (err)
 574                 return err;
 575
 576         switch (sw->sw_alg) {
 577         case CRYPTO_MD5_HMAC:
 578         case CRYPTO_SHA1_HMAC:
 579         case CRYPTO_SHA2_256_HMAC:
 580         case CRYPTO_SHA2_384_HMAC:
 581         case CRYPTO_SHA2_512_HMAC:
 582         case CRYPTO_RIPEMD160_HMAC:
 583                 if (sw->sw_octx == NULL)
 584                         return EINVAL;
 585
 586                 axf->Final(aalg, &ctx);
 587                 bcopy(sw->sw_octx, &ctx, axf->ctxsize);
 588                 axf->Update(&ctx, aalg, axf->hashsize);
 589                 axf->Final(aalg, &ctx);
 590                 break;
 591
 592         case CRYPTO_MD5_KPDK:
 593         case CRYPTO_SHA1_KPDK:
 594                 if (sw->sw_octx == NULL)
 595                         return EINVAL;
 596
 597                 axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
 598                 axf->Final(aalg, &ctx);
 599                 break;
 600
 601         case CRYPTO_NULL_HMAC:
 602                 axf->Final(aalg, &ctx);
 603                 break;
 604         }
 605
 606         /* Inject the authentication data */
 607         crypto_copyback(flags, buf, crd->crd_inject,
 608             sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
 609         return 0;
 610 }
 611
 612 /*
 613  * Apply a compression/decompression algorithm
 614  */
 615 static int
 616 swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
 617              caddr_t buf, int flags)
 618 {
 619         u_int8_t *data, *out;
 620         struct comp_algo *cxf;
 621         int adj;
 622         u_int32_t result;
 623
 624         cxf = sw->sw_cxf;
 625
 626         /*
 627          * We must handle the whole buffer of data in one time
 628          * then if there is not all the data in the mbuf, we must
 629          * copy in a buffer.
 630          */
 631         data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_INTWAIT);
 632         if (data == NULL)
 633                 return (EINVAL);
 634         crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
 635
 636         if (crd->crd_flags & CRD_F_COMP)
 637                 result = cxf->compress(data, crd->crd_len, &out);
 638         else
 639                 result = cxf->decompress(data, crd->crd_len, &out);
 640
 641         kfree(data, M_CRYPTO_DATA);
 642         if (result == 0)
 643                 return EINVAL;
 644
 645         /* Copy back the (de)compressed data. m_copyback is
 646          * extending the mbuf as necessary.
 647          */
 648         sw->sw_size = result;
 649         /* Check the compressed size when doing compression */
 650         if (crd->crd_flags & CRD_F_COMP) {
 651                 if (result >= crd->crd_len) {
 652                         /* Compression was useless, we lost time */
 653                         kfree(out, M_CRYPTO_DATA);
 654                         return 0;
 655                 }
 656         }
 657
 658         crypto_copyback(flags, buf, crd->crd_skip, result, out);
 659         if (result < crd->crd_len) {
 660                 adj = result - crd->crd_len;
 661                 if (flags & CRYPTO_F_IMBUF) {
 662                         adj = result - crd->crd_len;
 663                         m_adj((struct mbuf *)buf, adj);
 664                 } else if (flags & CRYPTO_F_IOV) {
 665                         struct uio *uio = (struct uio *)buf;
 666                         int ind;
 667
 668                         adj = crd->crd_len - result;
 669                         ind = uio->uio_iovcnt - 1;
 670
 671                         while (adj > 0 && ind >= 0) {
 672                                 if (adj < uio->uio_iov[ind].iov_len) {
 673                                         uio->uio_iov[ind].iov_len -= adj;
 674                                         break;
 675                                 }
 676
 677                                 adj -= uio->uio_iov[ind].iov_len;
 678                                 uio->uio_iov[ind].iov_len = 0;
 679                                 ind--;
 680                                 uio->uio_iovcnt--;
 681                         }
 682                 }
 683         }
 684         kfree(out, M_CRYPTO_DATA);
 685         return 0;
 686 }
 687
 688 /*
 689  * Generate a new software session.
 690  */
 691 static int
 692 swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
 693 {
 694         struct swcr_data *swd_base;
 695         struct swcr_data **swd;
 696         struct swcr_data **oswd;
 697         struct auth_hash *axf;
 698         struct enc_xform *txf;
 699         struct comp_algo *cxf;
 700         u_int32_t i;
 701         u_int32_t n;
 702         int error;
 703
 704         if (sid == NULL || cri == NULL)
 705                 return EINVAL;
 706
 707         swd_base = NULL;
 708         swd = &swd_base;
 709
 710         while (cri) {
 711                 *swd = kmalloc(sizeof(struct swcr_data),
 712                                M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 713
 714                 switch (cri->cri_alg) {
 715                 case CRYPTO_DES_CBC:
 716                         txf = &enc_xform_des;
 717                         goto enccommon;
 718                 case CRYPTO_3DES_CBC:
 719                         txf = &enc_xform_3des;
 720                         goto enccommon;
 721                 case CRYPTO_BLF_CBC:
 722                         txf = &enc_xform_blf;
 723                         goto enccommon;
 724                 case CRYPTO_CAST_CBC:
 725                         txf = &enc_xform_cast5;
 726                         goto enccommon;
 727                 case CRYPTO_SKIPJACK_CBC:
 728                         txf = &enc_xform_skipjack;
 729                         goto enccommon;
 730                 case CRYPTO_RIJNDAEL128_CBC:
 731                         txf = &enc_xform_rijndael128;
 732                         goto enccommon;
 733                 case CRYPTO_AES_XTS:
 734                         txf = &enc_xform_aes_xts;
 735                         goto enccommon;
 736                 case CRYPTO_AES_CTR:
 737                         txf = &enc_xform_aes_ctr;
 738                         goto enccommon;
 739                 case CRYPTO_CAMELLIA_CBC:
 740                         txf = &enc_xform_camellia;
 741                         goto enccommon;
 742                 case CRYPTO_NULL_CBC:
 743                         txf = &enc_xform_null;
 744                         goto enccommon;
 745                 enccommon:
 746                         if (cri->cri_key != NULL) {
 747                                 error = txf->setkey(&((*swd)->sw_kschedule),
 748                                                     cri->cri_key,
 749                                                     cri->cri_klen / 8);
 750                                 if (error) {
 751                                         swcr_freesession_slot(&swd_base, 0);
 752                                         return error;
 753                                 }
 754                         }
 755                         (*swd)->sw_exf = txf;
 756                         break;
 757
 758                 case CRYPTO_MD5_HMAC:
 759                         axf = &auth_hash_hmac_md5;
 760                         goto authcommon;
 761                 case CRYPTO_SHA1_HMAC:
 762                         axf = &auth_hash_hmac_sha1;
 763                         goto authcommon;
 764                 case CRYPTO_SHA2_256_HMAC:
 765                         axf = &auth_hash_hmac_sha2_256;
 766                         goto authcommon;
 767                 case CRYPTO_SHA2_384_HMAC:
 768                         axf = &auth_hash_hmac_sha2_384;
 769                         goto authcommon;
 770                 case CRYPTO_SHA2_512_HMAC:
 771                         axf = &auth_hash_hmac_sha2_512;
 772                         goto authcommon;
 773                 case CRYPTO_NULL_HMAC:
 774                         axf = &auth_hash_null;
 775                         goto authcommon;
 776                 case CRYPTO_RIPEMD160_HMAC:
 777                         axf = &auth_hash_hmac_ripemd_160;
 778                 authcommon:
 779                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 780                                                   M_WAITOK);
 781                         if ((*swd)->sw_ictx == NULL) {
 782                                 swcr_freesession_slot(&swd_base, 0);
 783                                 return ENOBUFS;
 784                         }
 785
 786                         (*swd)->sw_octx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 787                                                   M_WAITOK);
 788                         if ((*swd)->sw_octx == NULL) {
 789                                 swcr_freesession_slot(&swd_base, 0);
 790                                 return ENOBUFS;
 791                         }
 792
 793                         if (cri->cri_key != NULL) {
 794                                 swcr_authprepare(axf, *swd, cri->cri_key,
 795                                     cri->cri_klen);
 796                         }
 797
 798                         (*swd)->sw_mlen = cri->cri_mlen;
 799                         (*swd)->sw_axf = axf;
 800                         break;
 801
 802                 case CRYPTO_MD5_KPDK:
 803                         axf = &auth_hash_key_md5;
 804                         goto auth2common;
 805
 806                 case CRYPTO_SHA1_KPDK:
 807                         axf = &auth_hash_key_sha1;
 808                 auth2common:
 809                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 810                                                   M_WAITOK);
 811                         if ((*swd)->sw_ictx == NULL) {
 812                                 swcr_freesession_slot(&swd_base, 0);
 813                                 return ENOBUFS;
 814                         }
 815
 816                         (*swd)->sw_octx = kmalloc(cri->cri_klen / 8,
 817                                                   M_CRYPTO_DATA, M_WAITOK);
 818                         if ((*swd)->sw_octx == NULL) {
 819                                 swcr_freesession_slot(&swd_base, 0);
 820                                 return ENOBUFS;
 821                         }
 822
 823                         /* Store the key so we can "append" it to the payload */
 824                         if (cri->cri_key != NULL) {
 825                                 swcr_authprepare(axf, *swd, cri->cri_key,
 826                                     cri->cri_klen);
 827                         }
 828
 829                         (*swd)->sw_mlen = cri->cri_mlen;
 830                         (*swd)->sw_axf = axf;
 831                         break;
 832 #ifdef notdef
 833                 case CRYPTO_MD5:
 834                         axf = &auth_hash_md5;
 835                         goto auth3common;
 836
 837                 case CRYPTO_SHA1:
 838                         axf = &auth_hash_sha1;
 839                 auth3common:
 840                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 841                                                   M_WAITOK);
 842                         if ((*swd)->sw_ictx == NULL) {
 843                                 swcr_freesession_slot(&swd_base, 0);
 844                                 return ENOBUFS;
 845                         }
 846
 847                         axf->Init((*swd)->sw_ictx);
 848                         (*swd)->sw_mlen = cri->cri_mlen;
 849                         (*swd)->sw_axf = axf;
 850                         break;
 851 #endif
 852                 case CRYPTO_DEFLATE_COMP:
 853                         cxf = &comp_algo_deflate;
 854                         (*swd)->sw_cxf = cxf;
 855                         break;
 856                 default:
 857                         swcr_freesession_slot(&swd_base, 0);
 858                         return EINVAL;
 859                 }
 860
 861                 (*swd)->sw_alg = cri->cri_alg;
 862                 cri = cri->cri_next;
 863                 swd = &((*swd)->sw_next);
 864         }
 865
 866         for (;;) {
 867                 /*
 868                  * Atomically allocate a session
 869                  */
 870                 spin_lock(&swcr_spin);
 871                 for (i = swcr_minsesnum; i < swcr_sesnum; ++i) {
 872                         if (swcr_sessions[i] == NULL)
 873                                 break;
 874                 }
 875                 if (i < swcr_sesnum) {
 876                         swcr_sessions[i] = swd_base;
 877                         swcr_minsesnum = i + 1;
 878                         spin_unlock(&swcr_spin);
 879                         break;
 880                 }
 881                 n = swcr_sesnum;
 882                 spin_unlock(&swcr_spin);
 883
 884                 /*
 885                  * A larger allocation is required, reallocate the array
 886                  * and replace, checking for SMP races.
 887                  */
 888                 if (n < CRYPTO_SW_SESSIONS)
 889                         n = CRYPTO_SW_SESSIONS;
 890                 else
 891                         n = n * 3 / 2;
 892                 swd = kmalloc(n * sizeof(struct swcr_data *),
 893                               M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 894
 895                 spin_lock(&swcr_spin);
 896                 if (swcr_sesnum >= n) {
 897                         spin_unlock(&swcr_spin);
 898                         kfree(swd, M_CRYPTO_DATA);
 899                 } else if (swcr_sesnum) {
 900                         bcopy(swcr_sessions, swd,
 901                               swcr_sesnum * sizeof(struct swcr_data *));
 902                         oswd = swcr_sessions;
 903                         swcr_sessions = swd;
 904                         swcr_sesnum = n;
 905                         spin_unlock(&swcr_spin);
 906                         kfree(oswd, M_CRYPTO_DATA);
 907                 } else {
 908                         swcr_sessions = swd;
 909                         swcr_sesnum = n;
 910                         spin_unlock(&swcr_spin);
 911                 }
 912         }
 913
 914         *sid = i;
 915         return 0;
 916 }
 917
 918 /*
 919  * Free a session.
 920  */
 921 static int
 922 swcr_freesession(device_t dev, u_int64_t tid)
 923 {
 924         u_int32_t sid = CRYPTO_SESID2LID(tid);
 925
 926         if (sid > swcr_sesnum || swcr_sessions == NULL ||
 927             swcr_sessions[sid] == NULL) {
 928                 return EINVAL;
 929         }
 930
 931         /* Silently accept and return */
 932         if (sid == 0)
 933                 return 0;
 934
 935         return(swcr_freesession_slot(&swcr_sessions[sid], sid));
 936 }
 937
 938 static
 939 int
 940 swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid)
 941 {
 942         struct enc_xform *txf;
 943         struct auth_hash *axf;
 944         struct comp_algo *cxf;
 945         struct swcr_data *swd;
 946         struct swcr_data *swnext;
 947
 948         /*
 949          * Protect session detachment with the spinlock.
 950          */
 951         spin_lock(&swcr_spin);
 952         swnext = *swdp;
 953         *swdp = NULL;
 954         if (sid && swcr_minsesnum > sid)
 955                 swcr_minsesnum = sid;
 956         spin_unlock(&swcr_spin);
 957
 958         /*
 959          * Clean up at our leisure.
 960          */
 961         while ((swd = swnext) != NULL) {
 962                 swnext = swd->sw_next;
 963
 964                 swd->sw_next = NULL;
 965
 966                 switch (swd->sw_alg) {
 967                 case CRYPTO_DES_CBC:
 968                 case CRYPTO_3DES_CBC:
 969                 case CRYPTO_BLF_CBC:
 970                 case CRYPTO_CAST_CBC:
 971                 case CRYPTO_SKIPJACK_CBC:
 972                 case CRYPTO_RIJNDAEL128_CBC:
 973                 case CRYPTO_AES_XTS:
 974                 case CRYPTO_AES_CTR:
 975                 case CRYPTO_CAMELLIA_CBC:
 976                 case CRYPTO_NULL_CBC:
 977                         txf = swd->sw_exf;
 978
 979                         if (swd->sw_kschedule)
 980                                 txf->zerokey(&(swd->sw_kschedule));
 981                         break;
 982
 983                 case CRYPTO_MD5_HMAC:
 984                 case CRYPTO_SHA1_HMAC:
 985                 case CRYPTO_SHA2_256_HMAC:
 986                 case CRYPTO_SHA2_384_HMAC:
 987                 case CRYPTO_SHA2_512_HMAC:
 988                 case CRYPTO_RIPEMD160_HMAC:
 989                 case CRYPTO_NULL_HMAC:
 990                         axf = swd->sw_axf;
 991
 992                         if (swd->sw_ictx) {
 993                                 bzero(swd->sw_ictx, axf->ctxsize);
 994                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
 995                         }
 996                         if (swd->sw_octx) {
 997                                 bzero(swd->sw_octx, axf->ctxsize);
 998                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
 999                         }
1000                         break;
1001
1002                 case CRYPTO_MD5_KPDK:
1003                 case CRYPTO_SHA1_KPDK:
1004                         axf = swd->sw_axf;
1005
1006                         if (swd->sw_ictx) {
1007                                 bzero(swd->sw_ictx, axf->ctxsize);
1008                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
1009                         }
1010                         if (swd->sw_octx) {
1011                                 bzero(swd->sw_octx, swd->sw_klen);
1012                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
1013                         }
1014                         break;
1015
1016                 case CRYPTO_MD5:
1017                 case CRYPTO_SHA1:
1018                         axf = swd->sw_axf;
1019
1020                         if (swd->sw_ictx) {
1021                                 bzero(swd->sw_ictx, axf->ctxsize);
1022                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
1023                         }
1024                         break;
1025
1026                 case CRYPTO_DEFLATE_COMP:
1027                         cxf = swd->sw_cxf;
1028                         break;
1029                 }
1030
1031                 //FREE(swd, M_CRYPTO_DATA);
1032                 kfree(swd, M_CRYPTO_DATA);
1033         }
1034         return 0;
1035 }
1036
1037 /*
1038  * Process a software request.
1039  */
1040 static int
1041 swcr_process(device_t dev, struct cryptop *crp, int hint)
1042 {
1043         struct cryptodesc *crd;
1044         struct swcr_data *sw;
1045         u_int32_t lid;
1046
1047         /* Sanity check */
1048         if (crp == NULL)
1049                 return EINVAL;
1050
1051         if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
1052                 crp->crp_etype = EINVAL;
1053                 goto done;
1054         }
1055
1056         lid = crp->crp_sid & 0xffffffff;
1057         if (lid >= swcr_sesnum || lid == 0 || swcr_sessions[lid] == NULL) {
1058                 crp->crp_etype = ENOENT;
1059                 goto done;
1060         }
1061
1062         /* Go through crypto descriptors, processing as we go */
1063         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1064                 /*
1065                  * Find the crypto context.
1066                  *
1067                  * XXX Note that the logic here prevents us from having
1068                  * XXX the same algorithm multiple times in a session
1069                  * XXX (or rather, we can but it won't give us the right
1070                  * XXX results). To do that, we'd need some way of differentiating
1071                  * XXX between the various instances of an algorithm (so we can
1072                  * XXX locate the correct crypto context).
1073                  */
1074                 for (sw = swcr_sessions[lid];
1075                     sw && sw->sw_alg != crd->crd_alg;
1076                     sw = sw->sw_next)
1077                         ;
1078
1079                 /* No such context ? */
1080                 if (sw == NULL) {
1081                         crp->crp_etype = EINVAL;
1082                         goto done;
1083                 }
1084                 switch (sw->sw_alg) {
1085                 case CRYPTO_DES_CBC:
1086                 case CRYPTO_3DES_CBC:
1087                 case CRYPTO_BLF_CBC:
1088                 case CRYPTO_CAST_CBC:
1089                 case CRYPTO_SKIPJACK_CBC:
1090                 case CRYPTO_RIJNDAEL128_CBC:
1091                 case CRYPTO_AES_XTS:
1092                 case CRYPTO_AES_CTR:
1093                 case CRYPTO_CAMELLIA_CBC:
1094                         if ((crp->crp_etype = swcr_encdec(crd, sw,
1095                             crp->crp_buf, crp->crp_flags)) != 0)
1096                                 goto done;
1097                         break;
1098                 case CRYPTO_NULL_CBC:
1099                         crp->crp_etype = 0;
1100                         break;
1101                 case CRYPTO_MD5_HMAC:
1102                 case CRYPTO_SHA1_HMAC:
1103                 case CRYPTO_SHA2_256_HMAC:
1104                 case CRYPTO_SHA2_384_HMAC:
1105                 case CRYPTO_SHA2_512_HMAC:
1106                 case CRYPTO_RIPEMD160_HMAC:
1107                 case CRYPTO_NULL_HMAC:
1108                 case CRYPTO_MD5_KPDK:
1109                 case CRYPTO_SHA1_KPDK:
1110                 case CRYPTO_MD5:
1111                 case CRYPTO_SHA1:
1112                         if ((crp->crp_etype = swcr_authcompute(crd, sw,
1113                             crp->crp_buf, crp->crp_flags)) != 0)
1114                                 goto done;
1115                         break;
1116
1117                 case CRYPTO_DEFLATE_COMP:
1118                         if ((crp->crp_etype = swcr_compdec(crd, sw,
1119                             crp->crp_buf, crp->crp_flags)) != 0)
1120                                 goto done;
1121                         else
1122                                 crp->crp_olen = (int)sw->sw_size;
1123                         break;
1124
1125                 default:
1126                         /* Unknown/unsupported algorithm */
1127                         crp->crp_etype = EINVAL;
1128                         goto done;
1129                 }
1130         }
1131
1132 done:
1133         crypto_done(crp);
1134         lwkt_yield();
1135         return 0;
1136 }
1137
1138 static void
1139 swcr_identify(driver_t *drv, device_t parent)
1140 {
1141         /* NB: order 10 is so we get attached after h/w devices */
1142         /* XXX: wouldn't bet about this BUS_ADD_CHILD correctness */
1143         if (device_find_child(parent, "cryptosoft", -1) == NULL &&
1144             BUS_ADD_CHILD(parent, parent, 10, "cryptosoft", -1) == 0)
1145                 panic("cryptosoft: could not attach");
1146 }
1147
1148 static int
1149 swcr_probe(device_t dev)
1150 {
1151         device_set_desc(dev, "software crypto");
1152         return (0);
1153 }
1154
1155 static int
1156 swcr_attach(device_t dev)
1157 {
1158         memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
1159         memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
1160
1161         swcr_id = crypto_get_driverid(dev, CRYPTOCAP_F_SOFTWARE |
1162                                            CRYPTOCAP_F_SYNC |
1163                                            CRYPTOCAP_F_SMP);
1164         if (swcr_id < 0) {
1165                 device_printf(dev, "cannot initialize!");
1166                 return ENOMEM;
1167         }
1168 #define REGISTER(alg) \
1169         crypto_register(swcr_id, alg, 0,0)
1170         REGISTER(CRYPTO_DES_CBC);
1171         REGISTER(CRYPTO_3DES_CBC);
1172         REGISTER(CRYPTO_BLF_CBC);
1173         REGISTER(CRYPTO_CAST_CBC);
1174         REGISTER(CRYPTO_SKIPJACK_CBC);
1175         REGISTER(CRYPTO_NULL_CBC);
1176         REGISTER(CRYPTO_MD5_HMAC);
1177         REGISTER(CRYPTO_SHA1_HMAC);
1178         REGISTER(CRYPTO_SHA2_256_HMAC);
1179         REGISTER(CRYPTO_SHA2_384_HMAC);
1180         REGISTER(CRYPTO_SHA2_512_HMAC);
1181         REGISTER(CRYPTO_RIPEMD160_HMAC);
1182         REGISTER(CRYPTO_NULL_HMAC);
1183         REGISTER(CRYPTO_MD5_KPDK);
1184         REGISTER(CRYPTO_SHA1_KPDK);
1185         REGISTER(CRYPTO_MD5);
1186         REGISTER(CRYPTO_SHA1);
1187         REGISTER(CRYPTO_RIJNDAEL128_CBC);
1188         REGISTER(CRYPTO_AES_XTS);
1189         REGISTER(CRYPTO_AES_CTR);
1190         REGISTER(CRYPTO_CAMELLIA_CBC);
1191         REGISTER(CRYPTO_DEFLATE_COMP);
1192 #undef REGISTER
1193
1194         return 0;
1195 }
1196
1197 static int
1198 swcr_detach(device_t dev)
1199 {
1200         crypto_unregister_all(swcr_id);
1201         if (swcr_sessions != NULL)
1202                 kfree(swcr_sessions, M_CRYPTO_DATA);
1203         return 0;
1204 }
1205
1206 static device_method_t swcr_methods[] = {
1207         DEVMETHOD(device_identify,      swcr_identify),
1208         DEVMETHOD(device_probe,         swcr_probe),
1209         DEVMETHOD(device_attach,        swcr_attach),
1210         DEVMETHOD(device_detach,        swcr_detach),
1211
1212         DEVMETHOD(cryptodev_newsession, swcr_newsession),
1213         DEVMETHOD(cryptodev_freesession,swcr_freesession),
1214         DEVMETHOD(cryptodev_process,    swcr_process),
1215
1216         {0, 0},
1217 };
1218
1219 static driver_t swcr_driver = {
1220         "cryptosoft",
1221         swcr_methods,
1222         0,              /* NB: no softc */
1223 };
1224 static devclass_t swcr_devclass;
1225
1226 /*
1227  * NB: We explicitly reference the crypto module so we
1228  * get the necessary ordering when built as a loadable
1229  * module.  This is required because we bundle the crypto
1230  * module code together with the cryptosoft driver (otherwise
1231  * normal module dependencies would handle things).
1232  */
1233 extern int crypto_modevent(struct module *, int, void *);
1234 /* XXX where to attach */
1235 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0);
1236 MODULE_VERSION(cryptosoft, 1);
1237 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);