sys/opencrypto/cryptosoft.c

   1 /*-
   2  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
   3  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
   4  *
   5  * This code was written by Angelos D. Keromytis in Athens, Greece, in
   6  * February 2000. Network Security Technologies Inc. (NSTI) kindly
   7  * supported the development of this code.
   8  *
   9  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  10  *
  11  * SMP modifications by Matthew Dillon for the DragonFlyBSD Project
  12  *
  13  * Permission to use, copy, and modify this software with or without fee
  14  * is hereby granted, provided that this entire notice is included in
  15  * all source code copies of any software which is or includes a copy or
  16  * modification of this software.
  17  *
  18  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  19  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  20  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  21  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  22  * PURPOSE.
  23  *
  24  * $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $
  25  * $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $
  26  */
  27
  28 #include <sys/param.h>
  29 #include <sys/systm.h>
  30 #include <sys/malloc.h>
  31 #include <sys/mbuf.h>
  32 #include <sys/module.h>
  33 #include <sys/sysctl.h>
  34 #include <sys/errno.h>
  35 #include <sys/random.h>
  36 #include <sys/kernel.h>
  37 #include <sys/uio.h>
  38 #include <sys/spinlock2.h>
  39
  40 #include <crypto/blowfish/blowfish.h>
  41 #include <crypto/sha1.h>
  42 #include <opencrypto/rmd160.h>
  43 #include <opencrypto/cast.h>
  44 #include <opencrypto/skipjack.h>
  45 #include <sys/md5.h>
  46
  47 #include <opencrypto/cryptodev.h>
  48 #include <opencrypto/cryptosoft.h>
  49 #include <opencrypto/xform.h>
  50
  51 #include <sys/kobj.h>
  52 #include <sys/bus.h>
  53 #include "cryptodev_if.h"
  54
  55 static  int32_t swcr_id;
  56 static  struct swcr_data **swcr_sessions = NULL;
  57 static  u_int32_t swcr_sesnum;
  58 static  u_int32_t swcr_minsesnum = 1;
  59
  60 static struct spinlock swcr_spin = SPINLOCK_INITIALIZER(swcr_spin);
  61
  62 u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
  63 u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
  64
  65 static  int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  66 static  int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  67 static  int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
  68 static  int swcr_freesession(device_t dev, u_int64_t tid);
  69 static  int swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid);
  70
  71 /*
  72  * Apply a symmetric encryption/decryption algorithm.
  73  */
  74 static int
  75 swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
  76     int flags)
  77 {
  78         unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
  79         unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN];
  80         u_int8_t *kschedule;
  81         u_int8_t *okschedule;
  82         struct enc_xform *exf;
  83         int i, k, j, blks, ivlen;
  84         int error;
  85         int explicit_kschedule;
  86
  87         exf = sw->sw_exf;
  88         blks = exf->blocksize;
  89         ivlen = exf->ivsize;
  90
  91         /* Check for non-padded data */
  92         if (crd->crd_len % blks)
  93                 return EINVAL;
  94
  95         /* Initialize the IV */
  96         if (crd->crd_flags & CRD_F_ENCRYPT) {
  97                 /* IV explicitly provided ? */
  98                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
  99                         bcopy(crd->crd_iv, iv, ivlen);
 100                 else
 101                         karc4rand(iv, ivlen);
 102
 103                 /* Do we need to write the IV */
 104                 if (!(crd->crd_flags & CRD_F_IV_PRESENT))
 105                         crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
 106
 107         } else {        /* Decryption */
 108                         /* IV explicitly provided ? */
 109                 if (crd->crd_flags & CRD_F_IV_EXPLICIT)
 110                         bcopy(crd->crd_iv, iv, ivlen);
 111                 else {
 112                         /* Get IV off buf */
 113                         crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
 114                 }
 115         }
 116
 117         ivp = iv;
 118
 119         /*
 120          * The semantics are seriously broken because the session key
 121          * storage was never designed for concurrent ops.
 122          */
 123         if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
 124                 kschedule = NULL;
 125                 explicit_kschedule = 1;
 126                 error = exf->setkey(&kschedule,
 127                                     crd->crd_key, crd->crd_klen / 8);
 128                 if (error)
 129                         goto done;
 130         } else {
 131                 spin_lock(&swcr_spin);
 132                 kschedule = sw->sw_kschedule;
 133                 ++sw->sw_kschedule_refs;
 134                 spin_unlock(&swcr_spin);
 135                 explicit_kschedule = 0;
 136         }
 137
 138         /*
 139          * xforms that provide a reinit method perform all IV
 140          * handling themselves.
 141          */
 142         if (exf->reinit)
 143                 exf->reinit(kschedule, iv);
 144
 145         if (flags & CRYPTO_F_IMBUF) {
 146                 struct mbuf *m = (struct mbuf *) buf;
 147
 148                 /* Find beginning of data */
 149                 m = m_getptr(m, crd->crd_skip, &k);
 150                 if (m == NULL) {
 151                         error = EINVAL;
 152                         goto done;
 153                 }
 154
 155                 i = crd->crd_len;
 156
 157                 while (i > 0) {
 158                         /*
 159                          * If there's insufficient data at the end of
 160                          * an mbuf, we have to do some copying.
 161                          */
 162                         if (m->m_len < k + blks && m->m_len != k) {
 163                                 m_copydata(m, k, blks, blk);
 164
 165                                 /* Actual encryption/decryption */
 166                                 if (exf->reinit) {
 167                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 168                                                 exf->encrypt(kschedule,
 169                                                     blk, iv);
 170                                         } else {
 171                                                 exf->decrypt(kschedule,
 172                                                     blk, iv);
 173                                         }
 174                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 175                                         /* XOR with previous block */
 176                                         for (j = 0; j < blks; j++)
 177                                                 blk[j] ^= ivp[j];
 178
 179                                         exf->encrypt(kschedule, blk, iv);
 180
 181                                         /*
 182                                          * Keep encrypted block for XOR'ing
 183                                          * with next block
 184                                          */
 185                                         bcopy(blk, iv, blks);
 186                                         ivp = iv;
 187                                 } else {        /* decrypt */
 188                                         /*
 189                                          * Keep encrypted block for XOR'ing
 190                                          * with next block
 191                                          */
 192                                         nivp = (ivp == iv) ? iv2 : iv;
 193                                         bcopy(blk, nivp, blks);
 194
 195                                         exf->decrypt(kschedule, blk, iv);
 196
 197                                         /* XOR with previous block */
 198                                         for (j = 0; j < blks; j++)
 199                                                 blk[j] ^= ivp[j];
 200
 201                                         ivp = nivp;
 202                                 }
 203
 204                                 /* Copy back decrypted block */
 205                                 m_copyback(m, k, blks, blk);
 206
 207                                 /* Advance pointer */
 208                                 m = m_getptr(m, k + blks, &k);
 209                                 if (m == NULL) {
 210                                         error = EINVAL;
 211                                         goto done;
 212                                 }
 213
 214                                 i -= blks;
 215
 216                                 /* Could be done... */
 217                                 if (i == 0)
 218                                         break;
 219                         }
 220
 221                         /* Skip possibly empty mbufs */
 222                         if (k == m->m_len) {
 223                                 for (m = m->m_next; m && m->m_len == 0;
 224                                     m = m->m_next)
 225                                         ;
 226                                 k = 0;
 227                         }
 228
 229                         /* Sanity check */
 230                         if (m == NULL) {
 231                                 error = EINVAL;
 232                                 goto done;
 233                         }
 234
 235                         /*
 236                          * Warning: idat may point to garbage here, but
 237                          * we only use it in the while() loop, only if
 238                          * there are indeed enough data.
 239                          */
 240                         idat = mtod(m, unsigned char *) + k;
 241
 242                         while (m->m_len >= k + blks && i > 0) {
 243                                 if (exf->reinit) {
 244                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 245                                                 exf->encrypt(kschedule,
 246                                                     idat, iv);
 247                                         } else {
 248                                                 exf->decrypt(kschedule,
 249                                                     idat, iv);
 250                                         }
 251                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 252                                         /* XOR with previous block/IV */
 253                                         for (j = 0; j < blks; j++)
 254                                                 idat[j] ^= ivp[j];
 255
 256                                         exf->encrypt(kschedule, idat, iv);
 257                                         ivp = idat;
 258                                 } else {        /* decrypt */
 259                                         /*
 260                                          * Keep encrypted block to be used
 261                                          * in next block's processing.
 262                                          */
 263                                         nivp = (ivp == iv) ? iv2 : iv;
 264                                         bcopy(idat, nivp, blks);
 265
 266                                         exf->decrypt(kschedule, idat, iv);
 267
 268                                         /* XOR with previous block/IV */
 269                                         for (j = 0; j < blks; j++)
 270                                                 idat[j] ^= ivp[j];
 271
 272                                         ivp = nivp;
 273                                 }
 274
 275                                 idat += blks;
 276                                 k += blks;
 277                                 i -= blks;
 278                         }
 279                 }
 280                 error = 0;      /* Done with mbuf encryption/decryption */
 281         } else if (flags & CRYPTO_F_IOV) {
 282                 struct uio *uio = (struct uio *) buf;
 283                 struct iovec *iov;
 284
 285                 /* Find beginning of data */
 286                 iov = cuio_getptr(uio, crd->crd_skip, &k);
 287                 if (iov == NULL) {
 288                         error = EINVAL;
 289                         goto done;
 290                 }
 291
 292                 i = crd->crd_len;
 293
 294                 while (i > 0) {
 295                         /*
 296                          * If there's insufficient data at the end of
 297                          * an iovec, we have to do some copying.
 298                          */
 299                         if (iov->iov_len < k + blks && iov->iov_len != k) {
 300                                 cuio_copydata(uio, k, blks, blk);
 301
 302                                 /* Actual encryption/decryption */
 303                                 if (exf->reinit) {
 304                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 305                                                 exf->encrypt(kschedule,
 306                                                     blk, iv);
 307                                         } else {
 308                                                 exf->decrypt(kschedule,
 309                                                     blk, iv);
 310                                         }
 311                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 312                                         /* XOR with previous block */
 313                                         for (j = 0; j < blks; j++)
 314                                                 blk[j] ^= ivp[j];
 315
 316                                         exf->encrypt(kschedule, blk, iv);
 317
 318                                         /*
 319                                          * Keep encrypted block for XOR'ing
 320                                          * with next block
 321                                          */
 322                                         bcopy(blk, iv, blks);
 323                                         ivp = iv;
 324                                 } else {        /* decrypt */
 325                                         /*
 326                                          * Keep encrypted block for XOR'ing
 327                                          * with next block
 328                                          */
 329                                         nivp = (ivp == iv) ? iv2 : iv;
 330                                         bcopy(blk, nivp, blks);
 331
 332                                         exf->decrypt(kschedule, blk, iv);
 333
 334                                         /* XOR with previous block */
 335                                         for (j = 0; j < blks; j++)
 336                                                 blk[j] ^= ivp[j];
 337
 338                                         ivp = nivp;
 339                                 }
 340
 341                                 /* Copy back decrypted block */
 342                                 cuio_copyback(uio, k, blks, blk);
 343
 344                                 /* Advance pointer */
 345                                 iov = cuio_getptr(uio, k + blks, &k);
 346                                 if (iov == NULL) {
 347                                         error = EINVAL;
 348                                         goto done;
 349                                 }
 350
 351                                 i -= blks;
 352
 353                                 /* Could be done... */
 354                                 if (i == 0)
 355                                         break;
 356                         }
 357
 358                         /*
 359                          * Warning: idat may point to garbage here, but
 360                          * we only use it in the while() loop, only if
 361                          * there are indeed enough data.
 362                          */
 363                         idat = (char *)iov->iov_base + k;
 364
 365                         while (iov->iov_len >= k + blks && i > 0) {
 366                                 if (exf->reinit) {
 367                                         if (crd->crd_flags & CRD_F_ENCRYPT) {
 368                                                 exf->encrypt(kschedule,
 369                                                     idat, iv);
 370                                         } else {
 371                                                 exf->decrypt(kschedule,
 372                                                     idat, iv);
 373                                         }
 374                                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 375                                         /* XOR with previous block/IV */
 376                                         for (j = 0; j < blks; j++)
 377                                                 idat[j] ^= ivp[j];
 378
 379                                         exf->encrypt(kschedule, idat, iv);
 380                                         ivp = idat;
 381                                 } else {        /* decrypt */
 382                                         /*
 383                                          * Keep encrypted block to be used
 384                                          * in next block's processing.
 385                                          */
 386                                         nivp = (ivp == iv) ? iv2 : iv;
 387                                         bcopy(idat, nivp, blks);
 388
 389                                         exf->decrypt(kschedule, idat, iv);
 390
 391                                         /* XOR with previous block/IV */
 392                                         for (j = 0; j < blks; j++)
 393                                                 idat[j] ^= ivp[j];
 394
 395                                         ivp = nivp;
 396                                 }
 397
 398                                 idat += blks;
 399                                 k += blks;
 400                                 i -= blks;
 401                         }
 402                         if (k == iov->iov_len) {
 403                                 iov++;
 404                                 k = 0;
 405                         }
 406                 }
 407                 error = 0;      /* Done with iovec encryption/decryption */
 408         } else {
 409                 /*
 410                  * contiguous buffer
 411                  */
 412                 if (exf->reinit) {
 413                         for(i = crd->crd_skip;
 414                             i < crd->crd_skip + crd->crd_len; i += blks) {
 415                                 if (crd->crd_flags & CRD_F_ENCRYPT) {
 416                                         exf->encrypt(kschedule, buf + i, iv);
 417                                 } else {
 418                                         exf->decrypt(kschedule, buf + i, iv);
 419                                 }
 420                         }
 421                 } else if (crd->crd_flags & CRD_F_ENCRYPT) {
 422                         for (i = crd->crd_skip;
 423                             i < crd->crd_skip + crd->crd_len; i += blks) {
 424                                 /* XOR with the IV/previous block, as appropriate. */
 425                                 if (i == crd->crd_skip)
 426                                         for (k = 0; k < blks; k++)
 427                                                 buf[i + k] ^= ivp[k];
 428                                 else
 429                                         for (k = 0; k < blks; k++)
 430                                                 buf[i + k] ^= buf[i + k - blks];
 431                                 exf->encrypt(kschedule, buf + i, iv);
 432                         }
 433                 } else {                /* Decrypt */
 434                         /*
 435                          * Start at the end, so we don't need to keep the
 436                          * encrypted block as the IV for the next block.
 437                          */
 438                         for (i = crd->crd_skip + crd->crd_len - blks;
 439                             i >= crd->crd_skip; i -= blks) {
 440                                 exf->decrypt(kschedule, buf + i, iv);
 441
 442                                 /* XOR with the IV/previous block, as appropriate */
 443                                 if (i == crd->crd_skip)
 444                                         for (k = 0; k < blks; k++)
 445                                                 buf[i + k] ^= ivp[k];
 446                                 else
 447                                         for (k = 0; k < blks; k++)
 448                                                 buf[i + k] ^= buf[i + k - blks];
 449                         }
 450                 }
 451                 error = 0; /* Done w/contiguous buffer encrypt/decrypt */
 452         }
 453 done:
 454         /*
 455          * Cleanup - explicitly replace the session key if requested
 456          *           (horrible semantics for concurrent operation)
 457          */
 458         if (explicit_kschedule) {
 459                 spin_lock(&swcr_spin);
 460                 if (sw->sw_kschedule && sw->sw_kschedule_refs == 0) {
 461                         okschedule = sw->sw_kschedule;
 462                         sw->sw_kschedule = kschedule;
 463                 } else {
 464                         okschedule = NULL;
 465                 }
 466                 spin_unlock(&swcr_spin);
 467                 if (okschedule)
 468                         exf->zerokey(&okschedule);
 469         } else {
 470                 spin_lock(&swcr_spin);
 471                 --sw->sw_kschedule_refs;
 472                 spin_unlock(&swcr_spin);
 473         }
 474         return error;
 475 }
 476
 477 static void
 478 swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
 479     int klen)
 480 {
 481         int k;
 482
 483         klen /= 8;
 484
 485         switch (axf->type) {
 486         case CRYPTO_MD5_HMAC:
 487         case CRYPTO_SHA1_HMAC:
 488         case CRYPTO_SHA2_256_HMAC:
 489         case CRYPTO_SHA2_384_HMAC:
 490         case CRYPTO_SHA2_512_HMAC:
 491         case CRYPTO_NULL_HMAC:
 492         case CRYPTO_RIPEMD160_HMAC:
 493                 for (k = 0; k < klen; k++)
 494                         key[k] ^= HMAC_IPAD_VAL;
 495
 496                 axf->Init(sw->sw_ictx);
 497                 axf->Update(sw->sw_ictx, key, klen);
 498                 axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
 499
 500                 for (k = 0; k < klen; k++)
 501                         key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 502
 503                 axf->Init(sw->sw_octx);
 504                 axf->Update(sw->sw_octx, key, klen);
 505                 axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
 506
 507                 for (k = 0; k < klen; k++)
 508                         key[k] ^= HMAC_OPAD_VAL;
 509                 break;
 510         case CRYPTO_MD5_KPDK:
 511         case CRYPTO_SHA1_KPDK:
 512         {
 513                 /* We need a buffer that can hold an md5 and a sha1 result. */
 514                 u_char buf[SHA1_RESULTLEN];
 515
 516                 sw->sw_klen = klen;
 517                 bcopy(key, sw->sw_octx, klen);
 518                 axf->Init(sw->sw_ictx);
 519                 axf->Update(sw->sw_ictx, key, klen);
 520                 axf->Final(buf, sw->sw_ictx);
 521                 break;
 522         }
 523         default:
 524                 kprintf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
 525                     "doesn't use keys.\n", __func__, axf->type);
 526         }
 527 }
 528
 529 /*
 530  * Compute keyed-hash authenticator.
 531  */
 532 static int
 533 swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
 534     int flags)
 535 {
 536         unsigned char aalg[HASH_MAX_LEN];
 537         struct auth_hash *axf;
 538         union authctx ctx;
 539         int err;
 540
 541         if (sw->sw_ictx == 0)
 542                 return EINVAL;
 543
 544         axf = sw->sw_axf;
 545
 546         if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
 547                 swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
 548
 549         bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
 550
 551         err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
 552             (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
 553         if (err)
 554                 return err;
 555
 556         switch (sw->sw_alg) {
 557         case CRYPTO_MD5_HMAC:
 558         case CRYPTO_SHA1_HMAC:
 559         case CRYPTO_SHA2_256_HMAC:
 560         case CRYPTO_SHA2_384_HMAC:
 561         case CRYPTO_SHA2_512_HMAC:
 562         case CRYPTO_RIPEMD160_HMAC:
 563                 if (sw->sw_octx == NULL)
 564                         return EINVAL;
 565
 566                 axf->Final(aalg, &ctx);
 567                 bcopy(sw->sw_octx, &ctx, axf->ctxsize);
 568                 axf->Update(&ctx, aalg, axf->hashsize);
 569                 axf->Final(aalg, &ctx);
 570                 break;
 571
 572         case CRYPTO_MD5_KPDK:
 573         case CRYPTO_SHA1_KPDK:
 574                 if (sw->sw_octx == NULL)
 575                         return EINVAL;
 576
 577                 axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
 578                 axf->Final(aalg, &ctx);
 579                 break;
 580
 581         case CRYPTO_NULL_HMAC:
 582                 axf->Final(aalg, &ctx);
 583                 break;
 584         }
 585
 586         /* Inject the authentication data */
 587         crypto_copyback(flags, buf, crd->crd_inject,
 588             sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
 589         return 0;
 590 }
 591
 592 /*
 593  * Apply a compression/decompression algorithm
 594  */
 595 static int
 596 swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
 597              caddr_t buf, int flags)
 598 {
 599         u_int8_t *data, *out;
 600         struct comp_algo *cxf;
 601         int adj;
 602         u_int32_t result;
 603
 604         cxf = sw->sw_cxf;
 605
 606         /*
 607          * We must handle the whole buffer of data in one time
 608          * then if there is not all the data in the mbuf, we must
 609          * copy in a buffer.
 610          */
 611         data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_INTWAIT);
 612         if (data == NULL)
 613                 return (EINVAL);
 614         crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
 615
 616         if (crd->crd_flags & CRD_F_COMP)
 617                 result = cxf->compress(data, crd->crd_len, &out);
 618         else
 619                 result = cxf->decompress(data, crd->crd_len, &out);
 620
 621         kfree(data, M_CRYPTO_DATA);
 622         if (result == 0)
 623                 return EINVAL;
 624
 625         /* Copy back the (de)compressed data. m_copyback is
 626          * extending the mbuf as necessary.
 627          */
 628         sw->sw_size = result;
 629         /* Check the compressed size when doing compression */
 630         if (crd->crd_flags & CRD_F_COMP) {
 631                 if (result >= crd->crd_len) {
 632                         /* Compression was useless, we lost time */
 633                         kfree(out, M_CRYPTO_DATA);
 634                         return 0;
 635                 }
 636         }
 637
 638         crypto_copyback(flags, buf, crd->crd_skip, result, out);
 639         if (result < crd->crd_len) {
 640                 adj = result - crd->crd_len;
 641                 if (flags & CRYPTO_F_IMBUF) {
 642                         adj = result - crd->crd_len;
 643                         m_adj((struct mbuf *)buf, adj);
 644                 } else if (flags & CRYPTO_F_IOV) {
 645                         struct uio *uio = (struct uio *)buf;
 646                         int ind;
 647
 648                         adj = crd->crd_len - result;
 649                         ind = uio->uio_iovcnt - 1;
 650
 651                         while (adj > 0 && ind >= 0) {
 652                                 if (adj < uio->uio_iov[ind].iov_len) {
 653                                         uio->uio_iov[ind].iov_len -= adj;
 654                                         break;
 655                                 }
 656
 657                                 adj -= uio->uio_iov[ind].iov_len;
 658                                 uio->uio_iov[ind].iov_len = 0;
 659                                 ind--;
 660                                 uio->uio_iovcnt--;
 661                         }
 662                 }
 663         }
 664         kfree(out, M_CRYPTO_DATA);
 665         return 0;
 666 }
 667
 668 /*
 669  * Generate a new software session.
 670  */
 671 static int
 672 swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
 673 {
 674         struct swcr_data *swd_base;
 675         struct swcr_data **swd;
 676         struct swcr_data **oswd;
 677         struct auth_hash *axf;
 678         struct enc_xform *txf;
 679         struct comp_algo *cxf;
 680         u_int32_t i;
 681         u_int32_t n;
 682         int error;
 683
 684         if (sid == NULL || cri == NULL)
 685                 return EINVAL;
 686
 687         swd_base = NULL;
 688         swd = &swd_base;
 689
 690         while (cri) {
 691                 *swd = kmalloc(sizeof(struct swcr_data),
 692                                M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 693
 694                 switch (cri->cri_alg) {
 695                 case CRYPTO_DES_CBC:
 696                         txf = &enc_xform_des;
 697                         goto enccommon;
 698                 case CRYPTO_3DES_CBC:
 699                         txf = &enc_xform_3des;
 700                         goto enccommon;
 701                 case CRYPTO_BLF_CBC:
 702                         txf = &enc_xform_blf;
 703                         goto enccommon;
 704                 case CRYPTO_CAST_CBC:
 705                         txf = &enc_xform_cast5;
 706                         goto enccommon;
 707                 case CRYPTO_SKIPJACK_CBC:
 708                         txf = &enc_xform_skipjack;
 709                         goto enccommon;
 710                 case CRYPTO_RIJNDAEL128_CBC:
 711                         txf = &enc_xform_rijndael128;
 712                         goto enccommon;
 713                 case CRYPTO_AES_XTS:
 714                         txf = &enc_xform_aes_xts;
 715                         goto enccommon;
 716                 case CRYPTO_AES_CTR:
 717                         txf = &enc_xform_aes_ctr;
 718                         goto enccommon;
 719                 case CRYPTO_CAMELLIA_CBC:
 720                         txf = &enc_xform_camellia;
 721                         goto enccommon;
 722                 case CRYPTO_NULL_CBC:
 723                         txf = &enc_xform_null;
 724                         goto enccommon;
 725                 enccommon:
 726                         if (cri->cri_key != NULL) {
 727                                 error = txf->setkey(&((*swd)->sw_kschedule),
 728                                                     cri->cri_key,
 729                                                     cri->cri_klen / 8);
 730                                 if (error) {
 731                                         swcr_freesession_slot(&swd_base, 0);
 732                                         return error;
 733                                 }
 734                         }
 735                         (*swd)->sw_exf = txf;
 736                         break;
 737
 738                 case CRYPTO_MD5_HMAC:
 739                         axf = &auth_hash_hmac_md5;
 740                         goto authcommon;
 741                 case CRYPTO_SHA1_HMAC:
 742                         axf = &auth_hash_hmac_sha1;
 743                         goto authcommon;
 744                 case CRYPTO_SHA2_256_HMAC:
 745                         axf = &auth_hash_hmac_sha2_256;
 746                         goto authcommon;
 747                 case CRYPTO_SHA2_384_HMAC:
 748                         axf = &auth_hash_hmac_sha2_384;
 749                         goto authcommon;
 750                 case CRYPTO_SHA2_512_HMAC:
 751                         axf = &auth_hash_hmac_sha2_512;
 752                         goto authcommon;
 753                 case CRYPTO_NULL_HMAC:
 754                         axf = &auth_hash_null;
 755                         goto authcommon;
 756                 case CRYPTO_RIPEMD160_HMAC:
 757                         axf = &auth_hash_hmac_ripemd_160;
 758                 authcommon:
 759                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 760                                                   M_WAITOK);
 761                         if ((*swd)->sw_ictx == NULL) {
 762                                 swcr_freesession_slot(&swd_base, 0);
 763                                 return ENOBUFS;
 764                         }
 765
 766                         (*swd)->sw_octx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 767                                                   M_WAITOK);
 768                         if ((*swd)->sw_octx == NULL) {
 769                                 swcr_freesession_slot(&swd_base, 0);
 770                                 return ENOBUFS;
 771                         }
 772
 773                         if (cri->cri_key != NULL) {
 774                                 swcr_authprepare(axf, *swd, cri->cri_key,
 775                                     cri->cri_klen);
 776                         }
 777
 778                         (*swd)->sw_mlen = cri->cri_mlen;
 779                         (*swd)->sw_axf = axf;
 780                         break;
 781
 782                 case CRYPTO_MD5_KPDK:
 783                         axf = &auth_hash_key_md5;
 784                         goto auth2common;
 785
 786                 case CRYPTO_SHA1_KPDK:
 787                         axf = &auth_hash_key_sha1;
 788                 auth2common:
 789                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 790                                                   M_WAITOK);
 791                         if ((*swd)->sw_ictx == NULL) {
 792                                 swcr_freesession_slot(&swd_base, 0);
 793                                 return ENOBUFS;
 794                         }
 795
 796                         (*swd)->sw_octx = kmalloc(cri->cri_klen / 8,
 797                                                   M_CRYPTO_DATA, M_WAITOK);
 798                         if ((*swd)->sw_octx == NULL) {
 799                                 swcr_freesession_slot(&swd_base, 0);
 800                                 return ENOBUFS;
 801                         }
 802
 803                         /* Store the key so we can "append" it to the payload */
 804                         if (cri->cri_key != NULL) {
 805                                 swcr_authprepare(axf, *swd, cri->cri_key,
 806                                     cri->cri_klen);
 807                         }
 808
 809                         (*swd)->sw_mlen = cri->cri_mlen;
 810                         (*swd)->sw_axf = axf;
 811                         break;
 812 #ifdef notdef
 813                 case CRYPTO_MD5:
 814                         axf = &auth_hash_md5;
 815                         goto auth3common;
 816
 817                 case CRYPTO_SHA1:
 818                         axf = &auth_hash_sha1;
 819                 auth3common:
 820                         (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
 821                                                   M_WAITOK);
 822                         if ((*swd)->sw_ictx == NULL) {
 823                                 swcr_freesession_slot(&swd_base, 0);
 824                                 return ENOBUFS;
 825                         }
 826
 827                         axf->Init((*swd)->sw_ictx);
 828                         (*swd)->sw_mlen = cri->cri_mlen;
 829                         (*swd)->sw_axf = axf;
 830                         break;
 831 #endif
 832                 case CRYPTO_DEFLATE_COMP:
 833                         cxf = &comp_algo_deflate;
 834                         (*swd)->sw_cxf = cxf;
 835                         break;
 836                 default:
 837                         swcr_freesession_slot(&swd_base, 0);
 838                         return EINVAL;
 839                 }
 840
 841                 (*swd)->sw_alg = cri->cri_alg;
 842                 cri = cri->cri_next;
 843                 swd = &((*swd)->sw_next);
 844         }
 845
 846         for (;;) {
 847                 /*
 848                  * Atomically allocate a session
 849                  */
 850                 spin_lock(&swcr_spin);
 851                 for (i = swcr_minsesnum; i < swcr_sesnum; ++i) {
 852                         if (swcr_sessions[i] == NULL)
 853                                 break;
 854                 }
 855                 if (i < swcr_sesnum) {
 856                         swcr_sessions[i] = swd_base;
 857                         swcr_minsesnum = i + 1;
 858                         spin_unlock(&swcr_spin);
 859                         break;
 860                 }
 861                 n = swcr_sesnum;
 862                 spin_unlock(&swcr_spin);
 863
 864                 /*
 865                  * A larger allocation is required, reallocate the array
 866                  * and replace, checking for SMP races.
 867                  */
 868                 if (n < CRYPTO_SW_SESSIONS)
 869                         n = CRYPTO_SW_SESSIONS;
 870                 else
 871                         n = n * 3 / 2;
 872                 swd = kmalloc(n * sizeof(struct swcr_data *),
 873                               M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 874
 875                 spin_lock(&swcr_spin);
 876                 if (swcr_sesnum >= n) {
 877                         spin_unlock(&swcr_spin);
 878                         kfree(swd, M_CRYPTO_DATA);
 879                 } else if (swcr_sesnum) {
 880                         bcopy(swcr_sessions, swd,
 881                               swcr_sesnum * sizeof(struct swcr_data *));
 882                         oswd = swcr_sessions;
 883                         swcr_sessions = swd;
 884                         swcr_sesnum = n;
 885                         spin_unlock(&swcr_spin);
 886                         kfree(oswd, M_CRYPTO_DATA);
 887                 } else {
 888                         swcr_sessions = swd;
 889                         swcr_sesnum = n;
 890                         spin_unlock(&swcr_spin);
 891                 }
 892         }
 893
 894         *sid = i;
 895         return 0;
 896 }
 897
 898 /*
 899  * Free a session.
 900  */
 901 static int
 902 swcr_freesession(device_t dev, u_int64_t tid)
 903 {
 904         u_int32_t sid = CRYPTO_SESID2LID(tid);
 905
 906         if (sid > swcr_sesnum || swcr_sessions == NULL ||
 907             swcr_sessions[sid] == NULL) {
 908                 return EINVAL;
 909         }
 910
 911         /* Silently accept and return */
 912         if (sid == 0)
 913                 return 0;
 914
 915         return(swcr_freesession_slot(&swcr_sessions[sid], sid));
 916 }
 917
 918 static
 919 int
 920 swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid)
 921 {
 922         struct enc_xform *txf;
 923         struct auth_hash *axf;
 924         struct comp_algo *cxf;
 925         struct swcr_data *swd;
 926         struct swcr_data *swnext;
 927
 928         /*
 929          * Protect session detachment with the spinlock.
 930          */
 931         spin_lock(&swcr_spin);
 932         swnext = *swdp;
 933         *swdp = NULL;
 934         if (sid && swcr_minsesnum > sid)
 935                 swcr_minsesnum = sid;
 936         spin_unlock(&swcr_spin);
 937
 938         /*
 939          * Clean up at our leisure.
 940          */
 941         while ((swd = swnext) != NULL) {
 942                 swnext = swd->sw_next;
 943
 944                 swd->sw_next = NULL;
 945
 946                 switch (swd->sw_alg) {
 947                 case CRYPTO_DES_CBC:
 948                 case CRYPTO_3DES_CBC:
 949                 case CRYPTO_BLF_CBC:
 950                 case CRYPTO_CAST_CBC:
 951                 case CRYPTO_SKIPJACK_CBC:
 952                 case CRYPTO_RIJNDAEL128_CBC:
 953                 case CRYPTO_AES_XTS:
 954                 case CRYPTO_AES_CTR:
 955                 case CRYPTO_CAMELLIA_CBC:
 956                 case CRYPTO_NULL_CBC:
 957                         txf = swd->sw_exf;
 958
 959                         if (swd->sw_kschedule)
 960                                 txf->zerokey(&(swd->sw_kschedule));
 961                         break;
 962
 963                 case CRYPTO_MD5_HMAC:
 964                 case CRYPTO_SHA1_HMAC:
 965                 case CRYPTO_SHA2_256_HMAC:
 966                 case CRYPTO_SHA2_384_HMAC:
 967                 case CRYPTO_SHA2_512_HMAC:
 968                 case CRYPTO_RIPEMD160_HMAC:
 969                 case CRYPTO_NULL_HMAC:
 970                         axf = swd->sw_axf;
 971
 972                         if (swd->sw_ictx) {
 973                                 bzero(swd->sw_ictx, axf->ctxsize);
 974                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
 975                         }
 976                         if (swd->sw_octx) {
 977                                 bzero(swd->sw_octx, axf->ctxsize);
 978                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
 979                         }
 980                         break;
 981
 982                 case CRYPTO_MD5_KPDK:
 983                 case CRYPTO_SHA1_KPDK:
 984                         axf = swd->sw_axf;
 985
 986                         if (swd->sw_ictx) {
 987                                 bzero(swd->sw_ictx, axf->ctxsize);
 988                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
 989                         }
 990                         if (swd->sw_octx) {
 991                                 bzero(swd->sw_octx, swd->sw_klen);
 992                                 kfree(swd->sw_octx, M_CRYPTO_DATA);
 993                         }
 994                         break;
 995
 996                 case CRYPTO_MD5:
 997                 case CRYPTO_SHA1:
 998                         axf = swd->sw_axf;
 999
1000                         if (swd->sw_ictx) {
1001                                 bzero(swd->sw_ictx, axf->ctxsize);
1002                                 kfree(swd->sw_ictx, M_CRYPTO_DATA);
1003                         }
1004                         break;
1005
1006                 case CRYPTO_DEFLATE_COMP:
1007                         cxf = swd->sw_cxf;
1008                         break;
1009                 }
1010
1011                 //FREE(swd, M_CRYPTO_DATA);
1012                 kfree(swd, M_CRYPTO_DATA);
1013         }
1014         return 0;
1015 }
1016
1017 /*
1018  * Process a software request.
1019  */
1020 static int
1021 swcr_process(device_t dev, struct cryptop *crp, int hint)
1022 {
1023         struct cryptodesc *crd;
1024         struct swcr_data *sw;
1025         u_int32_t lid;
1026
1027         /* Sanity check */
1028         if (crp == NULL)
1029                 return EINVAL;
1030
1031         if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
1032                 crp->crp_etype = EINVAL;
1033                 goto done;
1034         }
1035
1036         lid = crp->crp_sid & 0xffffffff;
1037         if (lid >= swcr_sesnum || lid == 0 || swcr_sessions[lid] == NULL) {
1038                 crp->crp_etype = ENOENT;
1039                 goto done;
1040         }
1041
1042         /* Go through crypto descriptors, processing as we go */
1043         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1044                 /*
1045                  * Find the crypto context.
1046                  *
1047                  * XXX Note that the logic here prevents us from having
1048                  * XXX the same algorithm multiple times in a session
1049                  * XXX (or rather, we can but it won't give us the right
1050                  * XXX results). To do that, we'd need some way of differentiating
1051                  * XXX between the various instances of an algorithm (so we can
1052                  * XXX locate the correct crypto context).
1053                  */
1054                 for (sw = swcr_sessions[lid];
1055                     sw && sw->sw_alg != crd->crd_alg;
1056                     sw = sw->sw_next)
1057                         ;
1058
1059                 /* No such context ? */
1060                 if (sw == NULL) {
1061                         crp->crp_etype = EINVAL;
1062                         goto done;
1063                 }
1064                 switch (sw->sw_alg) {
1065                 case CRYPTO_DES_CBC:
1066                 case CRYPTO_3DES_CBC:
1067                 case CRYPTO_BLF_CBC:
1068                 case CRYPTO_CAST_CBC:
1069                 case CRYPTO_SKIPJACK_CBC:
1070                 case CRYPTO_RIJNDAEL128_CBC:
1071                 case CRYPTO_AES_XTS:
1072                 case CRYPTO_AES_CTR:
1073                 case CRYPTO_CAMELLIA_CBC:
1074                         if ((crp->crp_etype = swcr_encdec(crd, sw,
1075                             crp->crp_buf, crp->crp_flags)) != 0)
1076                                 goto done;
1077                         break;
1078                 case CRYPTO_NULL_CBC:
1079                         crp->crp_etype = 0;
1080                         break;
1081                 case CRYPTO_MD5_HMAC:
1082                 case CRYPTO_SHA1_HMAC:
1083                 case CRYPTO_SHA2_256_HMAC:
1084                 case CRYPTO_SHA2_384_HMAC:
1085                 case CRYPTO_SHA2_512_HMAC:
1086                 case CRYPTO_RIPEMD160_HMAC:
1087                 case CRYPTO_NULL_HMAC:
1088                 case CRYPTO_MD5_KPDK:
1089                 case CRYPTO_SHA1_KPDK:
1090                 case CRYPTO_MD5:
1091                 case CRYPTO_SHA1:
1092                         if ((crp->crp_etype = swcr_authcompute(crd, sw,
1093                             crp->crp_buf, crp->crp_flags)) != 0)
1094                                 goto done;
1095                         break;
1096
1097                 case CRYPTO_DEFLATE_COMP:
1098                         if ((crp->crp_etype = swcr_compdec(crd, sw,
1099                             crp->crp_buf, crp->crp_flags)) != 0)
1100                                 goto done;
1101                         else
1102                                 crp->crp_olen = (int)sw->sw_size;
1103                         break;
1104
1105                 default:
1106                         /* Unknown/unsupported algorithm */
1107                         crp->crp_etype = EINVAL;
1108                         goto done;
1109                 }
1110         }
1111
1112 done:
1113         crypto_done(crp);
1114         lwkt_yield();
1115         return 0;
1116 }
1117
1118 static void
1119 swcr_identify(driver_t *drv, device_t parent)
1120 {
1121         /* NB: order 10 is so we get attached after h/w devices */
1122         /* XXX: wouldn't bet about this BUS_ADD_CHILD correctness */
1123         if (device_find_child(parent, "cryptosoft", -1) == NULL &&
1124             BUS_ADD_CHILD(parent, parent, 10, "cryptosoft", -1) == 0)
1125                 panic("cryptosoft: could not attach");
1126 }
1127
1128 static int
1129 swcr_probe(device_t dev)
1130 {
1131         device_set_desc(dev, "software crypto");
1132         return (0);
1133 }
1134
1135 static int
1136 swcr_attach(device_t dev)
1137 {
1138         memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
1139         memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
1140
1141         swcr_id = crypto_get_driverid(dev, CRYPTOCAP_F_SOFTWARE |
1142                                            CRYPTOCAP_F_SYNC |
1143                                            CRYPTOCAP_F_SMP);
1144         if (swcr_id < 0) {
1145                 device_printf(dev, "cannot initialize!");
1146                 return ENOMEM;
1147         }
1148 #define REGISTER(alg) \
1149         crypto_register(swcr_id, alg, 0,0)
1150         REGISTER(CRYPTO_DES_CBC);
1151         REGISTER(CRYPTO_3DES_CBC);
1152         REGISTER(CRYPTO_BLF_CBC);
1153         REGISTER(CRYPTO_CAST_CBC);
1154         REGISTER(CRYPTO_SKIPJACK_CBC);
1155         REGISTER(CRYPTO_NULL_CBC);
1156         REGISTER(CRYPTO_MD5_HMAC);
1157         REGISTER(CRYPTO_SHA1_HMAC);
1158         REGISTER(CRYPTO_SHA2_256_HMAC);
1159         REGISTER(CRYPTO_SHA2_384_HMAC);
1160         REGISTER(CRYPTO_SHA2_512_HMAC);
1161         REGISTER(CRYPTO_RIPEMD160_HMAC);
1162         REGISTER(CRYPTO_NULL_HMAC);
1163         REGISTER(CRYPTO_MD5_KPDK);
1164         REGISTER(CRYPTO_SHA1_KPDK);
1165         REGISTER(CRYPTO_MD5);
1166         REGISTER(CRYPTO_SHA1);
1167         REGISTER(CRYPTO_RIJNDAEL128_CBC);
1168         REGISTER(CRYPTO_AES_XTS);
1169         REGISTER(CRYPTO_AES_CTR);
1170         REGISTER(CRYPTO_CAMELLIA_CBC);
1171         REGISTER(CRYPTO_DEFLATE_COMP);
1172 #undef REGISTER
1173
1174         return 0;
1175 }
1176
1177 static int
1178 swcr_detach(device_t dev)
1179 {
1180         crypto_unregister_all(swcr_id);
1181         if (swcr_sessions != NULL)
1182                 kfree(swcr_sessions, M_CRYPTO_DATA);
1183         return 0;
1184 }
1185
1186 static device_method_t swcr_methods[] = {
1187         DEVMETHOD(device_identify,      swcr_identify),
1188         DEVMETHOD(device_probe,         swcr_probe),
1189         DEVMETHOD(device_attach,        swcr_attach),
1190         DEVMETHOD(device_detach,        swcr_detach),
1191
1192         DEVMETHOD(cryptodev_newsession, swcr_newsession),
1193         DEVMETHOD(cryptodev_freesession,swcr_freesession),
1194         DEVMETHOD(cryptodev_process,    swcr_process),
1195
1196         {0, 0},
1197 };
1198
1199 static driver_t swcr_driver = {
1200         "cryptosoft",
1201         swcr_methods,
1202         0,              /* NB: no softc */
1203 };
1204 static devclass_t swcr_devclass;
1205
1206 /*
1207  * NB: We explicitly reference the crypto module so we
1208  * get the necessary ordering when built as a loadable
1209  * module.  This is required because we bundle the crypto
1210  * module code together with the cryptosoft driver (otherwise
1211  * normal module dependencies would handle things).
1212  */
1213 extern int crypto_modevent(struct module *, int, void *);
1214 /* XXX where to attach */
1215 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,NULL);
1216 MODULE_VERSION(cryptosoft, 1);
1217 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);