sys/opencrypto/crypto.c

   1 /*      $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.7 2003/06/03 00:09:02 sam Exp $   */
   2 /*      $DragonFly: src/sys/opencrypto/crypto.c,v 1.12 2006/09/05 03:48:13 dillon Exp $ */
   3 /*      $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $        */
   4 /*
   5  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
   6  *
   7  * This code was written by Angelos D. Keromytis in Athens, Greece, in
   8  * February 2000. Network Security Technologies Inc. (NSTI) kindly
   9  * supported the development of this code.
  10  *
  11  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  12  *
  13  * Permission to use, copy, and modify this software with or without fee
  14  * is hereby granted, provided that this entire notice is included in
  15  * all source code copies of any software which is or includes a copy or
  16  * modification of this software.
  17  *
  18  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  19  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  20  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  21  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  22  * PURPOSE.
  23  */
  24
  25 #define CRYPTO_TIMING                   /* enable cryptop timing stuff */
  26
  27 #include <sys/param.h>
  28 #include <sys/systm.h>
  29 #include <sys/eventhandler.h>
  30 #include <sys/kernel.h>
  31 #include <sys/kthread.h>
  32 #include <sys/malloc.h>
  33 #include <sys/proc.h>
  34 #include <sys/sysctl.h>
  35 #include <sys/interrupt.h>
  36 #include <sys/thread2.h>
  37 #include <machine/ipl.h>
  38
  39 #include <vm/vm_zone.h>
  40 #include <opencrypto/cryptodev.h>
  41 #include <opencrypto/xform.h>                   /* XXX for M_XDATA */
  42
  43 #define SESID2HID(sid)  (((sid) >> 32) & 0xffffffff)
  44
  45 /*
  46  * Crypto drivers register themselves by allocating a slot in the
  47  * crypto_drivers table with crypto_get_driverid() and then registering
  48  * each algorithm they support with crypto_register() and crypto_kregister().
  49  */
  50 static  struct cryptocap *crypto_drivers = NULL;
  51 static  int crypto_drivers_num = 0;
  52
  53 /*
  54  * There are two queues for crypto requests; one for symmetric (e.g.
  55  * cipher) operations and one for asymmetric (e.g. MOD) operations.
  56  * See below for how synchronization is handled.
  57  */
  58 static  TAILQ_HEAD(,cryptop) crp_q;             /* request queues */
  59 static  TAILQ_HEAD(,cryptkop) crp_kq;
  60
  61 /*
  62  * There are two queues for processing completed crypto requests; one
  63  * for the symmetric and one for the asymmetric ops.  We only need one
  64  * but have two to avoid type futzing (cryptop vs. cryptkop).  See below
  65  * for how synchronization is handled.
  66  */
  67 static  TAILQ_HEAD(,cryptop) crp_ret_q;         /* callback queues */
  68 static  TAILQ_HEAD(,cryptkop) crp_ret_kq;
  69
  70 /*
  71  * Crypto op and desciptor data structures are allocated
  72  * from separate private zones.
  73  */
  74 static  vm_zone_t cryptop_zone;
  75 static  vm_zone_t cryptodesc_zone;
  76
  77 int     crypto_usercrypto = 1;          /* userland may open /dev/crypto */
  78 SYSCTL_INT(_kern, OID_AUTO, usercrypto, CTLFLAG_RW,
  79            &crypto_usercrypto, 0,
  80            "Enable/disable user-mode access to crypto support");
  81 int     crypto_userasymcrypto = 1;      /* userland may do asym crypto reqs */
  82 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
  83            &crypto_userasymcrypto, 0,
  84            "Enable/disable user-mode access to asymmetric crypto support");
  85 int     crypto_devallowsoft = 0;        /* only use hardware crypto for asym */
  86 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
  87            &crypto_devallowsoft, 0,
  88            "Enable/disable use of software asym crypto support");
  89
  90 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
  91
  92 /*
  93  * Synchronization: read carefully, this is non-trivial.
  94  *
  95  * Crypto requests are submitted via crypto_dispatch.  No critical
  96  * section or lock/interlock guarentees are made on entry.
  97  *
  98  * Requests are typically passed on the driver directly, but they
  99  * may also be queued for processing by a software interrupt thread,
 100  * cryptointr, that runs in a critical section.  This thread dispatches
 101  * the requests to crypto drivers (h/w or s/w) who call crypto_done
 102  * when a request is complete.  Hardware crypto drivers are assumed
 103  * to register their IRQ's as network devices so their interrupt handlers
 104  * and subsequent "done callbacks" happen at appropriate protection levels.
 105  *
 106  * Completed crypto ops are queued for a separate kernel thread that
 107  * handles the callbacks with no critical section or lock/interlock
 108  * guarentees.  This decoupling insures the crypto driver interrupt service
 109  * routine is not delayed while the callback takes place and that callbacks
 110  * are delivered after a context switch (as opposed to a software interrupt
 111  * that clients must block).
 112  *
 113  * This scheme is not intended for SMP machines.
 114  */
 115 static inthand2_t cryptointr;
 116 static  void cryptoret(void);           /* kernel thread for callbacks*/
 117 static  struct thread *cryptothread;
 118 static  void crypto_destroy(void);
 119 static  int crypto_invoke(struct cryptop *crp, int hint);
 120 static  int crypto_kinvoke(struct cryptkop *krp, int hint);
 121
 122 static struct cryptostats cryptostats;
 123 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
 124             cryptostats, "Crypto system statistics");
 125
 126 #ifdef CRYPTO_TIMING
 127 static  int crypto_timing = 0;
 128 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
 129            &crypto_timing, 0, "Enable/disable crypto timing support");
 130 #endif
 131
 132 static void *crypto_int_id;
 133
 134 static int
 135 crypto_init(void)
 136 {
 137         int error;
 138
 139         cryptop_zone = zinit("cryptop", sizeof (struct cryptop), 0, 0, 1);
 140         cryptodesc_zone = zinit("cryptodesc", sizeof (struct cryptodesc),
 141                                 0, 0, 1);
 142         if (cryptodesc_zone == NULL || cryptop_zone == NULL) {
 143                 printf("crypto_init: cannot setup crypto zones\n");
 144                 return ENOMEM;
 145         }
 146
 147         crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
 148         crypto_drivers = kmalloc(crypto_drivers_num *
 149             sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
 150         if (crypto_drivers == NULL) {
 151                 printf("crypto_init: cannot malloc driver table\n");
 152                 return ENOMEM;
 153         }
 154
 155         TAILQ_INIT(&crp_q);
 156         TAILQ_INIT(&crp_kq);
 157
 158         TAILQ_INIT(&crp_ret_q);
 159         TAILQ_INIT(&crp_ret_kq);
 160
 161         crypto_int_id = register_swi(SWI_CRYPTO, cryptointr, NULL,
 162                                         "swi_crypto", NULL);
 163         error = kthread_create((void (*)(void *)) cryptoret, NULL,
 164                     &cryptothread, "cryptoret");
 165         if (error) {
 166                 printf("crypto_init: cannot start cryptoret thread; error %d",
 167                         error);
 168                 crypto_destroy();
 169         }
 170         return error;
 171 }
 172
 173 static void
 174 crypto_destroy(void)
 175 {
 176         /* XXX no wait to reclaim zones */
 177         if (crypto_drivers != NULL)
 178                 kfree(crypto_drivers, M_CRYPTO_DATA);
 179         unregister_swi(crypto_int_id);
 180 }
 181
 182 /*
 183  * Initialization code, both for static and dynamic loading.
 184  */
 185 static int
 186 crypto_modevent(module_t mod, int type, void *unused)
 187 {
 188         int error = EINVAL;
 189
 190         switch (type) {
 191         case MOD_LOAD:
 192                 error = crypto_init();
 193                 if (error == 0 && bootverbose)
 194                         printf("crypto: <crypto core>\n");
 195                 break;
 196         case MOD_UNLOAD:
 197                 /*XXX disallow if active sessions */
 198                 error = 0;
 199                 crypto_destroy();
 200                 break;
 201         }
 202         return error;
 203 }
 204
 205 static moduledata_t crypto_mod = {
 206         "crypto",
 207         crypto_modevent,
 208         0
 209 };
 210 MODULE_VERSION(crypto, 1);
 211 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
 212
 213 /*
 214  * Create a new session.
 215  */
 216 int
 217 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard)
 218 {
 219         struct cryptoini *cr;
 220         u_int32_t hid, lid;
 221         int err = EINVAL;
 222
 223         crit_enter();
 224
 225         if (crypto_drivers == NULL)
 226                 goto done;
 227
 228         /*
 229          * The algorithm we use here is pretty stupid; just use the
 230          * first driver that supports all the algorithms we need.
 231          *
 232          * XXX We need more smarts here (in real life too, but that's
 233          * XXX another story altogether).
 234          */
 235
 236         for (hid = 0; hid < crypto_drivers_num; hid++) {
 237                 /*
 238                  * If it's not initialized or has remaining sessions
 239                  * referencing it, skip.
 240                  */
 241                 if (crypto_drivers[hid].cc_newsession == NULL ||
 242                     (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP))
 243                         continue;
 244
 245                 /* Hardware required -- ignore software drivers. */
 246                 if (hard > 0 &&
 247                     (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE))
 248                         continue;
 249                 /* Software required -- ignore hardware drivers. */
 250                 if (hard < 0 &&
 251                     (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0)
 252                         continue;
 253
 254                 /* See if all the algorithms are supported. */
 255                 for (cr = cri; cr; cr = cr->cri_next)
 256                         if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0)
 257                                 break;
 258
 259                 if (cr == NULL) {
 260                         /* Ok, all algorithms are supported. */
 261
 262                         /*
 263                          * Can't do everything in one session.
 264                          *
 265                          * XXX Fix this. We need to inject a "virtual" session layer right
 266                          * XXX about here.
 267                          */
 268
 269                         /* Call the driver initialization routine. */
 270                         lid = hid;              /* Pass the driver ID. */
 271                         err = crypto_drivers[hid].cc_newsession(
 272                                         crypto_drivers[hid].cc_arg, &lid, cri);
 273                         if (err == 0) {
 274                                 (*sid) = hid;
 275                                 (*sid) <<= 32;
 276                                 (*sid) |= (lid & 0xffffffff);
 277                                 crypto_drivers[hid].cc_sessions++;
 278                         }
 279                         break;
 280                 }
 281         }
 282 done:
 283         crit_exit();
 284         return err;
 285 }
 286
 287 /*
 288  * Delete an existing session (or a reserved session on an unregistered
 289  * driver).
 290  */
 291 int
 292 crypto_freesession(u_int64_t sid)
 293 {
 294         u_int32_t hid;
 295         int err;
 296
 297         crit_enter();
 298
 299         if (crypto_drivers == NULL) {
 300                 err = EINVAL;
 301                 goto done;
 302         }
 303
 304         /* Determine two IDs. */
 305         hid = SESID2HID(sid);
 306
 307         if (hid >= crypto_drivers_num) {
 308                 err = ENOENT;
 309                 goto done;
 310         }
 311
 312         if (crypto_drivers[hid].cc_sessions)
 313                 crypto_drivers[hid].cc_sessions--;
 314
 315         /* Call the driver cleanup routine, if available. */
 316         if (crypto_drivers[hid].cc_freesession)
 317                 err = crypto_drivers[hid].cc_freesession(
 318                                 crypto_drivers[hid].cc_arg, sid);
 319         else
 320                 err = 0;
 321
 322         /*
 323          * If this was the last session of a driver marked as invalid,
 324          * make the entry available for reuse.
 325          */
 326         if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) &&
 327             crypto_drivers[hid].cc_sessions == 0)
 328                 bzero(&crypto_drivers[hid], sizeof(struct cryptocap));
 329
 330 done:
 331         crit_exit();
 332         return err;
 333 }
 334
 335 /*
 336  * Return an unused driver id.  Used by drivers prior to registering
 337  * support for the algorithms they handle.
 338  */
 339 int32_t
 340 crypto_get_driverid(u_int32_t flags)
 341 {
 342         struct cryptocap *newdrv;
 343         int i;
 344
 345         crit_enter();
 346         for (i = 0; i < crypto_drivers_num; i++)
 347                 if (crypto_drivers[i].cc_process == NULL &&
 348                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 &&
 349                     crypto_drivers[i].cc_sessions == 0)
 350                         break;
 351
 352         /* Out of entries, allocate some more. */
 353         if (i == crypto_drivers_num) {
 354                 /* Be careful about wrap-around. */
 355                 if (2 * crypto_drivers_num <= crypto_drivers_num) {
 356                         crit_exit();
 357                         printf("crypto: driver count wraparound!\n");
 358                         return -1;
 359                 }
 360
 361                 newdrv = kmalloc(2 * crypto_drivers_num *
 362                     sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 363                 if (newdrv == NULL) {
 364                         crit_exit();
 365                         printf("crypto: no space to expand driver table!\n");
 366                         return -1;
 367                 }
 368
 369                 bcopy(crypto_drivers, newdrv,
 370                     crypto_drivers_num * sizeof(struct cryptocap));
 371
 372                 crypto_drivers_num *= 2;
 373
 374                 kfree(crypto_drivers, M_CRYPTO_DATA);
 375                 crypto_drivers = newdrv;
 376         }
 377
 378         /* NB: state is zero'd on free */
 379         crypto_drivers[i].cc_sessions = 1;      /* Mark */
 380         crypto_drivers[i].cc_flags = flags;
 381         if (bootverbose)
 382                 printf("crypto: assign driver %u, flags %u\n", i, flags);
 383
 384         crit_exit();
 385
 386         return i;
 387 }
 388
 389 static struct cryptocap *
 390 crypto_checkdriver(u_int32_t hid)
 391 {
 392         if (crypto_drivers == NULL)
 393                 return NULL;
 394         return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
 395 }
 396
 397 /*
 398  * Register support for a key-related algorithm.  This routine
 399  * is called once for each algorithm supported a driver.
 400  */
 401 int
 402 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags,
 403     int (*kprocess)(void*, struct cryptkop *, int),
 404     void *karg)
 405 {
 406         struct cryptocap *cap;
 407         int err;
 408
 409         crit_enter();
 410
 411         cap = crypto_checkdriver(driverid);
 412         if (cap != NULL &&
 413             (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
 414                 /*
 415                  * XXX Do some performance testing to determine placing.
 416                  * XXX We probably need an auxiliary data structure that
 417                  * XXX describes relative performances.
 418                  */
 419
 420                 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 421                 if (bootverbose)
 422                         printf("crypto: driver %u registers key alg %u flags %u\n"
 423                                 , driverid
 424                                 , kalg
 425                                 , flags
 426                         );
 427
 428                 if (cap->cc_kprocess == NULL) {
 429                         cap->cc_karg = karg;
 430                         cap->cc_kprocess = kprocess;
 431                 }
 432                 err = 0;
 433         } else
 434                 err = EINVAL;
 435
 436         crit_exit();
 437         return err;
 438 }
 439
 440 /*
 441  * Register support for a non-key-related algorithm.  This routine
 442  * is called once for each such algorithm supported by a driver.
 443  */
 444 int
 445 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
 446     u_int32_t flags,
 447     int (*newses)(void*, u_int32_t*, struct cryptoini*),
 448     int (*freeses)(void*, u_int64_t),
 449     int (*process)(void*, struct cryptop *, int),
 450     void *arg)
 451 {
 452         struct cryptocap *cap;
 453         int err;
 454
 455         crit_enter();
 456
 457         cap = crypto_checkdriver(driverid);
 458         /* NB: algorithms are in the range [1..max] */
 459         if (cap != NULL &&
 460             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
 461                 /*
 462                  * XXX Do some performance testing to determine placing.
 463                  * XXX We probably need an auxiliary data structure that
 464                  * XXX describes relative performances.
 465                  */
 466
 467                 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 468                 cap->cc_max_op_len[alg] = maxoplen;
 469                 if (bootverbose)
 470                         printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n"
 471                                 , driverid
 472                                 , alg
 473                                 , flags
 474                                 , maxoplen
 475                         );
 476
 477                 if (cap->cc_process == NULL) {
 478                         cap->cc_arg = arg;
 479                         cap->cc_newsession = newses;
 480                         cap->cc_process = process;
 481                         cap->cc_freesession = freeses;
 482                         cap->cc_sessions = 0;           /* Unmark */
 483                 }
 484                 err = 0;
 485         } else
 486                 err = EINVAL;
 487
 488         crit_exit();
 489         return err;
 490 }
 491
 492 /*
 493  * Unregister a crypto driver. If there are pending sessions using it,
 494  * leave enough information around so that subsequent calls using those
 495  * sessions will correctly detect the driver has been unregistered and
 496  * reroute requests.
 497  */
 498 int
 499 crypto_unregister(u_int32_t driverid, int alg)
 500 {
 501         int i, err;
 502         u_int32_t ses;
 503         struct cryptocap *cap;
 504
 505         crit_enter();
 506
 507         cap = crypto_checkdriver(driverid);
 508         if (cap != NULL &&
 509             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
 510             cap->cc_alg[alg] != 0) {
 511                 cap->cc_alg[alg] = 0;
 512                 cap->cc_max_op_len[alg] = 0;
 513
 514                 /* Was this the last algorithm ? */
 515                 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
 516                         if (cap->cc_alg[i] != 0)
 517                                 break;
 518
 519                 if (i == CRYPTO_ALGORITHM_MAX + 1) {
 520                         ses = cap->cc_sessions;
 521                         bzero(cap, sizeof(struct cryptocap));
 522                         if (ses != 0) {
 523                                 /*
 524                                  * If there are pending sessions, just mark as invalid.
 525                                  */
 526                                 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
 527                                 cap->cc_sessions = ses;
 528                         }
 529                 }
 530                 err = 0;
 531         } else
 532                 err = EINVAL;
 533
 534         crit_exit();
 535         return err;
 536 }
 537
 538 /*
 539  * Unregister all algorithms associated with a crypto driver.
 540  * If there are pending sessions using it, leave enough information
 541  * around so that subsequent calls using those sessions will
 542  * correctly detect the driver has been unregistered and reroute
 543  * requests.
 544  */
 545 int
 546 crypto_unregister_all(u_int32_t driverid)
 547 {
 548         int i, err;
 549         u_int32_t ses;
 550         struct cryptocap *cap;
 551
 552         crit_enter();
 553         cap = crypto_checkdriver(driverid);
 554         if (cap != NULL) {
 555                 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) {
 556                         cap->cc_alg[i] = 0;
 557                         cap->cc_max_op_len[i] = 0;
 558                 }
 559                 ses = cap->cc_sessions;
 560                 bzero(cap, sizeof(struct cryptocap));
 561                 if (ses != 0) {
 562                         /*
 563                          * If there are pending sessions, just mark as invalid.
 564                          */
 565                         cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
 566                         cap->cc_sessions = ses;
 567                 }
 568                 err = 0;
 569         } else
 570                 err = EINVAL;
 571
 572         crit_exit();
 573         return err;
 574 }
 575
 576 /*
 577  * Clear blockage on a driver.  The what parameter indicates whether
 578  * the driver is now ready for cryptop's and/or cryptokop's.
 579  */
 580 int
 581 crypto_unblock(u_int32_t driverid, int what)
 582 {
 583         struct cryptocap *cap;
 584         int needwakeup, err;
 585
 586         crit_enter();
 587         cap = crypto_checkdriver(driverid);
 588         if (cap != NULL) {
 589                 needwakeup = 0;
 590                 if (what & CRYPTO_SYMQ) {
 591                         needwakeup |= cap->cc_qblocked;
 592                         cap->cc_qblocked = 0;
 593                 }
 594                 if (what & CRYPTO_ASYMQ) {
 595                         needwakeup |= cap->cc_kqblocked;
 596                         cap->cc_kqblocked = 0;
 597                 }
 598                 if (needwakeup)
 599                         setsoftcrypto();
 600                 err = 0;
 601         } else
 602                 err = EINVAL;
 603         crit_exit();
 604
 605         return err;
 606 }
 607
 608 /*
 609  * Dispatch a crypto request to a driver or queue
 610  * it, to be processed by the kernel thread.
 611  */
 612 int
 613 crypto_dispatch(struct cryptop *crp)
 614 {
 615         u_int32_t hid = SESID2HID(crp->crp_sid);
 616         int result;
 617
 618         cryptostats.cs_ops++;
 619
 620 #ifdef CRYPTO_TIMING
 621         if (crypto_timing)
 622                 nanouptime(&crp->crp_tstamp);
 623 #endif
 624         crit_enter();
 625         if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
 626                 struct cryptocap *cap;
 627                 /*
 628                  * Caller marked the request to be processed
 629                  * immediately; dispatch it directly to the
 630                  * driver unless the driver is currently blocked.
 631                  */
 632                 cap = crypto_checkdriver(hid);
 633                 if (cap && !cap->cc_qblocked) {
 634                         result = crypto_invoke(crp, 0);
 635                         if (result == ERESTART) {
 636                                 /*
 637                                  * The driver ran out of resources, mark the
 638                                  * driver ``blocked'' for cryptop's and put
 639                                  * the op on the queue.
 640                                  */
 641                                 crypto_drivers[hid].cc_qblocked = 1;
 642                                 TAILQ_INSERT_HEAD(&crp_q, crp, crp_next);
 643                                 cryptostats.cs_blocks++;
 644                                 result = 0;
 645                         }
 646                 } else {
 647                         /*
 648                          * The driver is blocked, just queue the op until
 649                          * it unblocks and the swi thread gets kicked.
 650                          */
 651                         TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 652                         result = 0;
 653                 }
 654         } else {
 655                 int wasempty = TAILQ_EMPTY(&crp_q);
 656                 /*
 657                  * Caller marked the request as ``ok to delay'';
 658                  * queue it for the swi thread.  This is desirable
 659                  * when the operation is low priority and/or suitable
 660                  * for batching.
 661                  */
 662                 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 663                 if (wasempty)
 664                         setsoftcrypto();
 665                 result = 0;
 666         }
 667         crit_exit();
 668
 669         return result;
 670 }
 671
 672 /*
 673  * Add an asymetric crypto request to a queue,
 674  * to be processed by the kernel thread.
 675  */
 676 int
 677 crypto_kdispatch(struct cryptkop *krp)
 678 {
 679         struct cryptocap *cap;
 680         int result;
 681
 682         cryptostats.cs_kops++;
 683
 684         crit_enter();
 685         cap = crypto_checkdriver(krp->krp_hid);
 686         if (cap && !cap->cc_kqblocked) {
 687                 result = crypto_kinvoke(krp, 0);
 688                 if (result == ERESTART) {
 689                         /*
 690                          * The driver ran out of resources, mark the
 691                          * driver ``blocked'' for cryptop's and put
 692                          * the op on the queue.
 693                          */
 694                         crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
 695                         TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
 696                         cryptostats.cs_kblocks++;
 697                 }
 698         } else {
 699                 /*
 700                  * The driver is blocked, just queue the op until
 701                  * it unblocks and the swi thread gets kicked.
 702                  */
 703                 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
 704                 result = 0;
 705         }
 706         crit_exit();
 707
 708         return result;
 709 }
 710
 711 /*
 712  * Dispatch an assymetric crypto request to the appropriate crypto devices.
 713  */
 714 static int
 715 crypto_kinvoke(struct cryptkop *krp, int hint)
 716 {
 717         u_int32_t hid;
 718         int error;
 719
 720         /* Sanity checks. */
 721         if (krp == NULL)
 722                 return EINVAL;
 723         if (krp->krp_callback == NULL) {
 724                 kfree(krp, M_XDATA);            /* XXX allocated in cryptodev */
 725                 return EINVAL;
 726         }
 727
 728         for (hid = 0; hid < crypto_drivers_num; hid++) {
 729                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
 730                     !crypto_devallowsoft)
 731                         continue;
 732                 if (crypto_drivers[hid].cc_kprocess == NULL)
 733                         continue;
 734                 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] &
 735                     CRYPTO_ALG_FLAG_SUPPORTED) == 0)
 736                         continue;
 737                 break;
 738         }
 739         if (hid < crypto_drivers_num) {
 740                 krp->krp_hid = hid;
 741                 error = crypto_drivers[hid].cc_kprocess(
 742                                 crypto_drivers[hid].cc_karg, krp, hint);
 743         } else
 744                 error = ENODEV;
 745
 746         if (error) {
 747                 krp->krp_status = error;
 748                 crypto_kdone(krp);
 749         }
 750         return 0;
 751 }
 752
 753 #ifdef CRYPTO_TIMING
 754 static void
 755 crypto_tstat(struct cryptotstat *ts, struct timespec *tv)
 756 {
 757         struct timespec now, t;
 758
 759         nanouptime(&now);
 760         t.tv_sec = now.tv_sec - tv->tv_sec;
 761         t.tv_nsec = now.tv_nsec - tv->tv_nsec;
 762         if (t.tv_nsec < 0) {
 763                 t.tv_sec--;
 764                 t.tv_nsec += 1000000000;
 765         }
 766         timespecadd(&ts->acc, &t);
 767         if (timespeccmp(&t, &ts->min, <))
 768                 ts->min = t;
 769         if (timespeccmp(&t, &ts->max, >))
 770                 ts->max = t;
 771         ts->count++;
 772
 773         *tv = now;
 774 }
 775 #endif
 776
 777 /*
 778  * Dispatch a crypto request to the appropriate crypto devices.
 779  */
 780 static int
 781 crypto_invoke(struct cryptop *crp, int hint)
 782 {
 783         u_int32_t hid;
 784         int (*process)(void*, struct cryptop *, int);
 785
 786 #ifdef CRYPTO_TIMING
 787         if (crypto_timing)
 788                 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
 789 #endif
 790         /* Sanity checks. */
 791         if (crp == NULL)
 792                 return EINVAL;
 793         if (crp->crp_callback == NULL) {
 794                 crypto_freereq(crp);
 795                 return EINVAL;
 796         }
 797         if (crp->crp_desc == NULL) {
 798                 crp->crp_etype = EINVAL;
 799                 crypto_done(crp);
 800                 return 0;
 801         }
 802
 803         hid = SESID2HID(crp->crp_sid);
 804         if (hid < crypto_drivers_num) {
 805                 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)
 806                         crypto_freesession(crp->crp_sid);
 807                 process = crypto_drivers[hid].cc_process;
 808         } else {
 809                 process = NULL;
 810         }
 811
 812         if (process == NULL) {
 813                 struct cryptodesc *crd;
 814                 u_int64_t nid;
 815
 816                 /*
 817                  * Driver has unregistered; migrate the session and return
 818                  * an error to the caller so they'll resubmit the op.
 819                  */
 820                 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
 821                         crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
 822
 823                 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0)
 824                         crp->crp_sid = nid;
 825
 826                 crp->crp_etype = EAGAIN;
 827                 crypto_done(crp);
 828                 return 0;
 829         } else {
 830                 /*
 831                  * Invoke the driver to process the request.
 832                  */
 833                 return (*process)(crypto_drivers[hid].cc_arg, crp, hint);
 834         }
 835 }
 836
 837 /*
 838  * Release a set of crypto descriptors.
 839  */
 840 void
 841 crypto_freereq(struct cryptop *crp)
 842 {
 843         struct cryptodesc *crd;
 844
 845         if (crp) {
 846                 while ((crd = crp->crp_desc) != NULL) {
 847                         crp->crp_desc = crd->crd_next;
 848                         zfree(cryptodesc_zone, crd);
 849                 }
 850                 zfree(cryptop_zone, crp);
 851         }
 852 }
 853
 854 /*
 855  * Acquire a set of crypto descriptors.  The descriptors are self contained
 856  * so no special lock/interlock protection is necessary.
 857  */
 858 struct cryptop *
 859 crypto_getreq(int num)
 860 {
 861         struct cryptodesc *crd;
 862         struct cryptop *crp;
 863
 864         crp = zalloc(cryptop_zone);
 865         if (crp != NULL) {
 866                 bzero(crp, sizeof (*crp));
 867                 while (num--) {
 868                         crd = zalloc(cryptodesc_zone);
 869                         if (crd == NULL) {
 870                                 crypto_freereq(crp);
 871                                 crp = NULL;
 872                                 break;
 873                         }
 874                         bzero(crd, sizeof (*crd));
 875                         crd->crd_next = crp->crp_desc;
 876                         crp->crp_desc = crd;
 877                 }
 878         }
 879         return crp;
 880 }
 881
 882 /*
 883  * Invoke the callback on behalf of the driver.
 884  */
 885 void
 886 crypto_done(struct cryptop *crp)
 887 {
 888         KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
 889                 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
 890         crp->crp_flags |= CRYPTO_F_DONE;
 891         if (crp->crp_etype != 0)
 892                 cryptostats.cs_errs++;
 893 #ifdef CRYPTO_TIMING
 894         if (crypto_timing)
 895                 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
 896 #endif
 897         if (crp->crp_flags & CRYPTO_F_CBIMM) {
 898                 /*
 899                  * Do the callback directly.  This is ok when the
 900                  * callback routine does very little (e.g. the
 901                  * /dev/crypto callback method just does a wakeup).
 902                  */
 903 #ifdef CRYPTO_TIMING
 904                 if (crypto_timing) {
 905                         /*
 906                          * NB: We must copy the timestamp before
 907                          * doing the callback as the cryptop is
 908                          * likely to be reclaimed.
 909                          */
 910                         struct timespec t = crp->crp_tstamp;
 911                         crypto_tstat(&cryptostats.cs_cb, &t);
 912                         crp->crp_callback(crp);
 913                         crypto_tstat(&cryptostats.cs_finis, &t);
 914                 } else
 915 #endif
 916                         crp->crp_callback(crp);
 917         } else {
 918                 int wasempty;
 919                 /*
 920                  * Normal case; queue the callback for the thread.
 921                  *
 922                  * The return queue is manipulated by the swi thread
 923                  * and, potentially, by crypto device drivers calling
 924                  * back to mark operations completed.  Thus we need
 925                  * to mask both while manipulating the return queue.
 926                  */
 927                 crit_enter();
 928                 wasempty = TAILQ_EMPTY(&crp_ret_q);
 929                 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
 930                 if (wasempty)
 931                         wakeup_one(&crp_ret_q);
 932                 crit_exit();
 933         }
 934 }
 935
 936 /*
 937  * Invoke the callback on behalf of the driver.
 938  */
 939 void
 940 crypto_kdone(struct cryptkop *krp)
 941 {
 942         int wasempty;
 943
 944         if (krp->krp_status != 0)
 945                 cryptostats.cs_kerrs++;
 946         /*
 947          * The return queue is manipulated by the swi thread
 948          * and, potentially, by crypto device drivers calling
 949          * back to mark operations completed.  Thus we need
 950          * to mask both while manipulating the return queue.
 951          */
 952         crit_enter();
 953         wasempty = TAILQ_EMPTY(&crp_ret_kq);
 954         TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
 955         if (wasempty)
 956                 wakeup_one(&crp_ret_q);
 957         crit_exit();
 958 }
 959
 960 int
 961 crypto_getfeat(int *featp)
 962 {
 963         int hid, kalg, feat = 0;
 964
 965         crit_enter();
 966         if (!crypto_userasymcrypto)
 967                 goto out;
 968
 969         for (hid = 0; hid < crypto_drivers_num; hid++) {
 970                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
 971                     !crypto_devallowsoft) {
 972                         continue;
 973                 }
 974                 if (crypto_drivers[hid].cc_kprocess == NULL)
 975                         continue;
 976                 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
 977                         if ((crypto_drivers[hid].cc_kalg[kalg] &
 978                             CRYPTO_ALG_FLAG_SUPPORTED) != 0)
 979                                 feat |=  1 << kalg;
 980         }
 981 out:
 982         crit_exit();
 983         *featp = feat;
 984         return (0);
 985 }
 986
 987 /*
 988  * Software interrupt thread to dispatch crypto requests.
 989  */
 990 static void
 991 cryptointr(void *dummy, void *frame)
 992 {
 993         struct cryptop *crp, *submit;
 994         struct cryptkop *krp;
 995         struct cryptocap *cap;
 996         int result, hint;
 997
 998         cryptostats.cs_intrs++;
 999         crit_enter();
1000         do {
1001                 /*
1002                  * Find the first element in the queue that can be
1003                  * processed and look-ahead to see if multiple ops
1004                  * are ready for the same driver.
1005                  */
1006                 submit = NULL;
1007                 hint = 0;
1008                 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1009                         u_int32_t hid = SESID2HID(crp->crp_sid);
1010                         cap = crypto_checkdriver(hid);
1011                         if (cap == NULL || cap->cc_process == NULL) {
1012                                 /* Op needs to be migrated, process it. */
1013                                 if (submit == NULL)
1014                                         submit = crp;
1015                                 break;
1016                         }
1017                         if (!cap->cc_qblocked) {
1018                                 if (submit != NULL) {
1019                                         /*
1020                                          * We stop on finding another op,
1021                                          * regardless whether its for the same
1022                                          * driver or not.  We could keep
1023                                          * searching the queue but it might be
1024                                          * better to just use a per-driver
1025                                          * queue instead.
1026                                          */
1027                                         if (SESID2HID(submit->crp_sid) == hid)
1028                                                 hint = CRYPTO_HINT_MORE;
1029                                         break;
1030                                 } else {
1031                                         submit = crp;
1032                                         if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1033                                                 break;
1034                                         /* keep scanning for more are q'd */
1035                                 }
1036                         }
1037                 }
1038                 if (submit != NULL) {
1039                         TAILQ_REMOVE(&crp_q, submit, crp_next);
1040                         result = crypto_invoke(submit, hint);
1041                         if (result == ERESTART) {
1042                                 /*
1043                                  * The driver ran out of resources, mark the
1044                                  * driver ``blocked'' for cryptop's and put
1045                                  * the request back in the queue.  It would
1046                                  * best to put the request back where we got
1047                                  * it but that's hard so for now we put it
1048                                  * at the front.  This should be ok; putting
1049                                  * it at the end does not work.
1050                                  */
1051                                 /* XXX validate sid again? */
1052                                 crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1;
1053                                 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1054                                 cryptostats.cs_blocks++;
1055                         }
1056                 }
1057
1058                 /* As above, but for key ops */
1059                 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1060                         cap = crypto_checkdriver(krp->krp_hid);
1061                         if (cap == NULL || cap->cc_kprocess == NULL) {
1062                                 /* Op needs to be migrated, process it. */
1063                                 break;
1064                         }
1065                         if (!cap->cc_kqblocked)
1066                                 break;
1067                 }
1068                 if (krp != NULL) {
1069                         TAILQ_REMOVE(&crp_kq, krp, krp_next);
1070                         result = crypto_kinvoke(krp, 0);
1071                         if (result == ERESTART) {
1072                                 /*
1073                                  * The driver ran out of resources, mark the
1074                                  * driver ``blocked'' for cryptkop's and put
1075                                  * the request back in the queue.  It would
1076                                  * best to put the request back where we got
1077                                  * it but that's hard so for now we put it
1078                                  * at the front.  This should be ok; putting
1079                                  * it at the end does not work.
1080                                  */
1081                                 /* XXX validate sid again? */
1082                                 crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1083                                 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1084                                 cryptostats.cs_kblocks++;
1085                         }
1086                 }
1087         } while (submit != NULL || krp != NULL);
1088         crit_exit();
1089 }
1090
1091 /*
1092  * Kernel thread to do callbacks.
1093  */
1094 static void
1095 cryptoret(void)
1096 {
1097         struct cryptop *crp;
1098         struct cryptkop *krp;
1099
1100         crit_enter();
1101         for (;;) {
1102                 crp = TAILQ_FIRST(&crp_ret_q);
1103                 if (crp != NULL)
1104                         TAILQ_REMOVE(&crp_ret_q, crp, crp_next);
1105                 krp = TAILQ_FIRST(&crp_ret_kq);
1106                 if (krp != NULL)
1107                         TAILQ_REMOVE(&crp_ret_kq, krp, krp_next);
1108
1109                 if (crp != NULL || krp != NULL) {
1110                         crit_exit();            /* lower ipl for callbacks */
1111                         if (crp != NULL) {
1112 #ifdef CRYPTO_TIMING
1113                                 if (crypto_timing) {
1114                                         /*
1115                                          * NB: We must copy the timestamp before
1116                                          * doing the callback as the cryptop is
1117                                          * likely to be reclaimed.
1118                                          */
1119                                         struct timespec t = crp->crp_tstamp;
1120                                         crypto_tstat(&cryptostats.cs_cb, &t);
1121                                         crp->crp_callback(crp);
1122                                         crypto_tstat(&cryptostats.cs_finis, &t);
1123                                 } else
1124 #endif
1125                                         crp->crp_callback(crp);
1126                         }
1127                         if (krp != NULL)
1128                                 krp->krp_callback(krp);
1129                         crit_enter();
1130                 } else {
1131                         (void) tsleep(&crp_ret_q, 0, "crypto_wait", 0);
1132                         cryptostats.cs_rets++;
1133                 }
1134         }
1135         /* CODE NOT REACHED (crit_exit() would go here otherwise) */
1136 }