kernel - Add SMP support for software crypto driver
authorMatthew Dillon <dillon@apollo.backplane.com>
Fri, 20 Aug 2010 02:29:03 +0000 (19:29 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Fri, 20 Aug 2010 02:29:03 +0000 (19:29 -0700)
* Create a crypto thread for each cpu.

* Run the threads without the MP lock.

* Dispatch to the threads round-robin.

sys/opencrypto/crypto.c
sys/opencrypto/cryptodev.c
sys/opencrypto/cryptodev.h
sys/opencrypto/cryptosoft.c
sys/opencrypto/cryptosoft.h
sys/opencrypto/deflate.c
sys/opencrypto/xform.c

index 31fa1e2..e638ac8 100644 (file)
@@ -67,6 +67,7 @@
 #include <sys/proc.h>
 #include <sys/sysctl.h>
 #include <sys/thread2.h>
+#include <sys/mplock2.h>
 
 #include <vm/vm_zone.h>
 
@@ -117,6 +118,14 @@ struct cryptocap {
 static struct cryptocap *crypto_drivers = NULL;
 static int crypto_drivers_num = 0;
 
+typedef struct crypto_tdinfo {
+       TAILQ_HEAD(,cryptop)    crp_q;          /* request queues */
+       TAILQ_HEAD(,cryptkop)   crp_kq;
+       thread_t                crp_td;
+       struct lock             crp_lock;
+       int                     crp_sleep;
+} *crypto_tdinfo_t;
+
 /*
  * There are two queues for crypto requests; one for symmetric (e.g.
  * cipher) operations and one for asymmetric (e.g. MOD) operations.
@@ -125,12 +134,10 @@ static    int crypto_drivers_num = 0;
  * have one per-queue but having one simplifies handling of block/unblock
  * operations.
  */
-static int crp_sleep = 0;
-static TAILQ_HEAD(,cryptop) crp_q;             /* request queues */
-static TAILQ_HEAD(,cryptkop) crp_kq;
-static struct lock crypto_q_lock;
-#define        CRYPTO_Q_LOCK()         lockmgr(&crypto_q_lock, LK_EXCLUSIVE)
-#define        CRYPTO_Q_UNLOCK()       lockmgr(&crypto_q_lock, LK_RELEASE)
+static  struct crypto_tdinfo tdinfo_array[MAXCPU];
+
+#define        CRYPTO_Q_LOCK(tdinfo)   lockmgr(&tdinfo->crp_lock, LK_EXCLUSIVE)
+#define        CRYPTO_Q_UNLOCK(tdinfo) lockmgr(&tdinfo->crp_lock, LK_RELEASE)
 
 /*
  * There are two queues for processing completed crypto requests; one
@@ -165,9 +172,8 @@ SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
 
 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
 
-static void crypto_proc(void);
-static struct thread *cryptothread;
-static void crypto_ret_proc(void);
+static void crypto_proc(void *dummy);
+static void crypto_ret_proc(void *dummy);
 static struct thread *cryptoretthread;
 static void crypto_destroy(void);
 static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
@@ -186,14 +192,12 @@ SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
 static int
 crypto_init(void)
 {
+       crypto_tdinfo_t tdinfo;
        int error;
+       int n;
 
        lockinit(&crypto_drivers_lock, "crypto driver table", 0, LK_CANRECURSE);
 
-       TAILQ_INIT(&crp_q);
-       TAILQ_INIT(&crp_kq);
-       lockinit(&crypto_q_lock, "crypto op queues", 0, LK_CANRECURSE);
-
        TAILQ_INIT(&crp_ret_q);
        TAILQ_INIT(&crp_ret_kq);
        lockinit(&crypto_ret_q_lock, "crypto return queues", 0, LK_CANRECURSE);
@@ -208,29 +212,25 @@ crypto_init(void)
        }
 
        crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
-       crypto_drivers = kmalloc(crypto_drivers_num *
-           sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
+       crypto_drivers = kmalloc(crypto_drivers_num * sizeof(struct cryptocap),
+                                M_CRYPTO_DATA, M_WAITOK | M_ZERO);
        if (crypto_drivers == NULL) {
                kprintf("crypto_init: cannot malloc driver table\n");
                error = ENOMEM;
                goto bad;
        }
 
-       error = kthread_create((void (*)(void *)) crypto_proc, NULL,
-                   &cryptothread, "crypto");
-       if (error) {
-               kprintf("crypto_init: cannot start crypto thread; error %d",
-                       error);
-               goto bad;
-       }
-
-       error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL,
-                   &cryptoretthread, "crypto returns");
-       if (error) {
-               kprintf("crypto_init: cannot start cryptoret thread; error %d",
-                       error);
-               goto bad;
+       for (n = 0; n < ncpus; ++n) {
+               tdinfo = &tdinfo_array[n];
+               TAILQ_INIT(&tdinfo->crp_q);
+               TAILQ_INIT(&tdinfo->crp_kq);
+               lockinit(&tdinfo->crp_lock, "crypto op queues",
+                        0, LK_CANRECURSE);
+               kthread_create_cpu(crypto_proc, tdinfo, &tdinfo->crp_td,
+                                  n, "crypto %d", n);
        }
+       kthread_create(crypto_ret_proc, NULL,
+                      &cryptoretthread, "crypto returns");
        return 0;
 bad:
        crypto_destroy();
@@ -268,11 +268,18 @@ crypto_terminate(struct thread **tp, void *q)
 static void
 crypto_destroy(void)
 {
+       crypto_tdinfo_t tdinfo;
+       int n;
+
        /*
         * Terminate any crypto threads.
         */
        CRYPTO_DRIVER_LOCK();
-       crypto_terminate(&cryptothread, &crp_q);
+       for (n = 0; n < ncpus; ++n) {
+               tdinfo = &tdinfo_array[n];
+               crypto_terminate(&tdinfo->crp_td, &tdinfo->crp_q);
+               lockuninit(&tdinfo->crp_lock);
+       }
        crypto_terminate(&cryptoretthread, &crp_ret_q);
        CRYPTO_DRIVER_UNLOCK();
 
@@ -288,7 +295,6 @@ crypto_destroy(void)
                zdestroy(cryptodesc_zone);
        if (cryptop_zone != NULL)
                zdestroy(cryptop_zone);
-       lockuninit(&crypto_q_lock);
        lockuninit(&crypto_ret_q_lock);
        lockuninit(&crypto_drivers_lock);
 }
@@ -509,7 +515,8 @@ crypto_get_driverid(device_t dev, int flags)
                }
 
                newdrv = kmalloc(2 * crypto_drivers_num *
-                   sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+                                sizeof(struct cryptocap),
+                                M_CRYPTO_DATA, M_WAITOK|M_ZERO);
                if (newdrv == NULL) {
                        CRYPTO_DRIVER_UNLOCK();
                        kprintf("crypto: no space to expand driver table!\n");
@@ -627,7 +634,7 @@ crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags)
  */
 int
 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
-    u_int32_t flags)
+               u_int32_t flags)
 {
        struct cryptocap *cap;
        int err;
@@ -704,15 +711,17 @@ crypto_unregister(u_int32_t driverid, int alg)
                cap->cc_max_op_len[alg] = 0;
 
                /* Was this the last algorithm ? */
-               for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
+               for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) {
                        if (cap->cc_alg[i] != 0)
                                break;
+               }
 
                if (i == CRYPTO_ALGORITHM_MAX + 1)
                        driver_finis(cap);
                err = 0;
-       } else
+       } else {
                err = EINVAL;
+       }
        CRYPTO_DRIVER_UNLOCK();
 
        return err;
@@ -736,8 +745,9 @@ crypto_unregister_all(u_int32_t driverid)
        if (cap != NULL) {
                driver_finis(cap);
                err = 0;
-       } else
+       } else {
                err = EINVAL;
+       }
        CRYPTO_DRIVER_UNLOCK();
 
        return err;
@@ -750,35 +760,47 @@ crypto_unregister_all(u_int32_t driverid)
 int
 crypto_unblock(u_int32_t driverid, int what)
 {
+       crypto_tdinfo_t tdinfo;
        struct cryptocap *cap;
        int err;
+       int n;
 
-       CRYPTO_Q_LOCK();
+       CRYPTO_DRIVER_LOCK();
        cap = crypto_checkdriver(driverid);
        if (cap != NULL) {
                if (what & CRYPTO_SYMQ)
                        cap->cc_qblocked = 0;
                if (what & CRYPTO_ASYMQ)
                        cap->cc_kqblocked = 0;
-               if (crp_sleep)
-                       wakeup_one(&crp_q);
+               for (n = 0; n < ncpus; ++n) {
+                       tdinfo = &tdinfo_array[n];
+                       CRYPTO_Q_LOCK(tdinfo);
+                       if (tdinfo[n].crp_sleep)
+                               wakeup_one(&tdinfo->crp_q);
+                       CRYPTO_Q_UNLOCK(tdinfo);
+               }
                err = 0;
-       } else
+       } else {
                err = EINVAL;
-       CRYPTO_Q_UNLOCK();
+       }
+       CRYPTO_DRIVER_UNLOCK();
 
        return err;
 }
 
+static volatile int dispatch_rover;
+
 /*
  * Add a crypto request to a queue, to be processed by the kernel thread.
  */
 int
 crypto_dispatch(struct cryptop *crp)
 {
+       crypto_tdinfo_t tdinfo;
        struct cryptocap *cap;
        u_int32_t hid;
        int result;
+       int n;
 
        cryptostats.cs_ops++;
 
@@ -808,11 +830,23 @@ crypto_dispatch(struct cryptop *crp)
                         */
                }
        }
-       CRYPTO_Q_LOCK();
-       TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
-       if (crp_sleep)
-               wakeup_one(&crp_q);
-       CRYPTO_Q_UNLOCK();
+
+       /*
+        * Dispatch to a cpu for action if possible
+        */
+       if (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SMP) {
+               n = atomic_fetchadd_int(&dispatch_rover, 1) & 255;
+               n = n % ncpus;
+       } else {
+               n = 0;
+       }
+       tdinfo = &tdinfo_array[n];
+
+       CRYPTO_Q_LOCK(tdinfo);
+       TAILQ_INSERT_TAIL(&tdinfo->crp_q, crp, crp_next);
+       if (tdinfo->crp_sleep)
+               wakeup_one(&tdinfo->crp_q);
+       CRYPTO_Q_UNLOCK(tdinfo);
        return 0;
 }
 
@@ -823,17 +857,28 @@ crypto_dispatch(struct cryptop *crp)
 int
 crypto_kdispatch(struct cryptkop *krp)
 {
+       crypto_tdinfo_t tdinfo;
        int error;
+       int n;
 
        cryptostats.cs_kops++;
 
+#if 0
+       /* not sure how to test F_SMP here */
+       n = atomic_fetchadd_int(&dispatch_rover, 1) & 255;
+       n = n % ncpus;
+#endif
+       n = 0;
+       tdinfo = &tdinfo_array[n];
+
        error = crypto_kinvoke(krp, krp->krp_crid);
+
        if (error == ERESTART) {
-               CRYPTO_Q_LOCK();
-               TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
-               if (crp_sleep)
-                       wakeup_one(&crp_q);
-               CRYPTO_Q_UNLOCK();
+               CRYPTO_Q_LOCK(tdinfo);
+               TAILQ_INSERT_TAIL(&tdinfo->crp_kq, krp, krp_next);
+               if (tdinfo->crp_sleep)
+                       wakeup_one(&tdinfo->crp_q);
+               CRYPTO_Q_UNLOCK(tdinfo);
                error = 0;
        }
        return error;
@@ -1047,29 +1092,35 @@ void
 crypto_freereq(struct cryptop *crp)
 {
        struct cryptodesc *crd;
+#ifdef DIAGNOSTIC
+       crypto_tdinfo_t tdinfo;
+       int n;
+#endif
 
        if (crp == NULL)
                return;
 
 #ifdef DIAGNOSTIC
-       {
+       for (n = 0; n < ncpus; ++n) {
                struct cryptop *crp2;
 
-               CRYPTO_Q_LOCK();
-               TAILQ_FOREACH(crp2, &crp_q, crp_next) {
+               tdinfo = &tdinfo_array[n];
+
+               CRYPTO_Q_LOCK(tdinfo);
+               TAILQ_FOREACH(crp2, &tdinfo->crp_q, crp_next) {
                        KASSERT(crp2 != crp,
                            ("Freeing cryptop from the crypto queue (%p).",
                            crp));
                }
-               CRYPTO_Q_UNLOCK();
-               CRYPTO_RETQ_LOCK();
-               TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
-                       KASSERT(crp2 != crp,
-                           ("Freeing cryptop from the return queue (%p).",
-                           crp));
-               }
-               CRYPTO_RETQ_UNLOCK();
+               CRYPTO_Q_UNLOCK(tdinfo);
+       }
+       CRYPTO_RETQ_LOCK();
+       TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
+               KASSERT(crp2 != crp,
+                   ("Freeing cryptop from the return queue (%p).",
+                   crp));
        }
+       CRYPTO_RETQ_UNLOCK();
 #endif
 
        while ((crd = crp->crp_desc) != NULL) {
@@ -1233,15 +1284,19 @@ crypto_finis(void *chan)
  * Crypto thread, dispatches crypto requests.
  */
 static void
-crypto_proc(void)
+crypto_proc(void *arg)
 {
+       crypto_tdinfo_t tdinfo = arg;
        struct cryptop *crp, *submit;
        struct cryptkop *krp;
        struct cryptocap *cap;
        u_int32_t hid;
        int result, hint;
 
-       CRYPTO_Q_LOCK();
+       rel_mplock();           /* release the mplock held on startup */
+
+       CRYPTO_Q_LOCK(tdinfo);
+
        for (;;) {
                /*
                 * Find the first element in the queue that can be
@@ -1250,7 +1305,7 @@ crypto_proc(void)
                 */
                submit = NULL;
                hint = 0;
-               TAILQ_FOREACH(crp, &crp_q, crp_next) {
+               TAILQ_FOREACH(crp, &tdinfo->crp_q, crp_next) {
                        hid = CRYPTO_SESID2HID(crp->crp_sid);
                        cap = crypto_checkdriver(hid);
                        /*
@@ -1287,12 +1342,16 @@ crypto_proc(void)
                        }
                }
                if (submit != NULL) {
-                       TAILQ_REMOVE(&crp_q, submit, crp_next);
+                       TAILQ_REMOVE(&tdinfo->crp_q, submit, crp_next);
                        hid = CRYPTO_SESID2HID(submit->crp_sid);
                        cap = crypto_checkdriver(hid);
                        KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
                            __func__, __LINE__));
+
+                       CRYPTO_Q_UNLOCK(tdinfo);
                        result = crypto_invoke(cap, submit, hint);
+                       CRYPTO_Q_LOCK(tdinfo);
+
                        if (result == ERESTART) {
                                /*
                                 * The driver ran out of resources, mark the
@@ -1305,13 +1364,14 @@ crypto_proc(void)
                                 */
                                /* XXX validate sid again? */
                                crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
-                               TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
+                               TAILQ_INSERT_HEAD(&tdinfo->crp_q,
+                                                 submit, crp_next);
                                cryptostats.cs_blocks++;
                        }
                }
 
                /* As above, but for key ops */
-               TAILQ_FOREACH(krp, &crp_kq, krp_next) {
+               TAILQ_FOREACH(krp, &tdinfo->crp_kq, krp_next) {
                        cap = crypto_checkdriver(krp->krp_hid);
                        if (cap == NULL || cap->cc_dev == NULL) {
                                /*
@@ -1331,8 +1391,12 @@ crypto_proc(void)
                                break;
                }
                if (krp != NULL) {
-                       TAILQ_REMOVE(&crp_kq, krp, krp_next);
+                       TAILQ_REMOVE(&tdinfo->crp_kq, krp, krp_next);
+
+                       CRYPTO_Q_UNLOCK(tdinfo);
                        result = crypto_kinvoke(krp, krp->krp_hid);
+                       CRYPTO_Q_LOCK(tdinfo);
+
                        if (result == ERESTART) {
                                /*
                                 * The driver ran out of resources, mark the
@@ -1345,7 +1409,8 @@ crypto_proc(void)
                                 */
                                /* XXX validate sid again? */
                                crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
-                               TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
+                               TAILQ_INSERT_HEAD(&tdinfo->crp_kq,
+                                                 krp, krp_next);
                                cryptostats.cs_kblocks++;
                        }
                }
@@ -1363,17 +1428,18 @@ crypto_proc(void)
                         * out of order if dispatched to different devices
                         * and some become blocked while others do not.
                         */
-                       crp_sleep = 1;
-                       lksleep (&crp_q, &crypto_q_lock, 0, "crypto_wait", 0);
-                       crp_sleep = 0;
-                       if (cryptothread == NULL)
+                       tdinfo->crp_sleep = 1;
+                       lksleep (&tdinfo->crp_q, &tdinfo->crp_lock,
+                                0, "crypto_wait", 0);
+                       tdinfo->crp_sleep = 0;
+                       if (tdinfo->crp_td == NULL)
                                break;
                        cryptostats.cs_intrs++;
                }
        }
-       CRYPTO_Q_UNLOCK();
+       CRYPTO_Q_UNLOCK(tdinfo);
 
-       crypto_finis(&crp_q);
+       crypto_finis(&tdinfo->crp_q);
 }
 
 /*
@@ -1382,7 +1448,7 @@ crypto_proc(void)
  * callbacks typically are expensive and would slow interrupt handling.
  */
 static void
-crypto_ret_proc(void)
+crypto_ret_proc(void *dummy __unused)
 {
        struct cryptop *crpt;
        struct cryptkop *krpt;
@@ -1427,10 +1493,10 @@ crypto_ret_proc(void)
                         * Nothing more to be processed.  Sleep until we're
                         * woken because there are more returns to process.
                         */
-                       lksleep (&crp_ret_q, &crypto_ret_q_lock, 0, "crypto_ret_wait", 0);
-                       if (cryptoretthread == NULL) {
+                       lksleep (&crp_ret_q, &crypto_ret_q_lock,
+                                0, "crypto_ret_wait", 0);
+                       if (cryptoretthread == NULL)
                                break;
-                       }
                        cryptostats.cs_rets++;
                }
        }
@@ -1470,7 +1536,9 @@ db_show_drivers(void)
 
 DB_SHOW_COMMAND(crypto, db_show_crypto)
 {
+       crypto_tdinfo_t tdinfo;
        struct cryptop *crp;
+       int n;
 
        db_show_drivers();
        db_printf("\n");
@@ -1478,16 +1546,21 @@ DB_SHOW_COMMAND(crypto, db_show_crypto)
        db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
            "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
            "Desc", "Callback");
-       TAILQ_FOREACH(crp, &crp_q, crp_next) {
-               db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
-                   , (int) CRYPTO_SESID2HID(crp->crp_sid)
-                   , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
-                   , crp->crp_ilen, crp->crp_olen
-                   , crp->crp_etype
-                   , crp->crp_flags
-                   , crp->crp_desc
-                   , crp->crp_callback
-               );
+
+       for (n = 0; n < ncpus; ++n) {
+               tdinfo = &tdinfo_array[n];
+
+               TAILQ_FOREACH(crp, &tdinfo->crp_q, crp_next) {
+                       db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
+                           , (int) CRYPTO_SESID2HID(crp->crp_sid)
+                           , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
+                           , crp->crp_ilen, crp->crp_olen
+                           , crp->crp_etype
+                           , crp->crp_flags
+                           , crp->crp_desc
+                           , crp->crp_callback
+                       );
+               }
        }
        if (!TAILQ_EMPTY(&crp_ret_q)) {
                db_printf("\n%4s %4s %4s %8s\n",
@@ -1505,21 +1578,28 @@ DB_SHOW_COMMAND(crypto, db_show_crypto)
 
 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
 {
+       crypto_tdinfo_t tdinfo;
        struct cryptkop *krp;
+       int n;
 
        db_show_drivers();
        db_printf("\n");
 
        db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
            "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
-       TAILQ_FOREACH(krp, &crp_kq, krp_next) {
-               db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
-                   , krp->krp_op
-                   , krp->krp_status
-                   , krp->krp_iparams, krp->krp_oparams
-                   , krp->krp_crid, krp->krp_hid
-                   , krp->krp_callback
-               );
+
+       for (n = 0; n < ncpus; ++n) {
+               tdinfo = &tdinfo_array[n];
+
+               TAILQ_FOREACH(krp, &tdinfo->crp_kq, krp_next) {
+                       db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
+                           , krp->krp_op
+                           , krp->krp_status
+                           , krp->krp_iparams, krp->krp_oparams
+                           , krp->krp_crid, krp->krp_hid
+                           , krp->krp_callback
+                       );
+               }
        }
        if (!TAILQ_EMPTY(&crp_ret_q)) {
                db_printf("%4s %5s %8s %4s %8s\n",
index c2944b7..e13fab3 100644 (file)
@@ -748,7 +748,7 @@ csecreate(struct fcrypt *fcr, u_int64_t sid, caddr_t key, u_int64_t keylen,
 {
        struct csession *cse;
 
-       cse = kmalloc(sizeof(struct csession), M_XDATA, M_NOWAIT);
+       cse = kmalloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO);
        if (cse == NULL)
                return NULL;
        lockinit(&cse->lock, "cryptodev", 0, LK_CANRECURSE);
index d151a87..75d4273 100644 (file)
@@ -384,6 +384,7 @@ extern      int crypto_freesession(u_int64_t sid);
 #define CRYPTOCAP_F_HARDWARE   CRYPTO_FLAG_HARDWARE
 #define CRYPTOCAP_F_SOFTWARE   CRYPTO_FLAG_SOFTWARE
 #define CRYPTOCAP_F_SYNC       0x04000000      /* operates synchronously */
+#define CRYPTOCAP_F_SMP                0x08000000      /* SMP dispatch ok */
 extern int32_t crypto_get_driverid(device_t dev, int flags);
 extern int crypto_find_driver(const char *);
 extern device_t crypto_find_device_byhid(int hid);
index 707c89e..673a08b 100644 (file)
@@ -1,6 +1,3 @@
-/*     $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $  */
-/*     $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */
-
 /*-
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
@@ -11,6 +8,8 @@
  *
  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  *
+ * SMP modifications by Matthew Dillon for the DragonFlyBSD Project
+ *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all source code copies of any software which is or includes a copy or
@@ -21,6 +20,9 @@
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
+ *
+ * $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $
+ * $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $
  */
 
 #include <sys/param.h>
@@ -33,6 +35,7 @@
 #include <sys/random.h>
 #include <sys/kernel.h>
 #include <sys/uio.h>
+#include <sys/spinlock2.h>
 
 #include <crypto/blowfish/blowfish.h>
 #include <crypto/sha1.h>
@@ -52,6 +55,9 @@
 static int32_t swcr_id;
 static struct swcr_data **swcr_sessions = NULL;
 static u_int32_t swcr_sesnum;
+static u_int32_t swcr_minsesnum = 1;
+
+static struct spinlock swcr_spin = SPINLOCK_INITIALIZER(swcr_spin);
 
 u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
 u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
@@ -60,6 +66,7 @@ static        int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
 static int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
 static int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
 static int swcr_freesession(device_t dev, u_int64_t tid);
+static int swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid);
 
 /*
  * Apply a symmetric encryption/decryption algorithm.
@@ -70,8 +77,12 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
 {
        unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
        unsigned char *ivp, piv[EALG_MAX_BLOCK_LEN];
+       u_int8_t *kschedule;
+       u_int8_t *okschedule;
        struct enc_xform *exf;
        int i, k, j, blks;
+       int error;
+       int explicit_kschedule;
 
        exf = sw->sw_exf;
        blks = exf->blocksize;
@@ -102,16 +113,25 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                }
        }
 
+       /*
+        * The semantics are seriously broken because the session key
+        * storage was never designed for concurrent ops.
+        */
        if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
-               int error;
-
-               if (sw->sw_kschedule)
-                       exf->zerokey(&(sw->sw_kschedule));
-               error = exf->setkey(&sw->sw_kschedule,
-                               crd->crd_key, crd->crd_klen / 8);
+               kschedule = NULL;
+               explicit_kschedule = 1;
+               error = exf->setkey(&kschedule,
+                                   crd->crd_key, crd->crd_klen / 8);
                if (error)
-                       return (error);
+                       goto done;
+       } else {
+               spin_lock_wr(&swcr_spin);
+               kschedule = sw->sw_kschedule;
+               ++sw->sw_kschedule_refs;
+               spin_unlock_wr(&swcr_spin);
+               explicit_kschedule = 0;
        }
+
        ivp = iv;
 
        if (flags & CRYPTO_F_IMBUF) {
@@ -119,8 +139,10 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
 
                /* Find beginning of data */
                m = m_getptr(m, crd->crd_skip, &k);
-               if (m == NULL)
-                       return EINVAL;
+               if (m == NULL) {
+                       error = EINVAL;
+                       goto done;
+               }
 
                i = crd->crd_len;
 
@@ -138,7 +160,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        for (j = 0; j < blks; j++)
                                                blk[j] ^= ivp[j];
 
-                                       exf->encrypt(sw->sw_kschedule, blk);
+                                       exf->encrypt(kschedule, blk);
 
                                        /*
                                         * Keep encrypted block for XOR'ing
@@ -156,7 +178,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        else
                                                bcopy(blk, iv, blks);
 
-                                       exf->decrypt(sw->sw_kschedule, blk);
+                                       exf->decrypt(kschedule, blk);
 
                                        /* XOR with previous block */
                                        for (j = 0; j < blks; j++)
@@ -173,8 +195,10 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
 
                                /* Advance pointer */
                                m = m_getptr(m, k + blks, &k);
-                               if (m == NULL)
-                                       return EINVAL;
+                               if (m == NULL) {
+                                       error = EINVAL;
+                                       goto done;
+                               }
 
                                i -= blks;
 
@@ -192,8 +216,10 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                        }
 
                        /* Sanity check */
-                       if (m == NULL)
-                               return EINVAL;
+                       if (m == NULL) {
+                               error = EINVAL;
+                               goto done;
+                       }
 
                        /*
                         * Warning: idat may point to garbage here, but
@@ -208,7 +234,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        for (j = 0; j < blks; j++)
                                                idat[j] ^= ivp[j];
 
-                                       exf->encrypt(sw->sw_kschedule, idat);
+                                       exf->encrypt(kschedule, idat);
                                        ivp = idat;
                                } else {        /* decrypt */
                                        /*
@@ -220,7 +246,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        else
                                                bcopy(idat, iv, blks);
 
-                                       exf->decrypt(sw->sw_kschedule, idat);
+                                       exf->decrypt(kschedule, idat);
 
                                        /* XOR with previous block/IV */
                                        for (j = 0; j < blks; j++)
@@ -237,16 +263,17 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                i -= blks;
                        }
                }
-
-               return 0; /* Done with mbuf encryption/decryption */
+               error = 0;      /* Done with mbuf encryption/decryption */
        } else if (flags & CRYPTO_F_IOV) {
                struct uio *uio = (struct uio *) buf;
                struct iovec *iov;
 
                /* Find beginning of data */
                iov = cuio_getptr(uio, crd->crd_skip, &k);
-               if (iov == NULL)
-                       return EINVAL;
+               if (iov == NULL) {
+                       error = EINVAL;
+                       goto done;
+               }
 
                i = crd->crd_len;
 
@@ -264,7 +291,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        for (j = 0; j < blks; j++)
                                                blk[j] ^= ivp[j];
 
-                                       exf->encrypt(sw->sw_kschedule, blk);
+                                       exf->encrypt(kschedule, blk);
 
                                        /*
                                         * Keep encrypted block for XOR'ing
@@ -282,7 +309,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        else
                                                bcopy(blk, iv, blks);
 
-                                       exf->decrypt(sw->sw_kschedule, blk);
+                                       exf->decrypt(kschedule, blk);
 
                                        /* XOR with previous block */
                                        for (j = 0; j < blks; j++)
@@ -299,8 +326,10 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
 
                                /* Advance pointer */
                                iov = cuio_getptr(uio, k + blks, &k);
-                               if (iov == NULL)
-                                       return EINVAL;
+                               if (iov == NULL) {
+                                       error = EINVAL;
+                                       goto done;
+                               }
 
                                i -= blks;
 
@@ -322,7 +351,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        for (j = 0; j < blks; j++)
                                                idat[j] ^= ivp[j];
 
-                                       exf->encrypt(sw->sw_kschedule, idat);
+                                       exf->encrypt(kschedule, idat);
                                        ivp = idat;
                                } else {        /* decrypt */
                                        /*
@@ -334,7 +363,7 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                        else
                                                bcopy(idat, iv, blks);
 
-                                       exf->decrypt(sw->sw_kschedule, idat);
+                                       exf->decrypt(kschedule, idat);
 
                                        /* XOR with previous block/IV */
                                        for (j = 0; j < blks; j++)
@@ -355,9 +384,11 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                k = 0;
                        }
                }
-
-               return 0; /* Done with iovec encryption/decryption */
-       } else {        /* contiguous buffer */
+               error = 0;      /* Done with iovec encryption/decryption */
+       } else {
+               /*
+                * contiguous buffer
+                */
                if (crd->crd_flags & CRD_F_ENCRYPT) {
                        for (i = crd->crd_skip;
                            i < crd->crd_skip + crd->crd_len; i += blks) {
@@ -368,16 +399,16 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                else
                                        for (k = 0; k < blks; k++)
                                                buf[i + k] ^= buf[i + k - blks];
-                               exf->encrypt(sw->sw_kschedule, buf + i);
+                               exf->encrypt(kschedule, buf + i);
                        }
                } else {                /* Decrypt */
                        /*
-                        * Start at the end, so we don't need to keep the encrypted
-                        * block as the IV for the next block.
+                        * Start at the end, so we don't need to keep the
+                        * encrypted block as the IV for the next block.
                         */
                        for (i = crd->crd_skip + crd->crd_len - blks;
                            i >= crd->crd_skip; i -= blks) {
-                               exf->decrypt(sw->sw_kschedule, buf + i);
+                               exf->decrypt(kschedule, buf + i);
 
                                /* XOR with the IV/previous block, as appropriate */
                                if (i == crd->crd_skip)
@@ -388,12 +419,30 @@ swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
                                                buf[i + k] ^= buf[i + k - blks];
                        }
                }
-
-               return 0; /* Done with contiguous buffer encryption/decryption */
+               error = 0; /* Done w/contiguous buffer encrypt/decrypt */
        }
-
-       /* Unreachable */
-       return EINVAL;
+done:
+       /*
+        * Cleanup - explicitly replace the session key if requested
+        *           (horrible semantics for concurrent operation)
+        */
+       if (explicit_kschedule) {
+               spin_lock_wr(&swcr_spin);
+               if (sw->sw_kschedule && sw->sw_kschedule_refs == 0) {
+                       okschedule = sw->sw_kschedule;
+                       sw->sw_kschedule = kschedule;
+               } else {
+                       okschedule = NULL;
+               }
+               spin_unlock_wr(&swcr_spin);
+               if (okschedule)
+                       exf->zerokey(&okschedule);
+       } else {
+               spin_lock_wr(&swcr_spin);
+               --sw->sw_kschedule_refs;
+               spin_unlock_wr(&swcr_spin);
+       }
+       return error;
 }
 
 static void
@@ -516,7 +565,7 @@ swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
  */
 static int
 swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
-    caddr_t buf, int flags)
+            caddr_t buf, int flags)
 {
        u_int8_t *data, *out;
        struct comp_algo *cxf;
@@ -525,12 +574,12 @@ swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
 
        cxf = sw->sw_cxf;
 
-       /* We must handle the whole buffer of data in one time
+       /*
+        * We must handle the whole buffer of data in one time
         * then if there is not all the data in the mbuf, we must
         * copy in a buffer.
         */
-
-       data = kmalloc(crd->crd_len, M_CRYPTO_DATA,  M_NOWAIT);
+       data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_INTWAIT);
        if (data == NULL)
                return (EINVAL);
        crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
@@ -593,61 +642,25 @@ swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
 static int
 swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
 {
+       struct swcr_data *swd_base;
        struct swcr_data **swd;
+       struct swcr_data **oswd;
        struct auth_hash *axf;
        struct enc_xform *txf;
        struct comp_algo *cxf;
        u_int32_t i;
+       u_int32_t n;
        int error;
 
        if (sid == NULL || cri == NULL)
                return EINVAL;
 
-       if (swcr_sessions) {
-               for (i = 1; i < swcr_sesnum; i++)
-                       if (swcr_sessions[i] == NULL)
-                               break;
-       } else
-               i = 1;          /* NB: to silence compiler warning */
-
-       if (swcr_sessions == NULL || i == swcr_sesnum) {
-               if (swcr_sessions == NULL) {
-                       i = 1; /* We leave swcr_sessions[0] empty */
-                       swcr_sesnum = CRYPTO_SW_SESSIONS;
-               } else
-                       swcr_sesnum *= 2;
-
-               swd = kmalloc(swcr_sesnum * sizeof(struct swcr_data *),
-                   M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
-               if (swd == NULL) {
-                       /* Reset session number */
-                       if (swcr_sesnum == CRYPTO_SW_SESSIONS)
-                               swcr_sesnum = 0;
-                       else
-                               swcr_sesnum /= 2;
-                       return ENOBUFS;
-               }
-
-               /* Copy existing sessions */
-               if (swcr_sessions != NULL) {
-                       bcopy(swcr_sessions, swd,
-                           (swcr_sesnum / 2) * sizeof(struct swcr_data *));
-                       kfree(swcr_sessions, M_CRYPTO_DATA);
-               }
-
-               swcr_sessions = swd;
-       }
-
-       swd = &swcr_sessions[i];
-       *sid = i;
+       swd_base = NULL;
+       swd = &swd_base;
 
        while (cri) {
                *swd = kmalloc(sizeof(struct swcr_data),
-                   M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
-               if (*swd == NULL) {
-                       swcr_freesession(dev, i);
-                       return ENOBUFS;
-               }
+                              M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 
                switch (cri->cri_alg) {
                case CRYPTO_DES_CBC:
@@ -677,15 +690,16 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
                enccommon:
                        if (cri->cri_key != NULL) {
                                error = txf->setkey(&((*swd)->sw_kschedule),
-                                   cri->cri_key, cri->cri_klen / 8);
+                                                   cri->cri_key,
+                                                   cri->cri_klen / 8);
                                if (error) {
-                                       swcr_freesession(dev, i);
+                                       swcr_freesession_slot(&swd_base, 0);
                                        return error;
                                }
                        }
                        (*swd)->sw_exf = txf;
                        break;
-       
+
                case CRYPTO_MD5_HMAC:
                        axf = &auth_hash_hmac_md5;
                        goto authcommon;
@@ -708,16 +722,16 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
                        axf = &auth_hash_hmac_ripemd_160;
                authcommon:
                        (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
-                           M_NOWAIT);
+                                                 M_WAITOK);
                        if ((*swd)->sw_ictx == NULL) {
-                               swcr_freesession(dev, i);
+                               swcr_freesession_slot(&swd_base, 0);
                                return ENOBUFS;
                        }
        
                        (*swd)->sw_octx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
-                           M_NOWAIT);
+                                                 M_WAITOK);
                        if ((*swd)->sw_octx == NULL) {
-                               swcr_freesession(dev, i);
+                               swcr_freesession_slot(&swd_base, 0);
                                return ENOBUFS;
                        }
        
@@ -738,16 +752,16 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
                        axf = &auth_hash_key_sha1;
                auth2common:
                        (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
-                           M_NOWAIT);
+                                                 M_WAITOK);
                        if ((*swd)->sw_ictx == NULL) {
-                               swcr_freesession(dev, i);
+                               swcr_freesession_slot(&swd_base, 0);
                                return ENOBUFS;
                        }
        
                        (*swd)->sw_octx = kmalloc(cri->cri_klen / 8,
-                           M_CRYPTO_DATA, M_NOWAIT);
+                                                 M_CRYPTO_DATA, M_WAITOK);
                        if ((*swd)->sw_octx == NULL) {
-                               swcr_freesession(dev, i);
+                               swcr_freesession_slot(&swd_base, 0);
                                return ENOBUFS;
                        }
        
@@ -769,9 +783,9 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
                        axf = &auth_hash_sha1;
                auth3common:
                        (*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
-                           M_NOWAIT);
+                                                 M_WAITOK);
                        if ((*swd)->sw_ictx == NULL) {
-                               swcr_freesession(dev, i);
+                               swcr_freesession_slot(&swd_base, 0);
                                return ENOBUFS;
                        }
 
@@ -785,7 +799,7 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
                        (*swd)->sw_cxf = cxf;
                        break;
                default:
-                       swcr_freesession(dev, i);
+                       swcr_freesession_slot(&swd_base, 0);
                        return EINVAL;
                }
        
@@ -793,6 +807,56 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
                cri = cri->cri_next;
                swd = &((*swd)->sw_next);
        }
+
+       for (;;) {
+               /*
+                * Atomically allocate a session
+                */
+               spin_lock_wr(&swcr_spin);
+               for (i = swcr_minsesnum; i < swcr_sesnum; ++i) {
+                       if (swcr_sessions[i] == NULL)
+                               break;
+               }
+               if (i < swcr_sesnum) {
+                       swcr_sessions[i] = swd_base;
+                       swcr_minsesnum = i + 1;
+                       spin_unlock_wr(&swcr_spin);
+                       break;
+               }
+               n = swcr_sesnum;
+               spin_unlock_wr(&swcr_spin);
+
+               /*
+                * A larger allocation is required, reallocate the array
+                * and replace, checking for SMP races.
+                */
+               if (n < CRYPTO_SW_SESSIONS)
+                       n = CRYPTO_SW_SESSIONS;
+               else
+                       n = n * 3 / 2;
+               swd = kmalloc(n * sizeof(struct swcr_data *),
+                             M_CRYPTO_DATA, M_WAITOK | M_ZERO);
+
+               spin_lock_wr(&swcr_spin);
+               if (swcr_sesnum >= n) {
+                       spin_unlock_wr(&swcr_spin);
+                       kfree(swd, M_CRYPTO_DATA);
+               } else if (swcr_sesnum) {
+                       bcopy(swcr_sessions, swd,
+                             swcr_sesnum * sizeof(struct swcr_data *));
+                       oswd = swcr_sessions;
+                       swcr_sessions = swd;
+                       swcr_sesnum = n;
+                       spin_unlock_wr(&swcr_spin);
+                       kfree(oswd, M_CRYPTO_DATA);
+               } else {
+                       swcr_sessions = swd;
+                       swcr_sesnum = n;
+                       spin_unlock_wr(&swcr_spin);
+               }
+       }
+
+       *sid = i;
        return 0;
 }
 
@@ -802,22 +866,47 @@ swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
 static int
 swcr_freesession(device_t dev, u_int64_t tid)
 {
-       struct swcr_data *swd;
-       struct enc_xform *txf;
-       struct auth_hash *axf;
-       struct comp_algo *cxf;
        u_int32_t sid = CRYPTO_SESID2LID(tid);
 
        if (sid > swcr_sesnum || swcr_sessions == NULL ||
-           swcr_sessions[sid] == NULL)
+           swcr_sessions[sid] == NULL) {
                return EINVAL;
+       }
 
        /* Silently accept and return */
        if (sid == 0)
                return 0;
 
-       while ((swd = swcr_sessions[sid]) != NULL) {
-               swcr_sessions[sid] = swd->sw_next;
+       return(swcr_freesession_slot(&swcr_sessions[sid], sid));
+}
+
+static
+int
+swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid)
+{
+       struct enc_xform *txf;
+       struct auth_hash *axf;
+       struct comp_algo *cxf;
+       struct swcr_data *swd;
+       struct swcr_data *swnext;
+
+       /*
+        * Protect session detachment with the spinlock.
+        */
+       spin_lock_wr(&swcr_spin);
+       swnext = *swdp;
+       *swdp = NULL;
+       if (sid && swcr_minsesnum > sid)
+               swcr_minsesnum = sid;
+       spin_unlock_wr(&swcr_spin);
+
+       /*
+        * Clean up at our leisure.
+        */
+       while ((swd = swnext) != NULL) {
+               swnext = swd->sw_next;
+
+               swd->sw_next = NULL;
 
                switch (swd->sw_alg) {
                case CRYPTO_DES_CBC:
@@ -1007,8 +1096,9 @@ swcr_attach(device_t dev)
        memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
        memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
 
-       swcr_id = crypto_get_driverid(dev,
-                       CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
+       swcr_id = crypto_get_driverid(dev, CRYPTOCAP_F_SOFTWARE |
+                                          CRYPTOCAP_F_SYNC |
+                                          CRYPTOCAP_F_SMP);
        if (swcr_id < 0) {
                device_printf(dev, "cannot initialize!");
                return ENOMEM;
index 7820a44..f12ac70 100644 (file)
@@ -37,6 +37,7 @@ struct swcr_data {
                        struct auth_hash *SW_axf;
                } SWCR_AUTH;
                struct {
+                       int              SW_kschedule_refs;
                        u_int8_t         *SW_kschedule;
                        struct enc_xform *SW_exf;
                } SWCR_ENC;
@@ -52,6 +53,7 @@ struct swcr_data {
 #define sw_mlen                SWCR_UN.SWCR_AUTH.SW_mlen
 #define sw_axf         SWCR_UN.SWCR_AUTH.SW_axf
 #define sw_kschedule   SWCR_UN.SWCR_ENC.SW_kschedule
+#define sw_kschedule_refs SWCR_UN.SWCR_ENC.SW_kschedule_refs
 #define sw_exf         SWCR_UN.SWCR_ENC.SW_exf
 #define sw_size                SWCR_UN.SWCR_COMP.SW_size
 #define sw_cxf         SWCR_UN.SWCR_COMP.SW_cxf
index 789ec60..ca63966 100644 (file)
@@ -73,8 +73,7 @@ deflate_global(u_int8_t *data, u_int32_t size, int decomp, u_int8_t **out)
        zbuf.avail_in = size;   /* Total length of data to be processed */
 
        if (!decomp) {
-               buf[i].out = kmalloc((u_long) size, M_CRYPTO_DATA,
-                   M_NOWAIT);
+               buf[i].out = kmalloc((u_long) size, M_CRYPTO_DATA, M_WAITOK);
                if (buf[i].out == NULL)
                        goto bad;
                buf[i].size = size;
@@ -89,7 +88,7 @@ deflate_global(u_int8_t *data, u_int32_t size, int decomp, u_int8_t **out)
                 */
 
                buf[i].out = kmalloc((u_long) (size * 4),
-                   M_CRYPTO_DATA, M_NOWAIT);
+                                    M_CRYPTO_DATA, M_WAITOK);
                if (buf[i].out == NULL)
                        goto bad;
                buf[i].size = size * 4;
@@ -116,7 +115,7 @@ deflate_global(u_int8_t *data, u_int32_t size, int decomp, u_int8_t **out)
                else if (zbuf.avail_out == 0 && i < (ZBUF - 1)) {
                        /* we need more output space, allocate size */
                        buf[i].out = kmalloc((u_long) size,
-                           M_CRYPTO_DATA, M_NOWAIT);
+                                            M_CRYPTO_DATA, M_WAITOK);
                        if (buf[i].out == NULL)
                                goto bad;
                        zbuf.next_out = buf[i].out;
@@ -131,7 +130,7 @@ deflate_global(u_int8_t *data, u_int32_t size, int decomp, u_int8_t **out)
 end:
        result = count = zbuf.total_out;
 
-       *out = kmalloc((u_long) result, M_CRYPTO_DATA, M_NOWAIT);
+       *out = kmalloc((u_long) result, M_CRYPTO_DATA, M_WAITOK);
        if (*out == NULL)
                goto bad;
        if (decomp)
@@ -172,7 +171,7 @@ z_alloc(void *nil, u_int type, u_int size)
 {
        void *ptr;
 
-       ptr = kmalloc(type *size, M_CRYPTO_DATA, M_NOWAIT);
+       ptr = kmalloc(type *size, M_CRYPTO_DATA, M_WAITOK);
        return ptr;
 }
 
index 8a3de01..6620a12 100644 (file)
@@ -312,7 +312,7 @@ des1_setkey(u_int8_t **sched, u_int8_t *key, int len)
        int err;
 
        p = kmalloc(sizeof (des_key_schedule),
-               M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+                   M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
        if (p != NULL) {
                des_set_key((des_cblock *) key, p[0]);
                err = 0;
@@ -354,8 +354,8 @@ des3_setkey(u_int8_t **sched, u_int8_t *key, int len)
        des_key_schedule *p;
        int err;
 
-       p = kmalloc(3*sizeof (des_key_schedule),
-               M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+       p = kmalloc(3 * sizeof(des_key_schedule),
+                   M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
        if (p != NULL) {
                des_set_key((des_cblock *)(key +  0), p[0]);
                des_set_key((des_cblock *)(key +  8), p[1]);
@@ -410,8 +410,7 @@ blf_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
        int err;
 
-       *sched = kmalloc(sizeof(BF_KEY),
-               M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+       *sched = kmalloc(sizeof(BF_KEY), M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
        if (*sched != NULL) {
                BF_set_key((BF_KEY *) *sched, len, key);
                err = 0;
@@ -445,7 +444,7 @@ cast5_setkey(u_int8_t **sched, u_int8_t *key, int len)
 {
        int err;
 
-       *sched = kmalloc(sizeof(cast_key), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+       *sched = kmalloc(sizeof(cast_key), M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
        if (*sched != NULL) {
                cast_setkey((cast_key *)*sched, key, len);
                err = 0;
@@ -481,7 +480,7 @@ skipjack_setkey(u_int8_t **sched, u_int8_t *key, int len)
 
        /* NB: allocate all the memory that's needed at once */
        *sched = kmalloc(10 * (sizeof(u_int8_t *) + 0x100),
-               M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+                        M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
        if (*sched != NULL) {
                u_int8_t** key_tables = (u_int8_t**) *sched;
                u_int8_t* table = (u_int8_t*) &key_tables[10];
@@ -527,7 +526,7 @@ rijndael128_setkey(u_int8_t **sched, u_int8_t *key, int len)
        if (len != 16 && len != 24 && len != 32)
                return (EINVAL);
        *sched = kmalloc(sizeof(rijndael_ctx), M_CRYPTO_DATA,
-           M_NOWAIT|M_ZERO);
+                        M_INTWAIT | M_ZERO);
        if (*sched != NULL) {
                rijndael_set_key((rijndael_ctx *) *sched, (u_char *) key,
                    len * 8);
@@ -566,7 +565,7 @@ cml_setkey(u_int8_t **sched, u_int8_t *key, int len)
        if (len != 16 && len != 24 && len != 32)
                return (EINVAL);
        *sched = kmalloc(sizeof(camellia_ctx), M_CRYPTO_DATA,
-           M_NOWAIT|M_ZERO);
+                        M_INTWAIT | M_ZERO);
        if (*sched != NULL) {
                camellia_set_key((camellia_ctx *) *sched, (u_char *) key,
                    len * 8);