* Create a crypto thread for each cpu.
* Run the threads without the MP lock.
* Dispatch to the threads round-robin.
#include <sys/proc.h>
#include <sys/sysctl.h>
#include <sys/thread2.h>
+#include <sys/mplock2.h>
#include <vm/vm_zone.h>
static struct cryptocap *crypto_drivers = NULL;
static int crypto_drivers_num = 0;
+typedef struct crypto_tdinfo {
+ TAILQ_HEAD(,cryptop) crp_q; /* request queues */
+ TAILQ_HEAD(,cryptkop) crp_kq;
+ thread_t crp_td;
+ struct lock crp_lock;
+ int crp_sleep;
+} *crypto_tdinfo_t;
+
/*
* There are two queues for crypto requests; one for symmetric (e.g.
* cipher) operations and one for asymmetric (e.g. MOD) operations.
* have one per-queue but having one simplifies handling of block/unblock
* operations.
*/
-static int crp_sleep = 0;
-static TAILQ_HEAD(,cryptop) crp_q; /* request queues */
-static TAILQ_HEAD(,cryptkop) crp_kq;
-static struct lock crypto_q_lock;
-#define CRYPTO_Q_LOCK() lockmgr(&crypto_q_lock, LK_EXCLUSIVE)
-#define CRYPTO_Q_UNLOCK() lockmgr(&crypto_q_lock, LK_RELEASE)
+static struct crypto_tdinfo tdinfo_array[MAXCPU];
+
+#define CRYPTO_Q_LOCK(tdinfo) lockmgr(&tdinfo->crp_lock, LK_EXCLUSIVE)
+#define CRYPTO_Q_UNLOCK(tdinfo) lockmgr(&tdinfo->crp_lock, LK_RELEASE)
/*
* There are two queues for processing completed crypto requests; one
MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
-static void crypto_proc(void);
-static struct thread *cryptothread;
-static void crypto_ret_proc(void);
+static void crypto_proc(void *dummy);
+static void crypto_ret_proc(void *dummy);
static struct thread *cryptoretthread;
static void crypto_destroy(void);
static int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
static int
crypto_init(void)
{
+ crypto_tdinfo_t tdinfo;
int error;
+ int n;
lockinit(&crypto_drivers_lock, "crypto driver table", 0, LK_CANRECURSE);
- TAILQ_INIT(&crp_q);
- TAILQ_INIT(&crp_kq);
- lockinit(&crypto_q_lock, "crypto op queues", 0, LK_CANRECURSE);
-
TAILQ_INIT(&crp_ret_q);
TAILQ_INIT(&crp_ret_kq);
lockinit(&crypto_ret_q_lock, "crypto return queues", 0, LK_CANRECURSE);
}
crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
- crypto_drivers = kmalloc(crypto_drivers_num *
- sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
+ crypto_drivers = kmalloc(crypto_drivers_num * sizeof(struct cryptocap),
+ M_CRYPTO_DATA, M_WAITOK | M_ZERO);
if (crypto_drivers == NULL) {
kprintf("crypto_init: cannot malloc driver table\n");
error = ENOMEM;
goto bad;
}
- error = kthread_create((void (*)(void *)) crypto_proc, NULL,
- &cryptothread, "crypto");
- if (error) {
- kprintf("crypto_init: cannot start crypto thread; error %d",
- error);
- goto bad;
- }
-
- error = kthread_create((void (*)(void *)) crypto_ret_proc, NULL,
- &cryptoretthread, "crypto returns");
- if (error) {
- kprintf("crypto_init: cannot start cryptoret thread; error %d",
- error);
- goto bad;
+ for (n = 0; n < ncpus; ++n) {
+ tdinfo = &tdinfo_array[n];
+ TAILQ_INIT(&tdinfo->crp_q);
+ TAILQ_INIT(&tdinfo->crp_kq);
+ lockinit(&tdinfo->crp_lock, "crypto op queues",
+ 0, LK_CANRECURSE);
+ kthread_create_cpu(crypto_proc, tdinfo, &tdinfo->crp_td,
+ n, "crypto %d", n);
}
+ kthread_create(crypto_ret_proc, NULL,
+ &cryptoretthread, "crypto returns");
return 0;
bad:
crypto_destroy();
static void
crypto_destroy(void)
{
+ crypto_tdinfo_t tdinfo;
+ int n;
+
/*
* Terminate any crypto threads.
*/
CRYPTO_DRIVER_LOCK();
- crypto_terminate(&cryptothread, &crp_q);
+ for (n = 0; n < ncpus; ++n) {
+ tdinfo = &tdinfo_array[n];
+ crypto_terminate(&tdinfo->crp_td, &tdinfo->crp_q);
+ lockuninit(&tdinfo->crp_lock);
+ }
crypto_terminate(&cryptoretthread, &crp_ret_q);
CRYPTO_DRIVER_UNLOCK();
zdestroy(cryptodesc_zone);
if (cryptop_zone != NULL)
zdestroy(cryptop_zone);
- lockuninit(&crypto_q_lock);
lockuninit(&crypto_ret_q_lock);
lockuninit(&crypto_drivers_lock);
}
}
newdrv = kmalloc(2 * crypto_drivers_num *
- sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+ sizeof(struct cryptocap),
+ M_CRYPTO_DATA, M_WAITOK|M_ZERO);
if (newdrv == NULL) {
CRYPTO_DRIVER_UNLOCK();
kprintf("crypto: no space to expand driver table!\n");
*/
int
crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
- u_int32_t flags)
+ u_int32_t flags)
{
struct cryptocap *cap;
int err;
cap->cc_max_op_len[alg] = 0;
/* Was this the last algorithm ? */
- for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
+ for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++) {
if (cap->cc_alg[i] != 0)
break;
+ }
if (i == CRYPTO_ALGORITHM_MAX + 1)
driver_finis(cap);
err = 0;
- } else
+ } else {
err = EINVAL;
+ }
CRYPTO_DRIVER_UNLOCK();
return err;
if (cap != NULL) {
driver_finis(cap);
err = 0;
- } else
+ } else {
err = EINVAL;
+ }
CRYPTO_DRIVER_UNLOCK();
return err;
int
crypto_unblock(u_int32_t driverid, int what)
{
+ crypto_tdinfo_t tdinfo;
struct cryptocap *cap;
int err;
+ int n;
- CRYPTO_Q_LOCK();
+ CRYPTO_DRIVER_LOCK();
cap = crypto_checkdriver(driverid);
if (cap != NULL) {
if (what & CRYPTO_SYMQ)
cap->cc_qblocked = 0;
if (what & CRYPTO_ASYMQ)
cap->cc_kqblocked = 0;
- if (crp_sleep)
- wakeup_one(&crp_q);
+ for (n = 0; n < ncpus; ++n) {
+ tdinfo = &tdinfo_array[n];
+ CRYPTO_Q_LOCK(tdinfo);
+ if (tdinfo[n].crp_sleep)
+ wakeup_one(&tdinfo->crp_q);
+ CRYPTO_Q_UNLOCK(tdinfo);
+ }
err = 0;
- } else
+ } else {
err = EINVAL;
- CRYPTO_Q_UNLOCK();
+ }
+ CRYPTO_DRIVER_UNLOCK();
return err;
}
+static volatile int dispatch_rover;
+
/*
* Add a crypto request to a queue, to be processed by the kernel thread.
*/
int
crypto_dispatch(struct cryptop *crp)
{
+ crypto_tdinfo_t tdinfo;
struct cryptocap *cap;
u_int32_t hid;
int result;
+ int n;
cryptostats.cs_ops++;
*/
}
}
- CRYPTO_Q_LOCK();
- TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
- if (crp_sleep)
- wakeup_one(&crp_q);
- CRYPTO_Q_UNLOCK();
+
+ /*
+ * Dispatch to a cpu for action if possible
+ */
+ if (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SMP) {
+ n = atomic_fetchadd_int(&dispatch_rover, 1) & 255;
+ n = n % ncpus;
+ } else {
+ n = 0;
+ }
+ tdinfo = &tdinfo_array[n];
+
+ CRYPTO_Q_LOCK(tdinfo);
+ TAILQ_INSERT_TAIL(&tdinfo->crp_q, crp, crp_next);
+ if (tdinfo->crp_sleep)
+ wakeup_one(&tdinfo->crp_q);
+ CRYPTO_Q_UNLOCK(tdinfo);
return 0;
}
int
crypto_kdispatch(struct cryptkop *krp)
{
+ crypto_tdinfo_t tdinfo;
int error;
+ int n;
cryptostats.cs_kops++;
+#if 0
+ /* not sure how to test F_SMP here */
+ n = atomic_fetchadd_int(&dispatch_rover, 1) & 255;
+ n = n % ncpus;
+#endif
+ n = 0;
+ tdinfo = &tdinfo_array[n];
+
error = crypto_kinvoke(krp, krp->krp_crid);
+
if (error == ERESTART) {
- CRYPTO_Q_LOCK();
- TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
- if (crp_sleep)
- wakeup_one(&crp_q);
- CRYPTO_Q_UNLOCK();
+ CRYPTO_Q_LOCK(tdinfo);
+ TAILQ_INSERT_TAIL(&tdinfo->crp_kq, krp, krp_next);
+ if (tdinfo->crp_sleep)
+ wakeup_one(&tdinfo->crp_q);
+ CRYPTO_Q_UNLOCK(tdinfo);
error = 0;
}
return error;
crypto_freereq(struct cryptop *crp)
{
struct cryptodesc *crd;
+#ifdef DIAGNOSTIC
+ crypto_tdinfo_t tdinfo;
+ int n;
+#endif
if (crp == NULL)
return;
#ifdef DIAGNOSTIC
- {
+ for (n = 0; n < ncpus; ++n) {
struct cryptop *crp2;
- CRYPTO_Q_LOCK();
- TAILQ_FOREACH(crp2, &crp_q, crp_next) {
+ tdinfo = &tdinfo_array[n];
+
+ CRYPTO_Q_LOCK(tdinfo);
+ TAILQ_FOREACH(crp2, &tdinfo->crp_q, crp_next) {
KASSERT(crp2 != crp,
("Freeing cryptop from the crypto queue (%p).",
crp));
}
- CRYPTO_Q_UNLOCK();
- CRYPTO_RETQ_LOCK();
- TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
- KASSERT(crp2 != crp,
- ("Freeing cryptop from the return queue (%p).",
- crp));
- }
- CRYPTO_RETQ_UNLOCK();
+ CRYPTO_Q_UNLOCK(tdinfo);
+ }
+ CRYPTO_RETQ_LOCK();
+ TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
+ KASSERT(crp2 != crp,
+ ("Freeing cryptop from the return queue (%p).",
+ crp));
}
+ CRYPTO_RETQ_UNLOCK();
#endif
while ((crd = crp->crp_desc) != NULL) {
* Crypto thread, dispatches crypto requests.
*/
static void
-crypto_proc(void)
+crypto_proc(void *arg)
{
+ crypto_tdinfo_t tdinfo = arg;
struct cryptop *crp, *submit;
struct cryptkop *krp;
struct cryptocap *cap;
u_int32_t hid;
int result, hint;
- CRYPTO_Q_LOCK();
+ rel_mplock(); /* release the mplock held on startup */
+
+ CRYPTO_Q_LOCK(tdinfo);
+
for (;;) {
/*
* Find the first element in the queue that can be
*/
submit = NULL;
hint = 0;
- TAILQ_FOREACH(crp, &crp_q, crp_next) {
+ TAILQ_FOREACH(crp, &tdinfo->crp_q, crp_next) {
hid = CRYPTO_SESID2HID(crp->crp_sid);
cap = crypto_checkdriver(hid);
/*
}
}
if (submit != NULL) {
- TAILQ_REMOVE(&crp_q, submit, crp_next);
+ TAILQ_REMOVE(&tdinfo->crp_q, submit, crp_next);
hid = CRYPTO_SESID2HID(submit->crp_sid);
cap = crypto_checkdriver(hid);
KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
__func__, __LINE__));
+
+ CRYPTO_Q_UNLOCK(tdinfo);
result = crypto_invoke(cap, submit, hint);
+ CRYPTO_Q_LOCK(tdinfo);
+
if (result == ERESTART) {
/*
* The driver ran out of resources, mark the
*/
/* XXX validate sid again? */
crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
- TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
+ TAILQ_INSERT_HEAD(&tdinfo->crp_q,
+ submit, crp_next);
cryptostats.cs_blocks++;
}
}
/* As above, but for key ops */
- TAILQ_FOREACH(krp, &crp_kq, krp_next) {
+ TAILQ_FOREACH(krp, &tdinfo->crp_kq, krp_next) {
cap = crypto_checkdriver(krp->krp_hid);
if (cap == NULL || cap->cc_dev == NULL) {
/*
break;
}
if (krp != NULL) {
- TAILQ_REMOVE(&crp_kq, krp, krp_next);
+ TAILQ_REMOVE(&tdinfo->crp_kq, krp, krp_next);
+
+ CRYPTO_Q_UNLOCK(tdinfo);
result = crypto_kinvoke(krp, krp->krp_hid);
+ CRYPTO_Q_LOCK(tdinfo);
+
if (result == ERESTART) {
/*
* The driver ran out of resources, mark the
*/
/* XXX validate sid again? */
crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
- TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
+ TAILQ_INSERT_HEAD(&tdinfo->crp_kq,
+ krp, krp_next);
cryptostats.cs_kblocks++;
}
}
* out of order if dispatched to different devices
* and some become blocked while others do not.
*/
- crp_sleep = 1;
- lksleep (&crp_q, &crypto_q_lock, 0, "crypto_wait", 0);
- crp_sleep = 0;
- if (cryptothread == NULL)
+ tdinfo->crp_sleep = 1;
+ lksleep (&tdinfo->crp_q, &tdinfo->crp_lock,
+ 0, "crypto_wait", 0);
+ tdinfo->crp_sleep = 0;
+ if (tdinfo->crp_td == NULL)
break;
cryptostats.cs_intrs++;
}
}
- CRYPTO_Q_UNLOCK();
+ CRYPTO_Q_UNLOCK(tdinfo);
- crypto_finis(&crp_q);
+ crypto_finis(&tdinfo->crp_q);
}
/*
* callbacks typically are expensive and would slow interrupt handling.
*/
static void
-crypto_ret_proc(void)
+crypto_ret_proc(void *dummy __unused)
{
struct cryptop *crpt;
struct cryptkop *krpt;
* Nothing more to be processed. Sleep until we're
* woken because there are more returns to process.
*/
- lksleep (&crp_ret_q, &crypto_ret_q_lock, 0, "crypto_ret_wait", 0);
- if (cryptoretthread == NULL) {
+ lksleep (&crp_ret_q, &crypto_ret_q_lock,
+ 0, "crypto_ret_wait", 0);
+ if (cryptoretthread == NULL)
break;
- }
cryptostats.cs_rets++;
}
}
DB_SHOW_COMMAND(crypto, db_show_crypto)
{
+ crypto_tdinfo_t tdinfo;
struct cryptop *crp;
+ int n;
db_show_drivers();
db_printf("\n");
db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
"HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
"Desc", "Callback");
- TAILQ_FOREACH(crp, &crp_q, crp_next) {
- db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
- , (int) CRYPTO_SESID2HID(crp->crp_sid)
- , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
- , crp->crp_ilen, crp->crp_olen
- , crp->crp_etype
- , crp->crp_flags
- , crp->crp_desc
- , crp->crp_callback
- );
+
+ for (n = 0; n < ncpus; ++n) {
+ tdinfo = &tdinfo_array[n];
+
+ TAILQ_FOREACH(crp, &tdinfo->crp_q, crp_next) {
+ db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
+ , (int) CRYPTO_SESID2HID(crp->crp_sid)
+ , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
+ , crp->crp_ilen, crp->crp_olen
+ , crp->crp_etype
+ , crp->crp_flags
+ , crp->crp_desc
+ , crp->crp_callback
+ );
+ }
}
if (!TAILQ_EMPTY(&crp_ret_q)) {
db_printf("\n%4s %4s %4s %8s\n",
DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
{
+ crypto_tdinfo_t tdinfo;
struct cryptkop *krp;
+ int n;
db_show_drivers();
db_printf("\n");
db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
"Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
- TAILQ_FOREACH(krp, &crp_kq, krp_next) {
- db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
- , krp->krp_op
- , krp->krp_status
- , krp->krp_iparams, krp->krp_oparams
- , krp->krp_crid, krp->krp_hid
- , krp->krp_callback
- );
+
+ for (n = 0; n < ncpus; ++n) {
+ tdinfo = &tdinfo_array[n];
+
+ TAILQ_FOREACH(krp, &tdinfo->crp_kq, krp_next) {
+ db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
+ , krp->krp_op
+ , krp->krp_status
+ , krp->krp_iparams, krp->krp_oparams
+ , krp->krp_crid, krp->krp_hid
+ , krp->krp_callback
+ );
+ }
}
if (!TAILQ_EMPTY(&crp_ret_q)) {
db_printf("%4s %5s %8s %4s %8s\n",
{
struct csession *cse;
- cse = kmalloc(sizeof(struct csession), M_XDATA, M_NOWAIT);
+ cse = kmalloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO);
if (cse == NULL)
return NULL;
lockinit(&cse->lock, "cryptodev", 0, LK_CANRECURSE);
#define CRYPTOCAP_F_HARDWARE CRYPTO_FLAG_HARDWARE
#define CRYPTOCAP_F_SOFTWARE CRYPTO_FLAG_SOFTWARE
#define CRYPTOCAP_F_SYNC 0x04000000 /* operates synchronously */
+#define CRYPTOCAP_F_SMP 0x08000000 /* SMP dispatch ok */
extern int32_t crypto_get_driverid(device_t dev, int flags);
extern int crypto_find_driver(const char *);
extern device_t crypto_find_device_byhid(int hid);
-/* $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $ */
-/* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */
-
/*-
* The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
* Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
*
* Copyright (c) 2000, 2001 Angelos D. Keromytis
*
+ * SMP modifications by Matthew Dillon for the DragonFlyBSD Project
+ *
* Permission to use, copy, and modify this software with or without fee
* is hereby granted, provided that this entire notice is included in
* all source code copies of any software which is or includes a copy or
* REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
* MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
* PURPOSE.
+ *
+ * $FreeBSD: src/sys/opencrypto/cryptosoft.c,v 1.23 2009/02/05 17:43:12 imp Exp $
+ * $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $
*/
#include <sys/param.h>
#include <sys/random.h>
#include <sys/kernel.h>
#include <sys/uio.h>
+#include <sys/spinlock2.h>
#include <crypto/blowfish/blowfish.h>
#include <crypto/sha1.h>
static int32_t swcr_id;
static struct swcr_data **swcr_sessions = NULL;
static u_int32_t swcr_sesnum;
+static u_int32_t swcr_minsesnum = 1;
+
+static struct spinlock swcr_spin = SPINLOCK_INITIALIZER(swcr_spin);
u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
static int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
static int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
static int swcr_freesession(device_t dev, u_int64_t tid);
+static int swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid);
/*
* Apply a symmetric encryption/decryption algorithm.
{
unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN], *idat;
unsigned char *ivp, piv[EALG_MAX_BLOCK_LEN];
+ u_int8_t *kschedule;
+ u_int8_t *okschedule;
struct enc_xform *exf;
int i, k, j, blks;
+ int error;
+ int explicit_kschedule;
exf = sw->sw_exf;
blks = exf->blocksize;
}
}
+ /*
+ * The semantics are seriously broken because the session key
+ * storage was never designed for concurrent ops.
+ */
if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
- int error;
-
- if (sw->sw_kschedule)
- exf->zerokey(&(sw->sw_kschedule));
- error = exf->setkey(&sw->sw_kschedule,
- crd->crd_key, crd->crd_klen / 8);
+ kschedule = NULL;
+ explicit_kschedule = 1;
+ error = exf->setkey(&kschedule,
+ crd->crd_key, crd->crd_klen / 8);
if (error)
- return (error);
+ goto done;
+ } else {
+ spin_lock_wr(&swcr_spin);
+ kschedule = sw->sw_kschedule;
+ ++sw->sw_kschedule_refs;
+ spin_unlock_wr(&swcr_spin);
+ explicit_kschedule = 0;
}
+
ivp = iv;
if (flags & CRYPTO_F_IMBUF) {
/* Find beginning of data */
m = m_getptr(m, crd->crd_skip, &k);
- if (m == NULL)
- return EINVAL;
+ if (m == NULL) {
+ error = EINVAL;
+ goto done;
+ }
i = crd->crd_len;
for (j = 0; j < blks; j++)
blk[j] ^= ivp[j];
- exf->encrypt(sw->sw_kschedule, blk);
+ exf->encrypt(kschedule, blk);
/*
* Keep encrypted block for XOR'ing
else
bcopy(blk, iv, blks);
- exf->decrypt(sw->sw_kschedule, blk);
+ exf->decrypt(kschedule, blk);
/* XOR with previous block */
for (j = 0; j < blks; j++)
/* Advance pointer */
m = m_getptr(m, k + blks, &k);
- if (m == NULL)
- return EINVAL;
+ if (m == NULL) {
+ error = EINVAL;
+ goto done;
+ }
i -= blks;
}
/* Sanity check */
- if (m == NULL)
- return EINVAL;
+ if (m == NULL) {
+ error = EINVAL;
+ goto done;
+ }
/*
* Warning: idat may point to garbage here, but
for (j = 0; j < blks; j++)
idat[j] ^= ivp[j];
- exf->encrypt(sw->sw_kschedule, idat);
+ exf->encrypt(kschedule, idat);
ivp = idat;
} else { /* decrypt */
/*
else
bcopy(idat, iv, blks);
- exf->decrypt(sw->sw_kschedule, idat);
+ exf->decrypt(kschedule, idat);
/* XOR with previous block/IV */
for (j = 0; j < blks; j++)
i -= blks;
}
}
-
- return 0; /* Done with mbuf encryption/decryption */
+ error = 0; /* Done with mbuf encryption/decryption */
} else if (flags & CRYPTO_F_IOV) {
struct uio *uio = (struct uio *) buf;
struct iovec *iov;
/* Find beginning of data */
iov = cuio_getptr(uio, crd->crd_skip, &k);
- if (iov == NULL)
- return EINVAL;
+ if (iov == NULL) {
+ error = EINVAL;
+ goto done;
+ }
i = crd->crd_len;
for (j = 0; j < blks; j++)
blk[j] ^= ivp[j];
- exf->encrypt(sw->sw_kschedule, blk);
+ exf->encrypt(kschedule, blk);
/*
* Keep encrypted block for XOR'ing
else
bcopy(blk, iv, blks);
- exf->decrypt(sw->sw_kschedule, blk);
+ exf->decrypt(kschedule, blk);
/* XOR with previous block */
for (j = 0; j < blks; j++)
/* Advance pointer */
iov = cuio_getptr(uio, k + blks, &k);
- if (iov == NULL)
- return EINVAL;
+ if (iov == NULL) {
+ error = EINVAL;
+ goto done;
+ }
i -= blks;
for (j = 0; j < blks; j++)
idat[j] ^= ivp[j];
- exf->encrypt(sw->sw_kschedule, idat);
+ exf->encrypt(kschedule, idat);
ivp = idat;
} else { /* decrypt */
/*
else
bcopy(idat, iv, blks);
- exf->decrypt(sw->sw_kschedule, idat);
+ exf->decrypt(kschedule, idat);
/* XOR with previous block/IV */
for (j = 0; j < blks; j++)
k = 0;
}
}
-
- return 0; /* Done with iovec encryption/decryption */
- } else { /* contiguous buffer */
+ error = 0; /* Done with iovec encryption/decryption */
+ } else {
+ /*
+ * contiguous buffer
+ */
if (crd->crd_flags & CRD_F_ENCRYPT) {
for (i = crd->crd_skip;
i < crd->crd_skip + crd->crd_len; i += blks) {
else
for (k = 0; k < blks; k++)
buf[i + k] ^= buf[i + k - blks];
- exf->encrypt(sw->sw_kschedule, buf + i);
+ exf->encrypt(kschedule, buf + i);
}
} else { /* Decrypt */
/*
- * Start at the end, so we don't need to keep the encrypted
- * block as the IV for the next block.
+ * Start at the end, so we don't need to keep the
+ * encrypted block as the IV for the next block.
*/
for (i = crd->crd_skip + crd->crd_len - blks;
i >= crd->crd_skip; i -= blks) {
- exf->decrypt(sw->sw_kschedule, buf + i);
+ exf->decrypt(kschedule, buf + i);
/* XOR with the IV/previous block, as appropriate */
if (i == crd->crd_skip)
buf[i + k] ^= buf[i + k - blks];
}
}
-
- return 0; /* Done with contiguous buffer encryption/decryption */
+ error = 0; /* Done w/contiguous buffer encrypt/decrypt */
}
-
- /* Unreachable */
- return EINVAL;
+done:
+ /*
+ * Cleanup - explicitly replace the session key if requested
+ * (horrible semantics for concurrent operation)
+ */
+ if (explicit_kschedule) {
+ spin_lock_wr(&swcr_spin);
+ if (sw->sw_kschedule && sw->sw_kschedule_refs == 0) {
+ okschedule = sw->sw_kschedule;
+ sw->sw_kschedule = kschedule;
+ } else {
+ okschedule = NULL;
+ }
+ spin_unlock_wr(&swcr_spin);
+ if (okschedule)
+ exf->zerokey(&okschedule);
+ } else {
+ spin_lock_wr(&swcr_spin);
+ --sw->sw_kschedule_refs;
+ spin_unlock_wr(&swcr_spin);
+ }
+ return error;
}
static void
*/
static int
swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
- caddr_t buf, int flags)
+ caddr_t buf, int flags)
{
u_int8_t *data, *out;
struct comp_algo *cxf;
cxf = sw->sw_cxf;
- /* We must handle the whole buffer of data in one time
+ /*
+ * We must handle the whole buffer of data in one time
* then if there is not all the data in the mbuf, we must
* copy in a buffer.
*/
-
- data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_NOWAIT);
+ data = kmalloc(crd->crd_len, M_CRYPTO_DATA, M_INTWAIT);
if (data == NULL)
return (EINVAL);
crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
static int
swcr_newsession(device_t dev, u_int32_t *sid, struct cryptoini *cri)
{
+ struct swcr_data *swd_base;
struct swcr_data **swd;
+ struct swcr_data **oswd;
struct auth_hash *axf;
struct enc_xform *txf;
struct comp_algo *cxf;
u_int32_t i;
+ u_int32_t n;
int error;
if (sid == NULL || cri == NULL)
return EINVAL;
- if (swcr_sessions) {
- for (i = 1; i < swcr_sesnum; i++)
- if (swcr_sessions[i] == NULL)
- break;
- } else
- i = 1; /* NB: to silence compiler warning */
-
- if (swcr_sessions == NULL || i == swcr_sesnum) {
- if (swcr_sessions == NULL) {
- i = 1; /* We leave swcr_sessions[0] empty */
- swcr_sesnum = CRYPTO_SW_SESSIONS;
- } else
- swcr_sesnum *= 2;
-
- swd = kmalloc(swcr_sesnum * sizeof(struct swcr_data *),
- M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
- if (swd == NULL) {
- /* Reset session number */
- if (swcr_sesnum == CRYPTO_SW_SESSIONS)
- swcr_sesnum = 0;
- else
- swcr_sesnum /= 2;
- return ENOBUFS;
- }
-
- /* Copy existing sessions */
- if (swcr_sessions != NULL) {
- bcopy(swcr_sessions, swd,
- (swcr_sesnum / 2) * sizeof(struct swcr_data *));
- kfree(swcr_sessions, M_CRYPTO_DATA);
- }
-
- swcr_sessions = swd;
- }
-
- swd = &swcr_sessions[i];
- *sid = i;
+ swd_base = NULL;
+ swd = &swd_base;
while (cri) {
*swd = kmalloc(sizeof(struct swcr_data),
- M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
- if (*swd == NULL) {
- swcr_freesession(dev, i);
- return ENOBUFS;
- }
+ M_CRYPTO_DATA, M_WAITOK | M_ZERO);
switch (cri->cri_alg) {
case CRYPTO_DES_CBC:
enccommon:
if (cri->cri_key != NULL) {
error = txf->setkey(&((*swd)->sw_kschedule),
- cri->cri_key, cri->cri_klen / 8);
+ cri->cri_key,
+ cri->cri_klen / 8);
if (error) {
- swcr_freesession(dev, i);
+ swcr_freesession_slot(&swd_base, 0);
return error;
}
}
(*swd)->sw_exf = txf;
break;
-
+
case CRYPTO_MD5_HMAC:
axf = &auth_hash_hmac_md5;
goto authcommon;
axf = &auth_hash_hmac_ripemd_160;
authcommon:
(*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
+ M_WAITOK);
if ((*swd)->sw_ictx == NULL) {
- swcr_freesession(dev, i);
+ swcr_freesession_slot(&swd_base, 0);
return ENOBUFS;
}
(*swd)->sw_octx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
+ M_WAITOK);
if ((*swd)->sw_octx == NULL) {
- swcr_freesession(dev, i);
+ swcr_freesession_slot(&swd_base, 0);
return ENOBUFS;
}
axf = &auth_hash_key_sha1;
auth2common:
(*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
+ M_WAITOK);
if ((*swd)->sw_ictx == NULL) {
- swcr_freesession(dev, i);
+ swcr_freesession_slot(&swd_base, 0);
return ENOBUFS;
}
(*swd)->sw_octx = kmalloc(cri->cri_klen / 8,
- M_CRYPTO_DATA, M_NOWAIT);
+ M_CRYPTO_DATA, M_WAITOK);
if ((*swd)->sw_octx == NULL) {
- swcr_freesession(dev, i);
+ swcr_freesession_slot(&swd_base, 0);
return ENOBUFS;
}
axf = &auth_hash_sha1;
auth3common:
(*swd)->sw_ictx = kmalloc(axf->ctxsize, M_CRYPTO_DATA,
- M_NOWAIT);
+ M_WAITOK);
if ((*swd)->sw_ictx == NULL) {
- swcr_freesession(dev, i);
+ swcr_freesession_slot(&swd_base, 0);
return ENOBUFS;
}
(*swd)->sw_cxf = cxf;
break;
default:
- swcr_freesession(dev, i);
+ swcr_freesession_slot(&swd_base, 0);
return EINVAL;
}
cri = cri->cri_next;
swd = &((*swd)->sw_next);
}
+
+ for (;;) {
+ /*
+ * Atomically allocate a session
+ */
+ spin_lock_wr(&swcr_spin);
+ for (i = swcr_minsesnum; i < swcr_sesnum; ++i) {
+ if (swcr_sessions[i] == NULL)
+ break;
+ }
+ if (i < swcr_sesnum) {
+ swcr_sessions[i] = swd_base;
+ swcr_minsesnum = i + 1;
+ spin_unlock_wr(&swcr_spin);
+ break;
+ }
+ n = swcr_sesnum;
+ spin_unlock_wr(&swcr_spin);
+
+ /*
+ * A larger allocation is required, reallocate the array
+ * and replace, checking for SMP races.
+ */
+ if (n < CRYPTO_SW_SESSIONS)
+ n = CRYPTO_SW_SESSIONS;
+ else
+ n = n * 3 / 2;
+ swd = kmalloc(n * sizeof(struct swcr_data *),
+ M_CRYPTO_DATA, M_WAITOK | M_ZERO);
+
+ spin_lock_wr(&swcr_spin);
+ if (swcr_sesnum >= n) {
+ spin_unlock_wr(&swcr_spin);
+ kfree(swd, M_CRYPTO_DATA);
+ } else if (swcr_sesnum) {
+ bcopy(swcr_sessions, swd,
+ swcr_sesnum * sizeof(struct swcr_data *));
+ oswd = swcr_sessions;
+ swcr_sessions = swd;
+ swcr_sesnum = n;
+ spin_unlock_wr(&swcr_spin);
+ kfree(oswd, M_CRYPTO_DATA);
+ } else {
+ swcr_sessions = swd;
+ swcr_sesnum = n;
+ spin_unlock_wr(&swcr_spin);
+ }
+ }
+
+ *sid = i;
return 0;
}
static int
swcr_freesession(device_t dev, u_int64_t tid)
{
- struct swcr_data *swd;
- struct enc_xform *txf;
- struct auth_hash *axf;
- struct comp_algo *cxf;
u_int32_t sid = CRYPTO_SESID2LID(tid);
if (sid > swcr_sesnum || swcr_sessions == NULL ||
- swcr_sessions[sid] == NULL)
+ swcr_sessions[sid] == NULL) {
return EINVAL;
+ }
/* Silently accept and return */
if (sid == 0)
return 0;
- while ((swd = swcr_sessions[sid]) != NULL) {
- swcr_sessions[sid] = swd->sw_next;
+ return(swcr_freesession_slot(&swcr_sessions[sid], sid));
+}
+
+static
+int
+swcr_freesession_slot(struct swcr_data **swdp, u_int32_t sid)
+{
+ struct enc_xform *txf;
+ struct auth_hash *axf;
+ struct comp_algo *cxf;
+ struct swcr_data *swd;
+ struct swcr_data *swnext;
+
+ /*
+ * Protect session detachment with the spinlock.
+ */
+ spin_lock_wr(&swcr_spin);
+ swnext = *swdp;
+ *swdp = NULL;
+ if (sid && swcr_minsesnum > sid)
+ swcr_minsesnum = sid;
+ spin_unlock_wr(&swcr_spin);
+
+ /*
+ * Clean up at our leisure.
+ */
+ while ((swd = swnext) != NULL) {
+ swnext = swd->sw_next;
+
+ swd->sw_next = NULL;
switch (swd->sw_alg) {
case CRYPTO_DES_CBC:
memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
- swcr_id = crypto_get_driverid(dev,
- CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
+ swcr_id = crypto_get_driverid(dev, CRYPTOCAP_F_SOFTWARE |
+ CRYPTOCAP_F_SYNC |
+ CRYPTOCAP_F_SMP);
if (swcr_id < 0) {
device_printf(dev, "cannot initialize!");
return ENOMEM;
struct auth_hash *SW_axf;
} SWCR_AUTH;
struct {
+ int SW_kschedule_refs;
u_int8_t *SW_kschedule;
struct enc_xform *SW_exf;
} SWCR_ENC;
#define sw_mlen SWCR_UN.SWCR_AUTH.SW_mlen
#define sw_axf SWCR_UN.SWCR_AUTH.SW_axf
#define sw_kschedule SWCR_UN.SWCR_ENC.SW_kschedule
+#define sw_kschedule_refs SWCR_UN.SWCR_ENC.SW_kschedule_refs
#define sw_exf SWCR_UN.SWCR_ENC.SW_exf
#define sw_size SWCR_UN.SWCR_COMP.SW_size
#define sw_cxf SWCR_UN.SWCR_COMP.SW_cxf
zbuf.avail_in = size; /* Total length of data to be processed */
if (!decomp) {
- buf[i].out = kmalloc((u_long) size, M_CRYPTO_DATA,
- M_NOWAIT);
+ buf[i].out = kmalloc((u_long) size, M_CRYPTO_DATA, M_WAITOK);
if (buf[i].out == NULL)
goto bad;
buf[i].size = size;
*/
buf[i].out = kmalloc((u_long) (size * 4),
- M_CRYPTO_DATA, M_NOWAIT);
+ M_CRYPTO_DATA, M_WAITOK);
if (buf[i].out == NULL)
goto bad;
buf[i].size = size * 4;
else if (zbuf.avail_out == 0 && i < (ZBUF - 1)) {
/* we need more output space, allocate size */
buf[i].out = kmalloc((u_long) size,
- M_CRYPTO_DATA, M_NOWAIT);
+ M_CRYPTO_DATA, M_WAITOK);
if (buf[i].out == NULL)
goto bad;
zbuf.next_out = buf[i].out;
end:
result = count = zbuf.total_out;
- *out = kmalloc((u_long) result, M_CRYPTO_DATA, M_NOWAIT);
+ *out = kmalloc((u_long) result, M_CRYPTO_DATA, M_WAITOK);
if (*out == NULL)
goto bad;
if (decomp)
{
void *ptr;
- ptr = kmalloc(type *size, M_CRYPTO_DATA, M_NOWAIT);
+ ptr = kmalloc(type *size, M_CRYPTO_DATA, M_WAITOK);
return ptr;
}
int err;
p = kmalloc(sizeof (des_key_schedule),
- M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+ M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
if (p != NULL) {
des_set_key((des_cblock *) key, p[0]);
err = 0;
des_key_schedule *p;
int err;
- p = kmalloc(3*sizeof (des_key_schedule),
- M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+ p = kmalloc(3 * sizeof(des_key_schedule),
+ M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
if (p != NULL) {
des_set_key((des_cblock *)(key + 0), p[0]);
des_set_key((des_cblock *)(key + 8), p[1]);
{
int err;
- *sched = kmalloc(sizeof(BF_KEY),
- M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+ *sched = kmalloc(sizeof(BF_KEY), M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
if (*sched != NULL) {
BF_set_key((BF_KEY *) *sched, len, key);
err = 0;
{
int err;
- *sched = kmalloc(sizeof(cast_key), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+ *sched = kmalloc(sizeof(cast_key), M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
if (*sched != NULL) {
cast_setkey((cast_key *)*sched, key, len);
err = 0;
/* NB: allocate all the memory that's needed at once */
*sched = kmalloc(10 * (sizeof(u_int8_t *) + 0x100),
- M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
+ M_CRYPTO_DATA, M_INTWAIT | M_ZERO);
if (*sched != NULL) {
u_int8_t** key_tables = (u_int8_t**) *sched;
u_int8_t* table = (u_int8_t*) &key_tables[10];
if (len != 16 && len != 24 && len != 32)
return (EINVAL);
*sched = kmalloc(sizeof(rijndael_ctx), M_CRYPTO_DATA,
- M_NOWAIT|M_ZERO);
+ M_INTWAIT | M_ZERO);
if (*sched != NULL) {
rijndael_set_key((rijndael_ctx *) *sched, (u_char *) key,
len * 8);
if (len != 16 && len != 24 && len != 32)
return (EINVAL);
*sched = kmalloc(sizeof(camellia_ctx), M_CRYPTO_DATA,
- M_NOWAIT|M_ZERO);
+ M_INTWAIT | M_ZERO);
if (*sched != NULL) {
camellia_set_key((camellia_ctx *) *sched, (u_char *) key,
len * 8);
tuxillo's projects / dragonfly.git / commitdiff