From: Matthew Dillon <dillon@apollo.backplane.com>
Date: Fri, 20 Aug 2010 20:05:54 +0000 (-0700)
Subject: kernel - opencrypto - optimize chained synchronous callbacks for soft crypto
X-Git-Tag: v2.9.0~470
X-Git-Url: https://gitweb.dragonflybsd.org/dragonfly.git/commitdiff_plain/345ee1fbc233997066e5042695d9c4a5a299803c

kernel - opencrypto - optimize chained synchronous callbacks for soft crypto

* When a crypto operation is dispatched to a crypto thread and makes a
  synchronous callback from that thread, and the callback function
  dispatches a followup crypto as part of a chain, we ignore F_BATCH
  and run the followup crypto directly.

  Since we are already in a crypto thread there's no point dispatching
  the followup crypto to another crypto thread.

  Increases dm_crypt's essiv performance w/software crypto by 7%.

* Warning: The chain length for crypto operations with synchronous
  callbacks should be limited to 2 or 3 to avoid blowing up the kernel's
  thread stack.
---

diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
index ae4743df01..8a77bae578 100644
--- a/sys/opencrypto/crypto.c
+++ b/sys/opencrypto/crypto.c
@@ -169,6 +169,10 @@ int	crypto_devallowsoft = 0;	/* only use hardware crypto for asym */
 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
 	   &crypto_devallowsoft, 0,
 	   "Enable/disable use of software asym crypto support");
+int	crypto_altdispatch = 0;		/* dispatch to alternative cpu */
+SYSCTL_INT(_kern, OID_AUTO, cryptoaltdispatch, CTLFLAG_RW,
+	   &crypto_altdispatch, 0,
+	   "Do not queue crypto op on current cpu");
 
 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
 
@@ -811,12 +815,16 @@ crypto_dispatch(struct cryptop *crp)
 
 	hid = CRYPTO_SESID2HID(crp->crp_sid);
 
-	if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
-		/*
-		 * Caller marked the request to be processed
-		 * immediately; dispatch it directly to the
-		 * driver unless the driver is currently blocked.
-		 */
+	/*
+	 * Dispatch the crypto op directly to the driver if the caller
+	 * marked the request to be processed immediately or this is
+	 * a synchronous callback chain occuring from within a crypto
+	 * processing thread.
+	 *
+	 * Fall through to queueing the driver is blocked.
+	 */
+	if ((crp->crp_flags & CRYPTO_F_BATCH) == 0 ||
+	    (curthread->td_flags & TDF_CRYPTO)) {
 		cap = crypto_checkdriver(hid);
 		/* Driver cannot disappeared when there is an active session. */
 		KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
@@ -832,10 +840,13 @@ crypto_dispatch(struct cryptop *crp)
 	}
 
 	/*
-	 * Dispatch to a cpu for action if possible
+	 * Dispatch to a cpu for action if possible.  Dispatch to a different
+	 * cpu than the current cpu.
 	 */
 	if (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SMP) {
 		n = atomic_fetchadd_int(&dispatch_rover, 1) & 255;
+		if (crypto_altdispatch && mycpu->gd_cpuid == n)
+			++n;
 		n = n % ncpus;
 	} else {
 		n = 0;
@@ -1296,6 +1307,8 @@ crypto_proc(void *arg)
 
 	CRYPTO_Q_LOCK(tdinfo);
 
+	curthread->td_flags |= TDF_CRYPTO;
+
 	for (;;) {
 		/*
 		 * Find the first element in the queue that can be
diff --git a/sys/sys/thread.h b/sys/sys/thread.h
index dd6ca56a5a..ac64135af6 100644
--- a/sys/sys/thread.h
+++ b/sys/sys/thread.h
@@ -310,6 +310,7 @@ struct thread {
 #define TDF_USINGFP		0x00800000	/* thread using fp coproc */
 #define TDF_KERNELFP		0x01000000	/* kernel using fp coproc */
 #define TDF_NETWORK		0x02000000	/* network proto thread */
+#define TDF_CRYPTO		0x04000000	/* crypto thread */
 
 /*
  * Thread priorities.  Typically only one thread from any given