kernel - opencrypto - optimize chained synchronous callbacks for soft crypto

* When a crypto operation is dispatched to a crypto thread and makes a
  synchronous callback from that thread, and the callback function
  dispatches a followup crypto as part of a chain, we ignore F_BATCH
  and run the followup crypto directly.

  Since we are already in a crypto thread there's no point dispatching
  the followup crypto to another crypto thread.

  Increases dm_crypt's essiv performance w/software crypto by 7%.

* Warning: The chain length for crypto operations with synchronous
  callbacks should be limited to 2 or 3 to avoid blowing up the kernel's
  thread stack.

diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
index ae4743d..8a77bae 100644 (file)
--- a/sys/opencrypto/crypto.c
+++ b/sys/opencrypto/crypto.c
@@ -169,6 +169,10 @@ int        crypto_devallowsoft = 0;        /* only use hardware crypto for asym */
 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
           &crypto_devallowsoft, 0,
           "Enable/disable use of software asym crypto support");
+int    crypto_altdispatch = 0;         /* dispatch to alternative cpu */
+SYSCTL_INT(_kern, OID_AUTO, cryptoaltdispatch, CTLFLAG_RW,
+          &crypto_altdispatch, 0,
+          "Do not queue crypto op on current cpu");
 
 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
 
@@ -811,12 +815,16 @@ crypto_dispatch(struct cryptop *crp)
 
        hid = CRYPTO_SESID2HID(crp->crp_sid);
 
-       if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
-               /*
-                * Caller marked the request to be processed
-                * immediately; dispatch it directly to the
-                * driver unless the driver is currently blocked.
-                */
+       /*
+        * Dispatch the crypto op directly to the driver if the caller
+        * marked the request to be processed immediately or this is
+        * a synchronous callback chain occuring from within a crypto
+        * processing thread.
+        *
+        * Fall through to queueing the driver is blocked.
+        */
+       if ((crp->crp_flags & CRYPTO_F_BATCH) == 0 ||
+           (curthread->td_flags & TDF_CRYPTO)) {
                cap = crypto_checkdriver(hid);
                /* Driver cannot disappeared when there is an active session. */
                KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
@@ -832,10 +840,13 @@ crypto_dispatch(struct cryptop *crp)
        }
 
        /*
-        * Dispatch to a cpu for action if possible
+        * Dispatch to a cpu for action if possible.  Dispatch to a different
+        * cpu than the current cpu.
         */
        if (CRYPTO_SESID2CAPS(crp->crp_sid) & CRYPTOCAP_F_SMP) {
                n = atomic_fetchadd_int(&dispatch_rover, 1) & 255;
+               if (crypto_altdispatch && mycpu->gd_cpuid == n)
+                       ++n;
                n = n % ncpus;
        } else {
                n = 0;
@@ -1296,6 +1307,8 @@ crypto_proc(void *arg)
 
        CRYPTO_Q_LOCK(tdinfo);
 
+       curthread->td_flags |= TDF_CRYPTO;
+
        for (;;) {
                /*
                 * Find the first element in the queue that can be

diff --git a/sys/sys/thread.h b/sys/sys/thread.h
index dd6ca56..ac64135 100644 (file)
--- a/sys/sys/thread.h
+++ b/sys/sys/thread.h
@@ -310,6 +310,7 @@ struct thread {
 #define TDF_USINGFP            0x00800000      /* thread using fp coproc */
 #define TDF_KERNELFP           0x01000000      /* kernel using fp coproc */
 #define TDF_NETWORK            0x02000000      /* network proto thread */
+#define TDF_CRYPTO             0x04000000      /* crypto thread */
 
 /*
  * Thread priorities.  Typically only one thread from any given