Remove all remaining SPL code. Replace the mtd_cpl field in the machine
[dragonfly.git] / sys / opencrypto / crypto.c
1 /*      $FreeBSD: src/sys/opencrypto/crypto.c,v 1.4.2.7 2003/06/03 00:09:02 sam Exp $   */
2 /*      $DragonFly: src/sys/opencrypto/crypto.c,v 1.9 2005/06/16 21:12:49 dillon Exp $  */
3 /*      $OpenBSD: crypto.c,v 1.38 2002/06/11 11:14:29 beck Exp $        */
4 /*
5  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
6  *
7  * This code was written by Angelos D. Keromytis in Athens, Greece, in
8  * February 2000. Network Security Technologies Inc. (NSTI) kindly
9  * supported the development of this code.
10  *
11  * Copyright (c) 2000, 2001 Angelos D. Keromytis
12  *
13  * Permission to use, copy, and modify this software with or without fee
14  * is hereby granted, provided that this entire notice is included in
15  * all source code copies of any software which is or includes a copy or
16  * modification of this software.
17  *
18  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
19  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
20  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
21  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
22  * PURPOSE.
23  */
24
25 #define CRYPTO_TIMING                   /* enable cryptop timing stuff */
26
27 #include <sys/param.h>
28 #include <sys/systm.h>
29 #include <sys/eventhandler.h>
30 #include <sys/kernel.h>
31 #include <sys/kthread.h>
32 #include <sys/malloc.h>
33 #include <sys/proc.h>
34 #include <sys/sysctl.h>
35 #include <sys/interrupt.h>
36 #include <sys/thread2.h>
37 #include <machine/ipl.h>
38
39 #include <vm/vm_zone.h>
40 #include <opencrypto/cryptodev.h>
41 #include <opencrypto/xform.h>                   /* XXX for M_XDATA */
42
43 #define SESID2HID(sid)  (((sid) >> 32) & 0xffffffff)
44
45 /*
46  * Crypto drivers register themselves by allocating a slot in the
47  * crypto_drivers table with crypto_get_driverid() and then registering
48  * each algorithm they support with crypto_register() and crypto_kregister().
49  */
50 static  struct cryptocap *crypto_drivers = NULL;
51 static  int crypto_drivers_num = 0;
52
53 /*
54  * There are two queues for crypto requests; one for symmetric (e.g.
55  * cipher) operations and one for asymmetric (e.g. MOD) operations.
56  * See below for how synchronization is handled.
57  */
58 static  TAILQ_HEAD(,cryptop) crp_q;             /* request queues */
59 static  TAILQ_HEAD(,cryptkop) crp_kq;
60
61 /*
62  * There are two queues for processing completed crypto requests; one
63  * for the symmetric and one for the asymmetric ops.  We only need one
64  * but have two to avoid type futzing (cryptop vs. cryptkop).  See below
65  * for how synchronization is handled.
66  */
67 static  TAILQ_HEAD(,cryptop) crp_ret_q;         /* callback queues */
68 static  TAILQ_HEAD(,cryptkop) crp_ret_kq;
69
70 /*
71  * Crypto op and desciptor data structures are allocated
72  * from separate private zones.
73  */
74 static  vm_zone_t cryptop_zone;
75 static  vm_zone_t cryptodesc_zone;
76
77 int     crypto_usercrypto = 1;          /* userland may open /dev/crypto */
78 SYSCTL_INT(_kern, OID_AUTO, usercrypto, CTLFLAG_RW,
79            &crypto_usercrypto, 0,
80            "Enable/disable user-mode access to crypto support");
81 int     crypto_userasymcrypto = 1;      /* userland may do asym crypto reqs */
82 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
83            &crypto_userasymcrypto, 0,
84            "Enable/disable user-mode access to asymmetric crypto support");
85 int     crypto_devallowsoft = 0;        /* only use hardware crypto for asym */
86 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
87            &crypto_devallowsoft, 0,
88            "Enable/disable use of software asym crypto support");
89
90 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
91
92 /*
93  * Synchronization: read carefully, this is non-trivial.
94  *
95  * Crypto requests are submitted via crypto_dispatch.  No critical
96  * section or lock/interlock guarentees are made on entry.
97  *
98  * Requests are typically passed on the driver directly, but they
99  * may also be queued for processing by a software interrupt thread,
100  * cryptointr, that runs in a critical section.  This thread dispatches 
101  * the requests to crypto drivers (h/w or s/w) who call crypto_done
102  * when a request is complete.  Hardware crypto drivers are assumed
103  * to register their IRQ's as network devices so their interrupt handlers
104  * and subsequent "done callbacks" happen at appropriate protection levels.
105  *
106  * Completed crypto ops are queued for a separate kernel thread that
107  * handles the callbacks with no critical section or lock/interlock
108  * guarentees.  This decoupling insures the crypto driver interrupt service
109  * routine is not delayed while the callback takes place and that callbacks
110  * are delivered after a context switch (as opposed to a software interrupt
111  * that clients must block).
112  *
113  * This scheme is not intended for SMP machines.
114  */ 
115 static  void cryptointr(void *dummy);   /* swi thread to dispatch ops */
116 static  void cryptoret(void);           /* kernel thread for callbacks*/
117 static  struct thread *cryptothread;
118 static  void crypto_destroy(void);
119 static  int crypto_invoke(struct cryptop *crp, int hint);
120 static  int crypto_kinvoke(struct cryptkop *krp, int hint);
121
122 static struct cryptostats cryptostats;
123 SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
124             cryptostats, "Crypto system statistics");
125
126 #ifdef CRYPTO_TIMING
127 static  int crypto_timing = 0;
128 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
129            &crypto_timing, 0, "Enable/disable crypto timing support");
130 #endif
131
132 static int
133 crypto_init(void)
134 {
135         int error;
136
137         cryptop_zone = zinit("cryptop", sizeof (struct cryptop), 0, 0, 1);
138         cryptodesc_zone = zinit("cryptodesc", sizeof (struct cryptodesc),
139                                 0, 0, 1);
140         if (cryptodesc_zone == NULL || cryptop_zone == NULL) {
141                 printf("crypto_init: cannot setup crypto zones\n");
142                 return ENOMEM;
143         }
144
145         crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
146         crypto_drivers = malloc(crypto_drivers_num *
147             sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
148         if (crypto_drivers == NULL) {
149                 printf("crypto_init: cannot malloc driver table\n");
150                 return ENOMEM;
151         }
152
153         TAILQ_INIT(&crp_q);
154         TAILQ_INIT(&crp_kq);
155
156         TAILQ_INIT(&crp_ret_q);
157         TAILQ_INIT(&crp_ret_kq);
158
159         register_swi(SWI_CRYPTO, cryptointr, NULL, "swi_crypto");
160         error = kthread_create((void (*)(void *)) cryptoret, NULL,
161                     &cryptothread, "cryptoret");
162         if (error) {
163                 printf("crypto_init: cannot start cryptoret thread; error %d",
164                         error);
165                 crypto_destroy();
166         }
167         return error;
168 }
169
170 static void
171 crypto_destroy(void)
172 {
173         /* XXX no wait to reclaim zones */
174         if (crypto_drivers != NULL)
175                 free(crypto_drivers, M_CRYPTO_DATA);
176         unregister_swi(SWI_CRYPTO, cryptointr);
177 }
178
179 /*
180  * Initialization code, both for static and dynamic loading.
181  */
182 static int
183 crypto_modevent(module_t mod, int type, void *unused)
184 {
185         int error = EINVAL;
186
187         switch (type) {
188         case MOD_LOAD:
189                 error = crypto_init();
190                 if (error == 0 && bootverbose)
191                         printf("crypto: <crypto core>\n");
192                 break;
193         case MOD_UNLOAD:
194                 /*XXX disallow if active sessions */
195                 error = 0;
196                 crypto_destroy();
197                 break;
198         }
199         return error;
200 }
201
202 static moduledata_t crypto_mod = {
203         "crypto",
204         crypto_modevent,
205         0
206 };
207 MODULE_VERSION(crypto, 1);
208 DECLARE_MODULE(crypto, crypto_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
209
210 /*
211  * Create a new session.
212  */
213 int
214 crypto_newsession(u_int64_t *sid, struct cryptoini *cri, int hard)
215 {
216         struct cryptoini *cr;
217         u_int32_t hid, lid;
218         int err = EINVAL;
219
220         crit_enter();
221
222         if (crypto_drivers == NULL)
223                 goto done;
224
225         /*
226          * The algorithm we use here is pretty stupid; just use the
227          * first driver that supports all the algorithms we need.
228          *
229          * XXX We need more smarts here (in real life too, but that's
230          * XXX another story altogether).
231          */
232
233         for (hid = 0; hid < crypto_drivers_num; hid++) {
234                 /*
235                  * If it's not initialized or has remaining sessions
236                  * referencing it, skip.
237                  */
238                 if (crypto_drivers[hid].cc_newsession == NULL ||
239                     (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP))
240                         continue;
241
242                 /* Hardware required -- ignore software drivers. */
243                 if (hard > 0 &&
244                     (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE))
245                         continue;
246                 /* Software required -- ignore hardware drivers. */
247                 if (hard < 0 &&
248                     (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) == 0)
249                         continue;
250
251                 /* See if all the algorithms are supported. */
252                 for (cr = cri; cr; cr = cr->cri_next)
253                         if (crypto_drivers[hid].cc_alg[cr->cri_alg] == 0)
254                                 break;
255
256                 if (cr == NULL) {
257                         /* Ok, all algorithms are supported. */
258
259                         /*
260                          * Can't do everything in one session.
261                          *
262                          * XXX Fix this. We need to inject a "virtual" session layer right
263                          * XXX about here.
264                          */
265
266                         /* Call the driver initialization routine. */
267                         lid = hid;              /* Pass the driver ID. */
268                         err = crypto_drivers[hid].cc_newsession(
269                                         crypto_drivers[hid].cc_arg, &lid, cri);
270                         if (err == 0) {
271                                 (*sid) = hid;
272                                 (*sid) <<= 32;
273                                 (*sid) |= (lid & 0xffffffff);
274                                 crypto_drivers[hid].cc_sessions++;
275                         }
276                         break;
277                 }
278         }
279 done:
280         crit_exit();
281         return err;
282 }
283
284 /*
285  * Delete an existing session (or a reserved session on an unregistered
286  * driver).
287  */
288 int
289 crypto_freesession(u_int64_t sid)
290 {
291         u_int32_t hid;
292         int err;
293
294         crit_enter();
295
296         if (crypto_drivers == NULL) {
297                 err = EINVAL;
298                 goto done;
299         }
300
301         /* Determine two IDs. */
302         hid = SESID2HID(sid);
303
304         if (hid >= crypto_drivers_num) {
305                 err = ENOENT;
306                 goto done;
307         }
308
309         if (crypto_drivers[hid].cc_sessions)
310                 crypto_drivers[hid].cc_sessions--;
311
312         /* Call the driver cleanup routine, if available. */
313         if (crypto_drivers[hid].cc_freesession)
314                 err = crypto_drivers[hid].cc_freesession(
315                                 crypto_drivers[hid].cc_arg, sid);
316         else
317                 err = 0;
318
319         /*
320          * If this was the last session of a driver marked as invalid,
321          * make the entry available for reuse.
322          */
323         if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP) &&
324             crypto_drivers[hid].cc_sessions == 0)
325                 bzero(&crypto_drivers[hid], sizeof(struct cryptocap));
326
327 done:
328         crit_exit();
329         return err;
330 }
331
332 /*
333  * Return an unused driver id.  Used by drivers prior to registering
334  * support for the algorithms they handle.
335  */
336 int32_t
337 crypto_get_driverid(u_int32_t flags)
338 {
339         struct cryptocap *newdrv;
340         int i;
341
342         crit_enter();
343         for (i = 0; i < crypto_drivers_num; i++)
344                 if (crypto_drivers[i].cc_process == NULL &&
345                     (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0 &&
346                     crypto_drivers[i].cc_sessions == 0)
347                         break;
348
349         /* Out of entries, allocate some more. */
350         if (i == crypto_drivers_num) {
351                 /* Be careful about wrap-around. */
352                 if (2 * crypto_drivers_num <= crypto_drivers_num) {
353                         crit_exit();
354                         printf("crypto: driver count wraparound!\n");
355                         return -1;
356                 }
357
358                 newdrv = malloc(2 * crypto_drivers_num *
359                     sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
360                 if (newdrv == NULL) {
361                         crit_exit();
362                         printf("crypto: no space to expand driver table!\n");
363                         return -1;
364                 }
365
366                 bcopy(crypto_drivers, newdrv,
367                     crypto_drivers_num * sizeof(struct cryptocap));
368
369                 crypto_drivers_num *= 2;
370
371                 free(crypto_drivers, M_CRYPTO_DATA);
372                 crypto_drivers = newdrv;
373         }
374
375         /* NB: state is zero'd on free */
376         crypto_drivers[i].cc_sessions = 1;      /* Mark */
377         crypto_drivers[i].cc_flags = flags;
378         if (bootverbose)
379                 printf("crypto: assign driver %u, flags %u\n", i, flags);
380
381         crit_exit();
382
383         return i;
384 }
385
386 static struct cryptocap *
387 crypto_checkdriver(u_int32_t hid)
388 {
389         if (crypto_drivers == NULL)
390                 return NULL;
391         return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
392 }
393
394 /*
395  * Register support for a key-related algorithm.  This routine
396  * is called once for each algorithm supported a driver.
397  */
398 int
399 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags,
400     int (*kprocess)(void*, struct cryptkop *, int),
401     void *karg)
402 {
403         struct cryptocap *cap;
404         int err;
405
406         crit_enter();
407
408         cap = crypto_checkdriver(driverid);
409         if (cap != NULL &&
410             (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
411                 /*
412                  * XXX Do some performance testing to determine placing.
413                  * XXX We probably need an auxiliary data structure that
414                  * XXX describes relative performances.
415                  */
416
417                 cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
418                 if (bootverbose)
419                         printf("crypto: driver %u registers key alg %u flags %u\n"
420                                 , driverid
421                                 , kalg
422                                 , flags
423                         );
424
425                 if (cap->cc_kprocess == NULL) {
426                         cap->cc_karg = karg;
427                         cap->cc_kprocess = kprocess;
428                 }
429                 err = 0;
430         } else
431                 err = EINVAL;
432
433         crit_exit();
434         return err;
435 }
436
437 /*
438  * Register support for a non-key-related algorithm.  This routine
439  * is called once for each such algorithm supported by a driver.
440  */
441 int
442 crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
443     u_int32_t flags,
444     int (*newses)(void*, u_int32_t*, struct cryptoini*),
445     int (*freeses)(void*, u_int64_t),
446     int (*process)(void*, struct cryptop *, int),
447     void *arg)
448 {
449         struct cryptocap *cap;
450         int err;
451
452         crit_enter();
453
454         cap = crypto_checkdriver(driverid);
455         /* NB: algorithms are in the range [1..max] */
456         if (cap != NULL &&
457             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
458                 /*
459                  * XXX Do some performance testing to determine placing.
460                  * XXX We probably need an auxiliary data structure that
461                  * XXX describes relative performances.
462                  */
463
464                 cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
465                 cap->cc_max_op_len[alg] = maxoplen;
466                 if (bootverbose)
467                         printf("crypto: driver %u registers alg %u flags %u maxoplen %u\n"
468                                 , driverid
469                                 , alg
470                                 , flags
471                                 , maxoplen
472                         );
473
474                 if (cap->cc_process == NULL) {
475                         cap->cc_arg = arg;
476                         cap->cc_newsession = newses;
477                         cap->cc_process = process;
478                         cap->cc_freesession = freeses;
479                         cap->cc_sessions = 0;           /* Unmark */
480                 }
481                 err = 0;
482         } else
483                 err = EINVAL;
484
485         crit_exit();
486         return err;
487 }
488
489 /*
490  * Unregister a crypto driver. If there are pending sessions using it,
491  * leave enough information around so that subsequent calls using those
492  * sessions will correctly detect the driver has been unregistered and
493  * reroute requests.
494  */
495 int
496 crypto_unregister(u_int32_t driverid, int alg)
497 {
498         int i, err;
499         u_int32_t ses;
500         struct cryptocap *cap;
501
502         crit_enter();
503
504         cap = crypto_checkdriver(driverid);
505         if (cap != NULL &&
506             (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
507             cap->cc_alg[alg] != 0) {
508                 cap->cc_alg[alg] = 0;
509                 cap->cc_max_op_len[alg] = 0;
510
511                 /* Was this the last algorithm ? */
512                 for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
513                         if (cap->cc_alg[i] != 0)
514                                 break;
515
516                 if (i == CRYPTO_ALGORITHM_MAX + 1) {
517                         ses = cap->cc_sessions;
518                         bzero(cap, sizeof(struct cryptocap));
519                         if (ses != 0) {
520                                 /*
521                                  * If there are pending sessions, just mark as invalid.
522                                  */
523                                 cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
524                                 cap->cc_sessions = ses;
525                         }
526                 }
527                 err = 0;
528         } else
529                 err = EINVAL;
530
531         crit_exit();
532         return err;
533 }
534
535 /*
536  * Unregister all algorithms associated with a crypto driver.
537  * If there are pending sessions using it, leave enough information
538  * around so that subsequent calls using those sessions will
539  * correctly detect the driver has been unregistered and reroute
540  * requests.
541  */
542 int
543 crypto_unregister_all(u_int32_t driverid)
544 {
545         int i, err;
546         u_int32_t ses;
547         struct cryptocap *cap;
548
549         crit_enter();
550         cap = crypto_checkdriver(driverid);
551         if (cap != NULL) {
552                 for (i = CRYPTO_ALGORITHM_MIN; i <= CRYPTO_ALGORITHM_MAX; i++) {
553                         cap->cc_alg[i] = 0;
554                         cap->cc_max_op_len[i] = 0;
555                 }
556                 ses = cap->cc_sessions;
557                 bzero(cap, sizeof(struct cryptocap));
558                 if (ses != 0) {
559                         /*
560                          * If there are pending sessions, just mark as invalid.
561                          */
562                         cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
563                         cap->cc_sessions = ses;
564                 }
565                 err = 0;
566         } else
567                 err = EINVAL;
568
569         crit_exit();
570         return err;
571 }
572
573 /*
574  * Clear blockage on a driver.  The what parameter indicates whether
575  * the driver is now ready for cryptop's and/or cryptokop's.
576  */
577 int
578 crypto_unblock(u_int32_t driverid, int what)
579 {
580         struct cryptocap *cap;
581         int needwakeup, err;
582
583         crit_enter();
584         cap = crypto_checkdriver(driverid);
585         if (cap != NULL) {
586                 needwakeup = 0;
587                 if (what & CRYPTO_SYMQ) {
588                         needwakeup |= cap->cc_qblocked;
589                         cap->cc_qblocked = 0;
590                 }
591                 if (what & CRYPTO_ASYMQ) {
592                         needwakeup |= cap->cc_kqblocked;
593                         cap->cc_kqblocked = 0;
594                 }
595                 if (needwakeup)
596                         setsoftcrypto();
597                 err = 0;
598         } else
599                 err = EINVAL;
600         crit_exit();
601
602         return err;
603 }
604
605 /*
606  * Dispatch a crypto request to a driver or queue
607  * it, to be processed by the kernel thread.
608  */
609 int
610 crypto_dispatch(struct cryptop *crp)
611 {
612         u_int32_t hid = SESID2HID(crp->crp_sid);
613         int result;
614
615         cryptostats.cs_ops++;
616
617 #ifdef CRYPTO_TIMING
618         if (crypto_timing)
619                 nanouptime(&crp->crp_tstamp);
620 #endif
621         crit_enter();
622         if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
623                 struct cryptocap *cap;
624                 /*
625                  * Caller marked the request to be processed
626                  * immediately; dispatch it directly to the
627                  * driver unless the driver is currently blocked.
628                  */
629                 cap = crypto_checkdriver(hid);
630                 if (cap && !cap->cc_qblocked) {
631                         result = crypto_invoke(crp, 0);
632                         if (result == ERESTART) {
633                                 /*
634                                  * The driver ran out of resources, mark the
635                                  * driver ``blocked'' for cryptop's and put
636                                  * the op on the queue.
637                                  */
638                                 crypto_drivers[hid].cc_qblocked = 1;
639                                 TAILQ_INSERT_HEAD(&crp_q, crp, crp_next);
640                                 cryptostats.cs_blocks++;
641                                 result = 0;
642                         }
643                 } else {
644                         /*
645                          * The driver is blocked, just queue the op until
646                          * it unblocks and the swi thread gets kicked.
647                          */
648                         TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
649                         result = 0;
650                 }
651         } else {
652                 int wasempty = TAILQ_EMPTY(&crp_q);
653                 /*
654                  * Caller marked the request as ``ok to delay'';
655                  * queue it for the swi thread.  This is desirable
656                  * when the operation is low priority and/or suitable
657                  * for batching.
658                  */
659                 TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
660                 if (wasempty)
661                         setsoftcrypto();
662                 result = 0;
663         }
664         crit_exit();
665
666         return result;
667 }
668
669 /*
670  * Add an asymetric crypto request to a queue,
671  * to be processed by the kernel thread.
672  */
673 int
674 crypto_kdispatch(struct cryptkop *krp)
675 {
676         struct cryptocap *cap;
677         int result;
678
679         cryptostats.cs_kops++;
680
681         crit_enter();
682         cap = crypto_checkdriver(krp->krp_hid);
683         if (cap && !cap->cc_kqblocked) {
684                 result = crypto_kinvoke(krp, 0);
685                 if (result == ERESTART) {
686                         /*
687                          * The driver ran out of resources, mark the
688                          * driver ``blocked'' for cryptop's and put
689                          * the op on the queue.
690                          */
691                         crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
692                         TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
693                         cryptostats.cs_kblocks++;
694                 }
695         } else {
696                 /*
697                  * The driver is blocked, just queue the op until
698                  * it unblocks and the swi thread gets kicked.
699                  */
700                 TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
701                 result = 0;
702         }
703         crit_exit();
704
705         return result;
706 }
707
708 /*
709  * Dispatch an assymetric crypto request to the appropriate crypto devices.
710  */
711 static int
712 crypto_kinvoke(struct cryptkop *krp, int hint)
713 {
714         u_int32_t hid;
715         int error;
716
717         /* Sanity checks. */
718         if (krp == NULL)
719                 return EINVAL;
720         if (krp->krp_callback == NULL) {
721                 free(krp, M_XDATA);             /* XXX allocated in cryptodev */
722                 return EINVAL;
723         }
724
725         for (hid = 0; hid < crypto_drivers_num; hid++) {
726                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
727                     !crypto_devallowsoft)
728                         continue;
729                 if (crypto_drivers[hid].cc_kprocess == NULL)
730                         continue;
731                 if ((crypto_drivers[hid].cc_kalg[krp->krp_op] &
732                     CRYPTO_ALG_FLAG_SUPPORTED) == 0)
733                         continue;
734                 break;
735         }
736         if (hid < crypto_drivers_num) {
737                 krp->krp_hid = hid;
738                 error = crypto_drivers[hid].cc_kprocess(
739                                 crypto_drivers[hid].cc_karg, krp, hint);
740         } else
741                 error = ENODEV;
742
743         if (error) {
744                 krp->krp_status = error;
745                 crypto_kdone(krp);
746         }
747         return 0;
748 }
749
750 #ifdef CRYPTO_TIMING
751 static void
752 crypto_tstat(struct cryptotstat *ts, struct timespec *tv)
753 {
754         struct timespec now, t;
755
756         nanouptime(&now);
757         t.tv_sec = now.tv_sec - tv->tv_sec;
758         t.tv_nsec = now.tv_nsec - tv->tv_nsec;
759         if (t.tv_nsec < 0) {
760                 t.tv_sec--;
761                 t.tv_nsec += 1000000000;
762         }
763         timespecadd(&ts->acc, &t);
764         if (timespeccmp(&t, &ts->min, <))
765                 ts->min = t;
766         if (timespeccmp(&t, &ts->max, >))
767                 ts->max = t;
768         ts->count++;
769
770         *tv = now;
771 }
772 #endif
773
774 /*
775  * Dispatch a crypto request to the appropriate crypto devices.
776  */
777 static int
778 crypto_invoke(struct cryptop *crp, int hint)
779 {
780         u_int32_t hid;
781         int (*process)(void*, struct cryptop *, int);
782
783 #ifdef CRYPTO_TIMING
784         if (crypto_timing)
785                 crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
786 #endif
787         /* Sanity checks. */
788         if (crp == NULL)
789                 return EINVAL;
790         if (crp->crp_callback == NULL) {
791                 crypto_freereq(crp);
792                 return EINVAL;
793         }
794         if (crp->crp_desc == NULL) {
795                 crp->crp_etype = EINVAL;
796                 crypto_done(crp);
797                 return 0;
798         }
799
800         hid = SESID2HID(crp->crp_sid);
801         if (hid < crypto_drivers_num) {
802                 if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_CLEANUP)
803                         crypto_freesession(crp->crp_sid);
804                 process = crypto_drivers[hid].cc_process;
805         } else {
806                 process = NULL;
807         }
808
809         if (process == NULL) {
810                 struct cryptodesc *crd;
811                 u_int64_t nid;
812
813                 /*
814                  * Driver has unregistered; migrate the session and return
815                  * an error to the caller so they'll resubmit the op.
816                  */
817                 for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
818                         crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
819
820                 if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0)
821                         crp->crp_sid = nid;
822
823                 crp->crp_etype = EAGAIN;
824                 crypto_done(crp);
825                 return 0;
826         } else {
827                 /*
828                  * Invoke the driver to process the request.
829                  */
830                 return (*process)(crypto_drivers[hid].cc_arg, crp, hint);
831         }
832 }
833
834 /*
835  * Release a set of crypto descriptors.
836  */
837 void
838 crypto_freereq(struct cryptop *crp)
839 {
840         struct cryptodesc *crd;
841
842         if (crp) {
843                 while ((crd = crp->crp_desc) != NULL) {
844                         crp->crp_desc = crd->crd_next;
845                         zfree(cryptodesc_zone, crd);
846                 }
847                 zfree(cryptop_zone, crp);
848         }
849 }
850
851 /*
852  * Acquire a set of crypto descriptors.  The descriptors are self contained
853  * so no special lock/interlock protection is necessary.
854  */
855 struct cryptop *
856 crypto_getreq(int num)
857 {
858         struct cryptodesc *crd;
859         struct cryptop *crp;
860
861         crp = zalloc(cryptop_zone);
862         if (crp != NULL) {
863                 bzero(crp, sizeof (*crp));
864                 while (num--) {
865                         crd = zalloc(cryptodesc_zone);
866                         if (crd == NULL) {
867                                 crypto_freereq(crp);
868                                 crp = NULL;
869                                 break;
870                         }
871                         bzero(crd, sizeof (*crd));
872                         crd->crd_next = crp->crp_desc;
873                         crp->crp_desc = crd;
874                 }
875         }
876         return crp;
877 }
878
879 /*
880  * Invoke the callback on behalf of the driver.
881  */
882 void
883 crypto_done(struct cryptop *crp)
884 {
885         KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
886                 ("crypto_done: op already done, flags 0x%x", crp->crp_flags));
887         crp->crp_flags |= CRYPTO_F_DONE;
888         if (crp->crp_etype != 0)
889                 cryptostats.cs_errs++;
890 #ifdef CRYPTO_TIMING
891         if (crypto_timing)
892                 crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
893 #endif
894         if (crp->crp_flags & CRYPTO_F_CBIMM) {
895                 /*
896                  * Do the callback directly.  This is ok when the
897                  * callback routine does very little (e.g. the
898                  * /dev/crypto callback method just does a wakeup).
899                  */
900 #ifdef CRYPTO_TIMING
901                 if (crypto_timing) {
902                         /*
903                          * NB: We must copy the timestamp before
904                          * doing the callback as the cryptop is
905                          * likely to be reclaimed.
906                          */
907                         struct timespec t = crp->crp_tstamp;
908                         crypto_tstat(&cryptostats.cs_cb, &t);
909                         crp->crp_callback(crp);
910                         crypto_tstat(&cryptostats.cs_finis, &t);
911                 } else
912 #endif
913                         crp->crp_callback(crp);
914         } else {
915                 int wasempty;
916                 /*
917                  * Normal case; queue the callback for the thread.
918                  *
919                  * The return queue is manipulated by the swi thread
920                  * and, potentially, by crypto device drivers calling
921                  * back to mark operations completed.  Thus we need
922                  * to mask both while manipulating the return queue.
923                  */
924                 crit_enter();
925                 wasempty = TAILQ_EMPTY(&crp_ret_q);
926                 TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
927                 if (wasempty)
928                         wakeup_one(&crp_ret_q);
929                 crit_exit();
930         }
931 }
932
933 /*
934  * Invoke the callback on behalf of the driver.
935  */
936 void
937 crypto_kdone(struct cryptkop *krp)
938 {
939         int wasempty;
940
941         if (krp->krp_status != 0)
942                 cryptostats.cs_kerrs++;
943         /*
944          * The return queue is manipulated by the swi thread
945          * and, potentially, by crypto device drivers calling
946          * back to mark operations completed.  Thus we need
947          * to mask both while manipulating the return queue.
948          */
949         crit_enter();
950         wasempty = TAILQ_EMPTY(&crp_ret_kq);
951         TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
952         if (wasempty)
953                 wakeup_one(&crp_ret_q);
954         crit_exit();
955 }
956
957 int
958 crypto_getfeat(int *featp)
959 {
960         int hid, kalg, feat = 0;
961
962         crit_enter();
963         if (!crypto_userasymcrypto)
964                 goto out;         
965
966         for (hid = 0; hid < crypto_drivers_num; hid++) {
967                 if ((crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE) &&
968                     !crypto_devallowsoft) {
969                         continue;
970                 }
971                 if (crypto_drivers[hid].cc_kprocess == NULL)
972                         continue;
973                 for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
974                         if ((crypto_drivers[hid].cc_kalg[kalg] &
975                             CRYPTO_ALG_FLAG_SUPPORTED) != 0)
976                                 feat |=  1 << kalg;
977         }
978 out:
979         crit_exit();
980         *featp = feat;
981         return (0);
982 }
983
984 /*
985  * Software interrupt thread to dispatch crypto requests.
986  */
987 static void
988 cryptointr(void *dummy)
989 {
990         struct cryptop *crp, *submit;
991         struct cryptkop *krp;
992         struct cryptocap *cap;
993         int result, hint;
994
995         cryptostats.cs_intrs++;
996         crit_enter();
997         do {
998                 /*
999                  * Find the first element in the queue that can be
1000                  * processed and look-ahead to see if multiple ops
1001                  * are ready for the same driver.
1002                  */
1003                 submit = NULL;
1004                 hint = 0;
1005                 TAILQ_FOREACH(crp, &crp_q, crp_next) {
1006                         u_int32_t hid = SESID2HID(crp->crp_sid);
1007                         cap = crypto_checkdriver(hid);
1008                         if (cap == NULL || cap->cc_process == NULL) {
1009                                 /* Op needs to be migrated, process it. */
1010                                 if (submit == NULL)
1011                                         submit = crp;
1012                                 break;
1013                         }
1014                         if (!cap->cc_qblocked) {
1015                                 if (submit != NULL) {
1016                                         /*
1017                                          * We stop on finding another op,
1018                                          * regardless whether its for the same
1019                                          * driver or not.  We could keep
1020                                          * searching the queue but it might be
1021                                          * better to just use a per-driver
1022                                          * queue instead.
1023                                          */
1024                                         if (SESID2HID(submit->crp_sid) == hid)
1025                                                 hint = CRYPTO_HINT_MORE;
1026                                         break;
1027                                 } else {
1028                                         submit = crp;
1029                                         if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
1030                                                 break;
1031                                         /* keep scanning for more are q'd */
1032                                 }
1033                         }
1034                 }
1035                 if (submit != NULL) {
1036                         TAILQ_REMOVE(&crp_q, submit, crp_next);
1037                         result = crypto_invoke(submit, hint);
1038                         if (result == ERESTART) {
1039                                 /*
1040                                  * The driver ran out of resources, mark the
1041                                  * driver ``blocked'' for cryptop's and put
1042                                  * the request back in the queue.  It would
1043                                  * best to put the request back where we got
1044                                  * it but that's hard so for now we put it
1045                                  * at the front.  This should be ok; putting
1046                                  * it at the end does not work.
1047                                  */
1048                                 /* XXX validate sid again? */
1049                                 crypto_drivers[SESID2HID(submit->crp_sid)].cc_qblocked = 1;
1050                                 TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
1051                                 cryptostats.cs_blocks++;
1052                         }
1053                 }
1054
1055                 /* As above, but for key ops */
1056                 TAILQ_FOREACH(krp, &crp_kq, krp_next) {
1057                         cap = crypto_checkdriver(krp->krp_hid);
1058                         if (cap == NULL || cap->cc_kprocess == NULL) {
1059                                 /* Op needs to be migrated, process it. */
1060                                 break;
1061                         }
1062                         if (!cap->cc_kqblocked)
1063                                 break;
1064                 }
1065                 if (krp != NULL) {
1066                         TAILQ_REMOVE(&crp_kq, krp, krp_next);
1067                         result = crypto_kinvoke(krp, 0);
1068                         if (result == ERESTART) {
1069                                 /*
1070                                  * The driver ran out of resources, mark the
1071                                  * driver ``blocked'' for cryptkop's and put
1072                                  * the request back in the queue.  It would
1073                                  * best to put the request back where we got
1074                                  * it but that's hard so for now we put it
1075                                  * at the front.  This should be ok; putting
1076                                  * it at the end does not work.
1077                                  */
1078                                 /* XXX validate sid again? */
1079                                 crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
1080                                 TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
1081                                 cryptostats.cs_kblocks++;
1082                         }
1083                 }
1084         } while (submit != NULL || krp != NULL);
1085         crit_exit();
1086 }
1087
1088 /*
1089  * Kernel thread to do callbacks.
1090  */
1091 static void
1092 cryptoret(void)
1093 {
1094         struct cryptop *crp;
1095         struct cryptkop *krp;
1096
1097         crit_enter();
1098         for (;;) {
1099                 crp = TAILQ_FIRST(&crp_ret_q);
1100                 if (crp != NULL)
1101                         TAILQ_REMOVE(&crp_ret_q, crp, crp_next);
1102                 krp = TAILQ_FIRST(&crp_ret_kq);
1103                 if (krp != NULL)
1104                         TAILQ_REMOVE(&crp_ret_kq, krp, krp_next);
1105
1106                 if (crp != NULL || krp != NULL) {
1107                         crit_exit();            /* lower ipl for callbacks */
1108                         if (crp != NULL) {
1109 #ifdef CRYPTO_TIMING
1110                                 if (crypto_timing) {
1111                                         /*
1112                                          * NB: We must copy the timestamp before
1113                                          * doing the callback as the cryptop is
1114                                          * likely to be reclaimed.
1115                                          */
1116                                         struct timespec t = crp->crp_tstamp;
1117                                         crypto_tstat(&cryptostats.cs_cb, &t);
1118                                         crp->crp_callback(crp);
1119                                         crypto_tstat(&cryptostats.cs_finis, &t);
1120                                 } else
1121 #endif
1122                                         crp->crp_callback(crp);
1123                         }
1124                         if (krp != NULL)
1125                                 krp->krp_callback(krp);
1126                         crit_enter();
1127                 } else {
1128                         (void) tsleep(&crp_ret_q, 0, "crypto_wait", 0);
1129                         cryptostats.cs_rets++;
1130                 }
1131         }
1132         /* CODE NOT REACHED (crit_exit() would go here otherwise) */ 
1133 }