sys/dev/crypto/ubsec/ubsec.c

   1 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.12 2003/06/04 17:56:59 sam Exp $ */
   2 /* $DragonFly: src/sys/dev/crypto/ubsec/ubsec.c,v 1.3 2003/08/07 21:16:50 dillon Exp $ */
   3 /*      $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $       */
   4
   5 /*
   6  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
   7  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
   8  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
   9  *
  10  * All rights reserved.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  * 3. All advertising materials mentioning features or use of this software
  21  *    must display the following acknowledgement:
  22  *      This product includes software developed by Jason L. Wright
  23  * 4. The name of the author may not be used to endorse or promote products
  24  *    derived from this software without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  27  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  28  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  29  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  30  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  31  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  32  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  35  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  36  * POSSIBILITY OF SUCH DAMAGE.
  37  *
  38  * Effort sponsored in part by the Defense Advanced Research Projects
  39  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  40  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  41  *
  42  */
  43
  44 /*
  45  * uBsec 5[56]01, 58xx hardware crypto accelerator
  46  */
  47
  48 #include "opt_ubsec.h"
  49
  50 #include <sys/param.h>
  51 #include <sys/systm.h>
  52 #include <sys/proc.h>
  53 #include <sys/errno.h>
  54 #include <sys/malloc.h>
  55 #include <sys/kernel.h>
  56 #include <sys/mbuf.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/endian.h>
  59
  60 #include <vm/vm.h>
  61 #include <vm/pmap.h>
  62
  63 #include <machine/clock.h>
  64 #include <machine/bus.h>
  65 #include <machine/resource.h>
  66 #include <sys/bus.h>
  67 #include <sys/rman.h>
  68
  69 #include <crypto/sha1.h>
  70 #include <opencrypto/cryptodev.h>
  71 #include <opencrypto/cryptosoft.h>
  72 #include <sys/md5.h>
  73 #include <sys/random.h>
  74
  75 #include <bus/pci/pcivar.h>
  76 #include <bus/pci/pcireg.h>
  77
  78 /* grr, #defines for gratuitous incompatibility in queue.h */
  79 #define SIMPLEQ_HEAD            STAILQ_HEAD
  80 #define SIMPLEQ_ENTRY           STAILQ_ENTRY
  81 #define SIMPLEQ_INIT            STAILQ_INIT
  82 #define SIMPLEQ_INSERT_TAIL     STAILQ_INSERT_TAIL
  83 #define SIMPLEQ_EMPTY           STAILQ_EMPTY
  84 #define SIMPLEQ_FIRST           STAILQ_FIRST
  85 #define SIMPLEQ_REMOVE_HEAD     STAILQ_REMOVE_HEAD_UNTIL
  86 #define SIMPLEQ_FOREACH         STAILQ_FOREACH
  87 /* ditto for endian.h */
  88 #define letoh16(x)              le16toh(x)
  89 #define letoh32(x)              le32toh(x)
  90
  91 #ifdef UBSEC_RNDTEST
  92 #include "../rndtest/rndtest.h"
  93 #endif
  94 #include "ubsecreg.h"
  95 #include "ubsecvar.h"
  96
  97 /*
  98  * Prototypes and count for the pci_device structure
  99  */
 100 static  int ubsec_probe(device_t);
 101 static  int ubsec_attach(device_t);
 102 static  int ubsec_detach(device_t);
 103 static  int ubsec_suspend(device_t);
 104 static  int ubsec_resume(device_t);
 105 static  void ubsec_shutdown(device_t);
 106
 107 static device_method_t ubsec_methods[] = {
 108         /* Device interface */
 109         DEVMETHOD(device_probe,         ubsec_probe),
 110         DEVMETHOD(device_attach,        ubsec_attach),
 111         DEVMETHOD(device_detach,        ubsec_detach),
 112         DEVMETHOD(device_suspend,       ubsec_suspend),
 113         DEVMETHOD(device_resume,        ubsec_resume),
 114         DEVMETHOD(device_shutdown,      ubsec_shutdown),
 115
 116         /* bus interface */
 117         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 118         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 119
 120         { 0, 0 }
 121 };
 122 static driver_t ubsec_driver = {
 123         "ubsec",
 124         ubsec_methods,
 125         sizeof (struct ubsec_softc)
 126 };
 127 static devclass_t ubsec_devclass;
 128
 129 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 130 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 131 #ifdef UBSEC_RNDTEST
 132 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 133 #endif
 134
 135 static  void ubsec_intr(void *);
 136 static  int ubsec_newsession(void *, u_int32_t *, struct cryptoini *);
 137 static  int ubsec_freesession(void *, u_int64_t);
 138 static  int ubsec_process(void *, struct cryptop *, int);
 139 static  void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 140 static  void ubsec_feed(struct ubsec_softc *);
 141 static  void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 142 static  void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 143 static  int ubsec_feed2(struct ubsec_softc *);
 144 static  void ubsec_rng(void *);
 145 static  int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 146                              struct ubsec_dma_alloc *, int);
 147 #define ubsec_dma_sync(_dma, _flags) \
 148         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 149 static  void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 150 static  int ubsec_dmamap_aligned(struct ubsec_operand *op);
 151
 152 static  void ubsec_reset_board(struct ubsec_softc *sc);
 153 static  void ubsec_init_board(struct ubsec_softc *sc);
 154 static  void ubsec_init_pciregs(device_t dev);
 155 static  void ubsec_totalreset(struct ubsec_softc *sc);
 156
 157 static  int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 158
 159 static  int ubsec_kprocess(void*, struct cryptkop *, int);
 160 static  int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 161 static  int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 162 static  int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 163 static  void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 164 static  int ubsec_ksigbits(struct crparam *);
 165 static  void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 166 static  void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 167
 168 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters");
 169
 170 #ifdef UBSEC_DEBUG
 171 static  void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 172 static  void ubsec_dump_mcr(struct ubsec_mcr *);
 173 static  void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 174
 175 static  int ubsec_debug = 0;
 176 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 177             0, "control debugging msgs");
 178 #endif
 179
 180 #define READ_REG(sc,r) \
 181         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 182
 183 #define WRITE_REG(sc,reg,val) \
 184         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 185
 186 #define SWAP32(x) (x) = htole32(ntohl((x)))
 187 #define HTOLE32(x) (x) = htole32(x)
 188
 189
 190 struct ubsec_stats ubsecstats;
 191 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 192             ubsec_stats, "driver statistics");
 193
 194 static int
 195 ubsec_probe(device_t dev)
 196 {
 197         if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 198             (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 199              pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 200                 return (0);
 201         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 202             (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 203              pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 204                 return (0);
 205         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 206             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 207              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 208              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 209              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 210              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 211              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 212              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823
 213              ))
 214                 return (0);
 215         return (ENXIO);
 216 }
 217
 218 static const char*
 219 ubsec_partname(struct ubsec_softc *sc)
 220 {
 221         /* XXX sprintf numbers when not decoded */
 222         switch (pci_get_vendor(sc->sc_dev)) {
 223         case PCI_VENDOR_BROADCOM:
 224                 switch (pci_get_device(sc->sc_dev)) {
 225                 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
 226                 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
 227                 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
 228                 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
 229                 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
 230                 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
 231                 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
 232                 }
 233                 return "Broadcom unknown-part";
 234         case PCI_VENDOR_BLUESTEEL:
 235                 switch (pci_get_device(sc->sc_dev)) {
 236                 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 237                 }
 238                 return "Bluesteel unknown-part";
 239         case PCI_VENDOR_SUN:
 240                 switch (pci_get_device(sc->sc_dev)) {
 241                 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 242                 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 243                 }
 244                 return "Sun unknown-part";
 245         }
 246         return "Unknown-vendor unknown-part";
 247 }
 248
 249 static void
 250 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 251 {
 252         u_int32_t *p = (u_int32_t *)buf;
 253         for (count /= sizeof (u_int32_t); count; count--)
 254                 add_true_randomness(*p++);
 255 }
 256
 257 static int
 258 ubsec_attach(device_t dev)
 259 {
 260         struct ubsec_softc *sc = device_get_softc(dev);
 261         struct ubsec_dma *dmap;
 262         u_int32_t cmd, i;
 263         int rid;
 264
 265         KASSERT(sc != NULL, ("ubsec_attach: null software carrier!"));
 266         bzero(sc, sizeof (*sc));
 267         sc->sc_dev = dev;
 268
 269         SIMPLEQ_INIT(&sc->sc_queue);
 270         SIMPLEQ_INIT(&sc->sc_qchip);
 271         SIMPLEQ_INIT(&sc->sc_queue2);
 272         SIMPLEQ_INIT(&sc->sc_qchip2);
 273         SIMPLEQ_INIT(&sc->sc_q2free);
 274
 275         /* XXX handle power management */
 276
 277         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 278
 279         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 280             pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 281                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 282
 283         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 284             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 285              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 286                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 287
 288         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 289             pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 290                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 291                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 292
 293         if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 294              (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 295               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 296               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) ||
 297             (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 298              (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 299               pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 300                 /* NB: the 5821/5822 defines some additional status bits */
 301                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 302                     BS_STAT_MCR2_ALLEMPTY;
 303                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 304                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 305         }
 306
 307         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 308         cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN;
 309         pci_write_config(dev, PCIR_COMMAND, cmd, 4);
 310         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 311
 312         if (!(cmd & PCIM_CMD_MEMEN)) {
 313                 device_printf(dev, "failed to enable memory mapping\n");
 314                 goto bad;
 315         }
 316
 317         if (!(cmd & PCIM_CMD_BUSMASTEREN)) {
 318                 device_printf(dev, "failed to enable bus mastering\n");
 319                 goto bad;
 320         }
 321
 322         /*
 323          * Setup memory-mapping of PCI registers.
 324          */
 325         rid = BS_BAR;
 326         sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid,
 327                                        0, ~0, 1, RF_ACTIVE);
 328         if (sc->sc_sr == NULL) {
 329                 device_printf(dev, "cannot map register space\n");
 330                 goto bad;
 331         }
 332         sc->sc_st = rman_get_bustag(sc->sc_sr);
 333         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 334
 335         /*
 336          * Arrange interrupt line.
 337          */
 338         rid = 0;
 339         sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid,
 340                                         0, ~0, 1, RF_SHAREABLE|RF_ACTIVE);
 341         if (sc->sc_irq == NULL) {
 342                 device_printf(dev, "could not map interrupt\n");
 343                 goto bad1;
 344         }
 345         /*
 346          * NB: Network code assumes we are blocked with splimp()
 347          *     so make sure the IRQ is mapped appropriately.
 348          */
 349         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET,
 350                            ubsec_intr, sc, &sc->sc_ih)) {
 351                 device_printf(dev, "could not establish interrupt\n");
 352                 goto bad2;
 353         }
 354
 355         sc->sc_cid = crypto_get_driverid(0);
 356         if (sc->sc_cid < 0) {
 357                 device_printf(dev, "could not get crypto driver id\n");
 358                 goto bad3;
 359         }
 360
 361         /*
 362          * Setup DMA descriptor area.
 363          */
 364         if (bus_dma_tag_create(NULL,                    /* parent */
 365                                1, 0,                    /* alignment, bounds */
 366                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 367                                BUS_SPACE_MAXADDR,       /* highaddr */
 368                                NULL, NULL,              /* filter, filterarg */
 369                                0x3ffff,                 /* maxsize */
 370                                UBS_MAX_SCATTER,         /* nsegments */
 371                                0xffff,                  /* maxsegsize */
 372                                BUS_DMA_ALLOCNOW,        /* flags */
 373                                &sc->sc_dmat)) {
 374                 device_printf(dev, "cannot allocate DMA tag\n");
 375                 goto bad4;
 376         }
 377         SIMPLEQ_INIT(&sc->sc_freequeue);
 378         dmap = sc->sc_dmaa;
 379         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 380                 struct ubsec_q *q;
 381
 382                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 383                     M_DEVBUF, M_NOWAIT);
 384                 if (q == NULL) {
 385                         device_printf(dev, "cannot allocate queue buffers\n");
 386                         break;
 387                 }
 388
 389                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 390                     &dmap->d_alloc, 0)) {
 391                         device_printf(dev, "cannot allocate dma buffers\n");
 392                         free(q, M_DEVBUF);
 393                         break;
 394                 }
 395                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 396
 397                 q->q_dma = dmap;
 398                 sc->sc_queuea[i] = q;
 399
 400                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 401         }
 402
 403         device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 404
 405         crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
 406             ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 407         crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
 408              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 409         crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
 410              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 411         crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
 412              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 413
 414         /*
 415          * Reset Broadcom chip
 416          */
 417         ubsec_reset_board(sc);
 418
 419         /*
 420          * Init Broadcom specific PCI settings
 421          */
 422         ubsec_init_pciregs(dev);
 423
 424         /*
 425          * Init Broadcom chip
 426          */
 427         ubsec_init_board(sc);
 428
 429 #ifndef UBSEC_NO_RNG
 430         if (sc->sc_flags & UBS_FLAGS_RNG) {
 431                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 432 #ifdef UBSEC_RNDTEST
 433                 sc->sc_rndtest = rndtest_attach(dev);
 434                 if (sc->sc_rndtest)
 435                         sc->sc_harvest = rndtest_harvest;
 436                 else
 437                         sc->sc_harvest = default_harvest;
 438 #else
 439                 sc->sc_harvest = default_harvest;
 440 #endif
 441
 442                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 443                     &sc->sc_rng.rng_q.q_mcr, 0))
 444                         goto skip_rng;
 445
 446                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 447                     &sc->sc_rng.rng_q.q_ctx, 0)) {
 448                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 449                         goto skip_rng;
 450                 }
 451
 452                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 453                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 454                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 455                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 456                         goto skip_rng;
 457                 }
 458
 459                 if (hz >= 100)
 460                         sc->sc_rnghz = hz / 100;
 461                 else
 462                         sc->sc_rnghz = 1;
 463                 callout_init(&sc->sc_rngto);
 464                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 465 skip_rng:
 466         ;
 467         }
 468 #endif /* UBSEC_NO_RNG */
 469
 470         if (sc->sc_flags & UBS_FLAGS_KEY) {
 471                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 472
 473                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0,
 474                         ubsec_kprocess, sc);
 475 #if 0
 476                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0,
 477                         ubsec_kprocess, sc);
 478 #endif
 479         }
 480         return (0);
 481 bad4:
 482         crypto_unregister_all(sc->sc_cid);
 483 bad3:
 484         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 485 bad2:
 486         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 487 bad1:
 488         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 489 bad:
 490         return (ENXIO);
 491 }
 492
 493 /*
 494  * Detach a device that successfully probed.
 495  */
 496 static int
 497 ubsec_detach(device_t dev)
 498 {
 499         struct ubsec_softc *sc = device_get_softc(dev);
 500         int s;
 501
 502         KASSERT(sc != NULL, ("ubsec_detach: null software carrier"));
 503
 504         /* XXX wait/abort active ops */
 505
 506         s = splimp();
 507
 508         callout_stop(&sc->sc_rngto);
 509
 510         crypto_unregister_all(sc->sc_cid);
 511
 512 #ifdef UBSEC_RNDTEST
 513         if (sc->sc_rndtest)
 514                 rndtest_detach(sc->sc_rndtest);
 515 #endif
 516
 517         while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 518                 struct ubsec_q *q;
 519
 520                 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 521                 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
 522                 ubsec_dma_free(sc, &q->q_dma->d_alloc);
 523                 free(q, M_DEVBUF);
 524         }
 525 #ifndef UBSEC_NO_RNG
 526         if (sc->sc_flags & UBS_FLAGS_RNG) {
 527                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 528                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 529                 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 530         }
 531 #endif /* UBSEC_NO_RNG */
 532
 533         bus_generic_detach(dev);
 534         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 535         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 536
 537         bus_dma_tag_destroy(sc->sc_dmat);
 538         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 539
 540         splx(s);
 541
 542         return (0);
 543 }
 544
 545 /*
 546  * Stop all chip i/o so that the kernel's probe routines don't
 547  * get confused by errant DMAs when rebooting.
 548  */
 549 static void
 550 ubsec_shutdown(device_t dev)
 551 {
 552 #ifdef notyet
 553         ubsec_stop(device_get_softc(dev));
 554 #endif
 555 }
 556
 557 /*
 558  * Device suspend routine.
 559  */
 560 static int
 561 ubsec_suspend(device_t dev)
 562 {
 563         struct ubsec_softc *sc = device_get_softc(dev);
 564
 565         KASSERT(sc != NULL, ("ubsec_suspend: null software carrier"));
 566 #ifdef notyet
 567         /* XXX stop the device and save PCI settings */
 568 #endif
 569         sc->sc_suspended = 1;
 570
 571         return (0);
 572 }
 573
 574 static int
 575 ubsec_resume(device_t dev)
 576 {
 577         struct ubsec_softc *sc = device_get_softc(dev);
 578
 579         KASSERT(sc != NULL, ("ubsec_resume: null software carrier"));
 580 #ifdef notyet
 581         /* XXX retore PCI settings and start the device */
 582 #endif
 583         sc->sc_suspended = 0;
 584         return (0);
 585 }
 586
 587 /*
 588  * UBSEC Interrupt routine
 589  */
 590 static void
 591 ubsec_intr(void *arg)
 592 {
 593         struct ubsec_softc *sc = arg;
 594         volatile u_int32_t stat;
 595         struct ubsec_q *q;
 596         struct ubsec_dma *dmap;
 597         int npkts = 0, i;
 598
 599         stat = READ_REG(sc, BS_STAT);
 600         stat &= sc->sc_statmask;
 601         if (stat == 0) {
 602                 return;
 603         }
 604
 605         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
 606
 607         /*
 608          * Check to see if we have any packets waiting for us
 609          */
 610         if ((stat & BS_STAT_MCR1_DONE)) {
 611                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 612                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
 613                         dmap = q->q_dma;
 614
 615                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 616                                 break;
 617
 618                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
 619
 620                         npkts = q->q_nstacked_mcrs;
 621                         sc->sc_nqchip -= 1+npkts;
 622                         /*
 623                          * search for further sc_qchip ubsec_q's that share
 624                          * the same MCR, and complete them too, they must be
 625                          * at the top.
 626                          */
 627                         for (i = 0; i < npkts; i++) {
 628                                 if(q->q_stacked_mcr[i]) {
 629                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
 630                                 } else {
 631                                         break;
 632                                 }
 633                         }
 634                         ubsec_callback(sc, q);
 635                 }
 636
 637                 /*
 638                  * Don't send any more packet to chip if there has been
 639                  * a DMAERR.
 640                  */
 641                 if (!(stat & BS_STAT_DMAERR))
 642                         ubsec_feed(sc);
 643         }
 644
 645         /*
 646          * Check to see if we have any key setups/rng's waiting for us
 647          */
 648         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 649             (stat & BS_STAT_MCR2_DONE)) {
 650                 struct ubsec_q2 *q2;
 651                 struct ubsec_mcr *mcr;
 652
 653                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 654                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 655
 656                         ubsec_dma_sync(&q2->q_mcr,
 657                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 658
 659                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 660                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 661                                 ubsec_dma_sync(&q2->q_mcr,
 662                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 663                                 break;
 664                         }
 665                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next);
 666                         ubsec_callback2(sc, q2);
 667                         /*
 668                          * Don't send any more packet to chip if there has been
 669                          * a DMAERR.
 670                          */
 671                         if (!(stat & BS_STAT_DMAERR))
 672                                 ubsec_feed2(sc);
 673                 }
 674         }
 675
 676         /*
 677          * Check to see if we got any DMA Error
 678          */
 679         if (stat & BS_STAT_DMAERR) {
 680 #ifdef UBSEC_DEBUG
 681                 if (ubsec_debug) {
 682                         volatile u_int32_t a = READ_REG(sc, BS_ERR);
 683
 684                         printf("dmaerr %s@%08x\n",
 685                             (a & BS_ERR_READ) ? "read" : "write",
 686                             a & BS_ERR_ADDR);
 687                 }
 688 #endif /* UBSEC_DEBUG */
 689                 ubsecstats.hst_dmaerr++;
 690                 ubsec_totalreset(sc);
 691                 ubsec_feed(sc);
 692         }
 693
 694         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 695                 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 696 #ifdef UBSEC_DEBUG
 697                 if (ubsec_debug)
 698                         device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 699                                 sc->sc_needwakeup);
 700 #endif /* UBSEC_DEBUG */
 701                 sc->sc_needwakeup &= ~wakeup;
 702                 crypto_unblock(sc->sc_cid, wakeup);
 703         }
 704 }
 705
 706 /*
 707  * ubsec_feed() - aggregate and post requests to chip
 708  */
 709 static void
 710 ubsec_feed(struct ubsec_softc *sc)
 711 {
 712         struct ubsec_q *q, *q2;
 713         int npkts, i;
 714         void *v;
 715         u_int32_t stat;
 716
 717         /*
 718          * Decide how many ops to combine in a single MCR.  We cannot
 719          * aggregate more than UBS_MAX_AGGR because this is the number
 720          * of slots defined in the data structure.  Note that
 721          * aggregation only happens if ops are marked batch'able.
 722          * Aggregating ops reduces the number of interrupts to the host
 723          * but also (potentially) increases the latency for processing
 724          * completed ops as we only get an interrupt when all aggregated
 725          * ops have completed.
 726          */
 727         if (sc->sc_nqueue == 0)
 728                 return;
 729         if (sc->sc_nqueue > 1) {
 730                 npkts = 0;
 731                 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 732                         npkts++;
 733                         if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 734                                 break;
 735                 }
 736         } else
 737                 npkts = 1;
 738         /*
 739          * Check device status before going any further.
 740          */
 741         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 742                 if (stat & BS_STAT_DMAERR) {
 743                         ubsec_totalreset(sc);
 744                         ubsecstats.hst_dmaerr++;
 745                 } else
 746                         ubsecstats.hst_mcr1full++;
 747                 return;
 748         }
 749         if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 750                 ubsecstats.hst_maxqueue = sc->sc_nqueue;
 751         if (npkts > UBS_MAX_AGGR)
 752                 npkts = UBS_MAX_AGGR;
 753         if (npkts < 2)                          /* special case 1 op */
 754                 goto feed1;
 755
 756         ubsecstats.hst_totbatch += npkts-1;
 757 #ifdef UBSEC_DEBUG
 758         if (ubsec_debug)
 759                 printf("merging %d records\n", npkts);
 760 #endif /* UBSEC_DEBUG */
 761
 762         q = SIMPLEQ_FIRST(&sc->sc_queue);
 763         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 764         --sc->sc_nqueue;
 765
 766         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 767         if (q->q_dst_map != NULL)
 768                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 769
 770         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
 771
 772         for (i = 0; i < q->q_nstacked_mcrs; i++) {
 773                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 774                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 775                     BUS_DMASYNC_PREWRITE);
 776                 if (q2->q_dst_map != NULL)
 777                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 778                             BUS_DMASYNC_PREREAD);
 779                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next);
 780                 --sc->sc_nqueue;
 781
 782                 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 783                     sizeof(struct ubsec_mcr_add));
 784                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 785                 q->q_stacked_mcr[i] = q2;
 786         }
 787         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 788         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 789         sc->sc_nqchip += npkts;
 790         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 791                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 792         ubsec_dma_sync(&q->q_dma->d_alloc,
 793             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 794         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 795             offsetof(struct ubsec_dmachunk, d_mcr));
 796         return;
 797
 798 feed1:
 799         q = SIMPLEQ_FIRST(&sc->sc_queue);
 800
 801         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 802         if (q->q_dst_map != NULL)
 803                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 804         ubsec_dma_sync(&q->q_dma->d_alloc,
 805             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 806
 807         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 808             offsetof(struct ubsec_dmachunk, d_mcr));
 809 #ifdef UBSEC_DEBUG
 810         if (ubsec_debug)
 811                 printf("feed1: q->chip %p %08x stat %08x\n",
 812                       q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 813                       stat);
 814 #endif /* UBSEC_DEBUG */
 815         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 816         --sc->sc_nqueue;
 817         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 818         sc->sc_nqchip++;
 819         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 820                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 821         return;
 822 }
 823
 824 /*
 825  * Allocate a new 'session' and return an encoded session id.  'sidp'
 826  * contains our registration id, and should contain an encoded session
 827  * id on successful allocation.
 828  */
 829 static int
 830 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
 831 {
 832         struct cryptoini *c, *encini = NULL, *macini = NULL;
 833         struct ubsec_softc *sc = arg;
 834         struct ubsec_session *ses = NULL;
 835         MD5_CTX md5ctx;
 836         SHA1_CTX sha1ctx;
 837         int i, sesn;
 838
 839         KASSERT(sc != NULL, ("ubsec_newsession: null softc"));
 840         if (sidp == NULL || cri == NULL || sc == NULL)
 841                 return (EINVAL);
 842
 843         for (c = cri; c != NULL; c = c->cri_next) {
 844                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 845                     c->cri_alg == CRYPTO_SHA1_HMAC) {
 846                         if (macini)
 847                                 return (EINVAL);
 848                         macini = c;
 849                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 850                     c->cri_alg == CRYPTO_3DES_CBC) {
 851                         if (encini)
 852                                 return (EINVAL);
 853                         encini = c;
 854                 } else
 855                         return (EINVAL);
 856         }
 857         if (encini == NULL && macini == NULL)
 858                 return (EINVAL);
 859
 860         if (sc->sc_sessions == NULL) {
 861                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 862                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 863                 if (ses == NULL)
 864                         return (ENOMEM);
 865                 sesn = 0;
 866                 sc->sc_nsessions = 1;
 867         } else {
 868                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 869                         if (sc->sc_sessions[sesn].ses_used == 0) {
 870                                 ses = &sc->sc_sessions[sesn];
 871                                 break;
 872                         }
 873                 }
 874
 875                 if (ses == NULL) {
 876                         sesn = sc->sc_nsessions;
 877                         ses = (struct ubsec_session *)malloc((sesn + 1) *
 878                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 879                         if (ses == NULL)
 880                                 return (ENOMEM);
 881                         bcopy(sc->sc_sessions, ses, sesn *
 882                             sizeof(struct ubsec_session));
 883                         bzero(sc->sc_sessions, sesn *
 884                             sizeof(struct ubsec_session));
 885                         free(sc->sc_sessions, M_DEVBUF);
 886                         sc->sc_sessions = ses;
 887                         ses = &sc->sc_sessions[sesn];
 888                         sc->sc_nsessions++;
 889                 }
 890         }
 891
 892         bzero(ses, sizeof(struct ubsec_session));
 893         ses->ses_used = 1;
 894         if (encini) {
 895                 /* get an IV, network byte order */
 896                 /* XXX may read fewer than requested */
 897                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 898
 899                 /* Go ahead and compute key in ubsec's byte order */
 900                 if (encini->cri_alg == CRYPTO_DES_CBC) {
 901                         bcopy(encini->cri_key, &ses->ses_deskey[0], 8);
 902                         bcopy(encini->cri_key, &ses->ses_deskey[2], 8);
 903                         bcopy(encini->cri_key, &ses->ses_deskey[4], 8);
 904                 } else
 905                         bcopy(encini->cri_key, ses->ses_deskey, 24);
 906
 907                 SWAP32(ses->ses_deskey[0]);
 908                 SWAP32(ses->ses_deskey[1]);
 909                 SWAP32(ses->ses_deskey[2]);
 910                 SWAP32(ses->ses_deskey[3]);
 911                 SWAP32(ses->ses_deskey[4]);
 912                 SWAP32(ses->ses_deskey[5]);
 913         }
 914
 915         if (macini) {
 916                 for (i = 0; i < macini->cri_klen / 8; i++)
 917                         macini->cri_key[i] ^= HMAC_IPAD_VAL;
 918
 919                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 920                         MD5Init(&md5ctx);
 921                         MD5Update(&md5ctx, macini->cri_key,
 922                             macini->cri_klen / 8);
 923                         MD5Update(&md5ctx, hmac_ipad_buffer,
 924                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 925                         bcopy(md5ctx.state, ses->ses_hminner,
 926                             sizeof(md5ctx.state));
 927                 } else {
 928                         SHA1Init(&sha1ctx);
 929                         SHA1Update(&sha1ctx, macini->cri_key,
 930                             macini->cri_klen / 8);
 931                         SHA1Update(&sha1ctx, hmac_ipad_buffer,
 932                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 933                         bcopy(sha1ctx.h.b32, ses->ses_hminner,
 934                             sizeof(sha1ctx.h.b32));
 935                 }
 936
 937                 for (i = 0; i < macini->cri_klen / 8; i++)
 938                         macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 939
 940                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 941                         MD5Init(&md5ctx);
 942                         MD5Update(&md5ctx, macini->cri_key,
 943                             macini->cri_klen / 8);
 944                         MD5Update(&md5ctx, hmac_opad_buffer,
 945                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 946                         bcopy(md5ctx.state, ses->ses_hmouter,
 947                             sizeof(md5ctx.state));
 948                 } else {
 949                         SHA1Init(&sha1ctx);
 950                         SHA1Update(&sha1ctx, macini->cri_key,
 951                             macini->cri_klen / 8);
 952                         SHA1Update(&sha1ctx, hmac_opad_buffer,
 953                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 954                         bcopy(sha1ctx.h.b32, ses->ses_hmouter,
 955                             sizeof(sha1ctx.h.b32));
 956                 }
 957
 958                 for (i = 0; i < macini->cri_klen / 8; i++)
 959                         macini->cri_key[i] ^= HMAC_OPAD_VAL;
 960         }
 961
 962         *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
 963         return (0);
 964 }
 965
 966 /*
 967  * Deallocate a session.
 968  */
 969 static int
 970 ubsec_freesession(void *arg, u_int64_t tid)
 971 {
 972         struct ubsec_softc *sc = arg;
 973         int session;
 974         u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 975
 976         KASSERT(sc != NULL, ("ubsec_freesession: null softc"));
 977         if (sc == NULL)
 978                 return (EINVAL);
 979
 980         session = UBSEC_SESSION(sid);
 981         if (session >= sc->sc_nsessions)
 982                 return (EINVAL);
 983
 984         bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
 985         return (0);
 986 }
 987
 988 static void
 989 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
 990 {
 991         struct ubsec_operand *op = arg;
 992
 993         KASSERT(nsegs <= UBS_MAX_SCATTER,
 994                 ("Too many DMA segments returned when mapping operand"));
 995 #ifdef UBSEC_DEBUG
 996         if (ubsec_debug)
 997                 printf("ubsec_op_cb: mapsize %u nsegs %d\n",
 998                         (u_int) mapsize, nsegs);
 999 #endif
1000         op->mapsize = mapsize;
1001         op->nsegs = nsegs;
1002         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1003 }
1004
1005 static int
1006 ubsec_process(void *arg, struct cryptop *crp, int hint)
1007 {
1008         struct ubsec_q *q = NULL;
1009         int err = 0, i, j, s, nicealign;
1010         struct ubsec_softc *sc = arg;
1011         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1012         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1013         int sskip, dskip, stheend, dtheend;
1014         int16_t coffset;
1015         struct ubsec_session *ses;
1016         struct ubsec_pktctx ctx;
1017         struct ubsec_dma *dmap = NULL;
1018
1019         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1020                 ubsecstats.hst_invalid++;
1021                 return (EINVAL);
1022         }
1023         if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1024                 ubsecstats.hst_badsession++;
1025                 return (EINVAL);
1026         }
1027
1028         s = splimp();
1029
1030         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1031                 ubsecstats.hst_queuefull++;
1032                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1033                 splx(s);
1034                 return (ERESTART);
1035         }
1036         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1037         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
1038         splx(s);
1039
1040         dmap = q->q_dma; /* Save dma pointer */
1041         bzero(q, sizeof(struct ubsec_q));
1042         bzero(&ctx, sizeof(ctx));
1043
1044         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1045         q->q_dma = dmap;
1046         ses = &sc->sc_sessions[q->q_sesn];
1047
1048         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1049                 q->q_src_m = (struct mbuf *)crp->crp_buf;
1050                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1051         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1052                 q->q_src_io = (struct uio *)crp->crp_buf;
1053                 q->q_dst_io = (struct uio *)crp->crp_buf;
1054         } else {
1055                 ubsecstats.hst_badflags++;
1056                 err = EINVAL;
1057                 goto errout;    /* XXX we don't handle contiguous blocks! */
1058         }
1059
1060         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1061
1062         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1063         dmap->d_dma->d_mcr.mcr_flags = 0;
1064         q->q_crp = crp;
1065
1066         crd1 = crp->crp_desc;
1067         if (crd1 == NULL) {
1068                 ubsecstats.hst_nodesc++;
1069                 err = EINVAL;
1070                 goto errout;
1071         }
1072         crd2 = crd1->crd_next;
1073
1074         if (crd2 == NULL) {
1075                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1076                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1077                         maccrd = crd1;
1078                         enccrd = NULL;
1079                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1080                     crd1->crd_alg == CRYPTO_3DES_CBC) {
1081                         maccrd = NULL;
1082                         enccrd = crd1;
1083                 } else {
1084                         ubsecstats.hst_badalg++;
1085                         err = EINVAL;
1086                         goto errout;
1087                 }
1088         } else {
1089                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1090                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1091                     (crd2->crd_alg == CRYPTO_DES_CBC ||
1092                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
1093                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1094                         maccrd = crd1;
1095                         enccrd = crd2;
1096                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1097                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
1098                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
1099                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1100                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
1101                         enccrd = crd1;
1102                         maccrd = crd2;
1103                 } else {
1104                         /*
1105                          * We cannot order the ubsec as requested
1106                          */
1107                         ubsecstats.hst_badalg++;
1108                         err = EINVAL;
1109                         goto errout;
1110                 }
1111         }
1112
1113         if (enccrd) {
1114                 encoffset = enccrd->crd_skip;
1115                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
1116
1117                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1118                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1119
1120                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1121                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1122                         else {
1123                                 ctx.pc_iv[0] = ses->ses_iv[0];
1124                                 ctx.pc_iv[1] = ses->ses_iv[1];
1125                         }
1126
1127                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1128                                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1129                                         m_copyback(q->q_src_m,
1130                                             enccrd->crd_inject,
1131                                             8, (caddr_t)ctx.pc_iv);
1132                                 else if (crp->crp_flags & CRYPTO_F_IOV)
1133                                         cuio_copyback(q->q_src_io,
1134                                             enccrd->crd_inject,
1135                                             8, (caddr_t)ctx.pc_iv);
1136                         }
1137                 } else {
1138                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
1139
1140                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1141                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1142                         else if (crp->crp_flags & CRYPTO_F_IMBUF)
1143                                 m_copydata(q->q_src_m, enccrd->crd_inject,
1144                                     8, (caddr_t)ctx.pc_iv);
1145                         else if (crp->crp_flags & CRYPTO_F_IOV)
1146                                 cuio_copydata(q->q_src_io,
1147                                     enccrd->crd_inject, 8,
1148                                     (caddr_t)ctx.pc_iv);
1149                 }
1150
1151                 ctx.pc_deskey[0] = ses->ses_deskey[0];
1152                 ctx.pc_deskey[1] = ses->ses_deskey[1];
1153                 ctx.pc_deskey[2] = ses->ses_deskey[2];
1154                 ctx.pc_deskey[3] = ses->ses_deskey[3];
1155                 ctx.pc_deskey[4] = ses->ses_deskey[4];
1156                 ctx.pc_deskey[5] = ses->ses_deskey[5];
1157                 SWAP32(ctx.pc_iv[0]);
1158                 SWAP32(ctx.pc_iv[1]);
1159         }
1160
1161         if (maccrd) {
1162                 macoffset = maccrd->crd_skip;
1163
1164                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1165                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1166                 else
1167                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1168
1169                 for (i = 0; i < 5; i++) {
1170                         ctx.pc_hminner[i] = ses->ses_hminner[i];
1171                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1172
1173                         HTOLE32(ctx.pc_hminner[i]);
1174                         HTOLE32(ctx.pc_hmouter[i]);
1175                 }
1176         }
1177
1178         if (enccrd && maccrd) {
1179                 /*
1180                  * ubsec cannot handle packets where the end of encryption
1181                  * and authentication are not the same, or where the
1182                  * encrypted part begins before the authenticated part.
1183                  */
1184                 if ((encoffset + enccrd->crd_len) !=
1185                     (macoffset + maccrd->crd_len)) {
1186                         ubsecstats.hst_lenmismatch++;
1187                         err = EINVAL;
1188                         goto errout;
1189                 }
1190                 if (enccrd->crd_skip < maccrd->crd_skip) {
1191                         ubsecstats.hst_skipmismatch++;
1192                         err = EINVAL;
1193                         goto errout;
1194                 }
1195                 sskip = maccrd->crd_skip;
1196                 cpskip = dskip = enccrd->crd_skip;
1197                 stheend = maccrd->crd_len;
1198                 dtheend = enccrd->crd_len;
1199                 coffset = enccrd->crd_skip - maccrd->crd_skip;
1200                 cpoffset = cpskip + dtheend;
1201 #ifdef UBSEC_DEBUG
1202                 if (ubsec_debug) {
1203                         printf("mac: skip %d, len %d, inject %d\n",
1204                             maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1205                         printf("enc: skip %d, len %d, inject %d\n",
1206                             enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1207                         printf("src: skip %d, len %d\n", sskip, stheend);
1208                         printf("dst: skip %d, len %d\n", dskip, dtheend);
1209                         printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1210                             coffset, stheend, cpskip, cpoffset);
1211                 }
1212 #endif
1213         } else {
1214                 cpskip = dskip = sskip = macoffset + encoffset;
1215                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1216                 cpoffset = cpskip + dtheend;
1217                 coffset = 0;
1218         }
1219         ctx.pc_offset = htole16(coffset >> 2);
1220
1221         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
1222                 ubsecstats.hst_nomap++;
1223                 err = ENOMEM;
1224                 goto errout;
1225         }
1226         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1227                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1228                     q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1229                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1230                         q->q_src_map = NULL;
1231                         ubsecstats.hst_noload++;
1232                         err = ENOMEM;
1233                         goto errout;
1234                 }
1235         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1236                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1237                     q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1238                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1239                         q->q_src_map = NULL;
1240                         ubsecstats.hst_noload++;
1241                         err = ENOMEM;
1242                         goto errout;
1243                 }
1244         }
1245         nicealign = ubsec_dmamap_aligned(&q->q_src);
1246
1247         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1248
1249 #ifdef UBSEC_DEBUG
1250         if (ubsec_debug)
1251                 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1252 #endif
1253         for (i = j = 0; i < q->q_src_nsegs; i++) {
1254                 struct ubsec_pktbuf *pb;
1255                 bus_size_t packl = q->q_src_segs[i].ds_len;
1256                 bus_addr_t packp = q->q_src_segs[i].ds_addr;
1257
1258                 if (sskip >= packl) {
1259                         sskip -= packl;
1260                         continue;
1261                 }
1262
1263                 packl -= sskip;
1264                 packp += sskip;
1265                 sskip = 0;
1266
1267                 if (packl > 0xfffc) {
1268                         err = EIO;
1269                         goto errout;
1270                 }
1271
1272                 if (j == 0)
1273                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1274                 else
1275                         pb = &dmap->d_dma->d_sbuf[j - 1];
1276
1277                 pb->pb_addr = htole32(packp);
1278
1279                 if (stheend) {
1280                         if (packl > stheend) {
1281                                 pb->pb_len = htole32(stheend);
1282                                 stheend = 0;
1283                         } else {
1284                                 pb->pb_len = htole32(packl);
1285                                 stheend -= packl;
1286                         }
1287                 } else
1288                         pb->pb_len = htole32(packl);
1289
1290                 if ((i + 1) == q->q_src_nsegs)
1291                         pb->pb_next = 0;
1292                 else
1293                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1294                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1295                 j++;
1296         }
1297
1298         if (enccrd == NULL && maccrd != NULL) {
1299                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1300                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1301                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1302                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1303 #ifdef UBSEC_DEBUG
1304                 if (ubsec_debug)
1305                         printf("opkt: %x %x %x\n",
1306                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1307                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1308                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1309 #endif
1310         } else {
1311                 if (crp->crp_flags & CRYPTO_F_IOV) {
1312                         if (!nicealign) {
1313                                 ubsecstats.hst_iovmisaligned++;
1314                                 err = EINVAL;
1315                                 goto errout;
1316                         }
1317                         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
1318                              &q->q_dst_map)) {
1319                                 ubsecstats.hst_nomap++;
1320                                 err = ENOMEM;
1321                                 goto errout;
1322                         }
1323                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1324                             q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
1325                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1326                                 q->q_dst_map = NULL;
1327                                 ubsecstats.hst_noload++;
1328                                 err = ENOMEM;
1329                                 goto errout;
1330                         }
1331                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1332                         if (nicealign) {
1333                                 q->q_dst = q->q_src;
1334                         } else {
1335                                 int totlen, len;
1336                                 struct mbuf *m, *top, **mp;
1337
1338                                 ubsecstats.hst_unaligned++;
1339                                 totlen = q->q_src_mapsize;
1340                                 if (q->q_src_m->m_flags & M_PKTHDR) {
1341                                         len = MHLEN;
1342                                         MGETHDR(m, M_DONTWAIT, MT_DATA);
1343                                         if (m && !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) {
1344                                                 m_free(m);
1345                                                 m = NULL;
1346                                         }
1347                                 } else {
1348                                         len = MLEN;
1349                                         MGET(m, M_DONTWAIT, MT_DATA);
1350                                 }
1351                                 if (m == NULL) {
1352                                         ubsecstats.hst_nombuf++;
1353                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1354                                         goto errout;
1355                                 }
1356                                 if (totlen >= MINCLSIZE) {
1357                                         MCLGET(m, M_DONTWAIT);
1358                                         if ((m->m_flags & M_EXT) == 0) {
1359                                                 m_free(m);
1360                                                 ubsecstats.hst_nomcl++;
1361                                                 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1362                                                 goto errout;
1363                                         }
1364                                         len = MCLBYTES;
1365                                 }
1366                                 m->m_len = len;
1367                                 top = NULL;
1368                                 mp = &top;
1369
1370                                 while (totlen > 0) {
1371                                         if (top) {
1372                                                 MGET(m, M_DONTWAIT, MT_DATA);
1373                                                 if (m == NULL) {
1374                                                         m_freem(top);
1375                                                         ubsecstats.hst_nombuf++;
1376                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1377                                                         goto errout;
1378                                                 }
1379                                                 len = MLEN;
1380                                         }
1381                                         if (top && totlen >= MINCLSIZE) {
1382                                                 MCLGET(m, M_DONTWAIT);
1383                                                 if ((m->m_flags & M_EXT) == 0) {
1384                                                         *mp = m;
1385                                                         m_freem(top);
1386                                                         ubsecstats.hst_nomcl++;
1387                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1388                                                         goto errout;
1389                                                 }
1390                                                 len = MCLBYTES;
1391                                         }
1392                                         m->m_len = len = min(totlen, len);
1393                                         totlen -= len;
1394                                         *mp = m;
1395                                         mp = &m->m_next;
1396                                 }
1397                                 q->q_dst_m = top;
1398                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1399                                     cpskip, cpoffset);
1400                                 if (bus_dmamap_create(sc->sc_dmat,
1401                                     BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
1402                                         ubsecstats.hst_nomap++;
1403                                         err = ENOMEM;
1404                                         goto errout;
1405                                 }
1406                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1407                                     q->q_dst_map, q->q_dst_m,
1408                                     ubsec_op_cb, &q->q_dst,
1409                                     BUS_DMA_NOWAIT) != 0) {
1410                                         bus_dmamap_destroy(sc->sc_dmat,
1411                                         q->q_dst_map);
1412                                         q->q_dst_map = NULL;
1413                                         ubsecstats.hst_noload++;
1414                                         err = ENOMEM;
1415                                         goto errout;
1416                                 }
1417                         }
1418                 } else {
1419                         ubsecstats.hst_badflags++;
1420                         err = EINVAL;
1421                         goto errout;
1422                 }
1423
1424 #ifdef UBSEC_DEBUG
1425                 if (ubsec_debug)
1426                         printf("dst skip: %d\n", dskip);
1427 #endif
1428                 for (i = j = 0; i < q->q_dst_nsegs; i++) {
1429                         struct ubsec_pktbuf *pb;
1430                         bus_size_t packl = q->q_dst_segs[i].ds_len;
1431                         bus_addr_t packp = q->q_dst_segs[i].ds_addr;
1432
1433                         if (dskip >= packl) {
1434                                 dskip -= packl;
1435                                 continue;
1436                         }
1437
1438                         packl -= dskip;
1439                         packp += dskip;
1440                         dskip = 0;
1441
1442                         if (packl > 0xfffc) {
1443                                 err = EIO;
1444                                 goto errout;
1445                         }
1446
1447                         if (j == 0)
1448                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1449                         else
1450                                 pb = &dmap->d_dma->d_dbuf[j - 1];
1451
1452                         pb->pb_addr = htole32(packp);
1453
1454                         if (dtheend) {
1455                                 if (packl > dtheend) {
1456                                         pb->pb_len = htole32(dtheend);
1457                                         dtheend = 0;
1458                                 } else {
1459                                         pb->pb_len = htole32(packl);
1460                                         dtheend -= packl;
1461                                 }
1462                         } else
1463                                 pb->pb_len = htole32(packl);
1464
1465                         if ((i + 1) == q->q_dst_nsegs) {
1466                                 if (maccrd)
1467                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1468                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1469                                 else
1470                                         pb->pb_next = 0;
1471                         } else
1472                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1473                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1474                         j++;
1475                 }
1476         }
1477
1478         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1479             offsetof(struct ubsec_dmachunk, d_ctx));
1480
1481         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1482                 struct ubsec_pktctx_long *ctxl;
1483
1484                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1485                     offsetof(struct ubsec_dmachunk, d_ctx));
1486
1487                 /* transform small context into long context */
1488                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1489                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1490                 ctxl->pc_flags = ctx.pc_flags;
1491                 ctxl->pc_offset = ctx.pc_offset;
1492                 for (i = 0; i < 6; i++)
1493                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1494                 for (i = 0; i < 5; i++)
1495                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1496                 for (i = 0; i < 5; i++)
1497                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1498                 ctxl->pc_iv[0] = ctx.pc_iv[0];
1499                 ctxl->pc_iv[1] = ctx.pc_iv[1];
1500         } else
1501                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1502                     offsetof(struct ubsec_dmachunk, d_ctx),
1503                     sizeof(struct ubsec_pktctx));
1504
1505         s = splimp();
1506         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1507         sc->sc_nqueue++;
1508         ubsecstats.hst_ipackets++;
1509         ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
1510         if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
1511                 ubsec_feed(sc);
1512         splx(s);
1513         return (0);
1514
1515 errout:
1516         if (q != NULL) {
1517                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1518                         m_freem(q->q_dst_m);
1519
1520                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1521                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1522                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1523                 }
1524                 if (q->q_src_map != NULL) {
1525                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1526                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1527                 }
1528
1529                 s = splimp();
1530                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1531                 splx(s);
1532         }
1533         if (err != ERESTART) {
1534                 crp->crp_etype = err;
1535                 crypto_done(crp);
1536         } else {
1537                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1538         }
1539         return (err);
1540 }
1541
1542 static void
1543 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1544 {
1545         struct cryptop *crp = (struct cryptop *)q->q_crp;
1546         struct cryptodesc *crd;
1547         struct ubsec_dma *dmap = q->q_dma;
1548
1549         ubsecstats.hst_opackets++;
1550         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1551
1552         ubsec_dma_sync(&dmap->d_alloc,
1553             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1554         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1555                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1556                     BUS_DMASYNC_POSTREAD);
1557                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1558                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1559         }
1560         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
1561         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1562         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1563
1564         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1565                 m_freem(q->q_src_m);
1566                 crp->crp_buf = (caddr_t)q->q_dst_m;
1567         }
1568         ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len;
1569
1570         /* copy out IV for future use */
1571         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1572                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1573                         if (crd->crd_alg != CRYPTO_DES_CBC &&
1574                             crd->crd_alg != CRYPTO_3DES_CBC)
1575                                 continue;
1576                         if (crp->crp_flags & CRYPTO_F_IMBUF)
1577                                 m_copydata((struct mbuf *)crp->crp_buf,
1578                                     crd->crd_skip + crd->crd_len - 8, 8,
1579                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1580                         else if (crp->crp_flags & CRYPTO_F_IOV) {
1581                                 cuio_copydata((struct uio *)crp->crp_buf,
1582                                     crd->crd_skip + crd->crd_len - 8, 8,
1583                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1584                         }
1585                         break;
1586                 }
1587         }
1588
1589         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1590                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1591                     crd->crd_alg != CRYPTO_SHA1_HMAC)
1592                         continue;
1593                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1594                         m_copyback((struct mbuf *)crp->crp_buf,
1595                             crd->crd_inject, 12,
1596                             (caddr_t)dmap->d_dma->d_macbuf);
1597                 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1598                         bcopy((caddr_t)dmap->d_dma->d_macbuf,
1599                             crp->crp_mac, 12);
1600                 break;
1601         }
1602         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1603         crypto_done(crp);
1604 }
1605
1606 static void
1607 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1608 {
1609         int i, j, dlen, slen;
1610         caddr_t dptr, sptr;
1611
1612         j = 0;
1613         sptr = srcm->m_data;
1614         slen = srcm->m_len;
1615         dptr = dstm->m_data;
1616         dlen = dstm->m_len;
1617
1618         while (1) {
1619                 for (i = 0; i < min(slen, dlen); i++) {
1620                         if (j < hoffset || j >= toffset)
1621                                 *dptr++ = *sptr++;
1622                         slen--;
1623                         dlen--;
1624                         j++;
1625                 }
1626                 if (slen == 0) {
1627                         srcm = srcm->m_next;
1628                         if (srcm == NULL)
1629                                 return;
1630                         sptr = srcm->m_data;
1631                         slen = srcm->m_len;
1632                 }
1633                 if (dlen == 0) {
1634                         dstm = dstm->m_next;
1635                         if (dstm == NULL)
1636                                 return;
1637                         dptr = dstm->m_data;
1638                         dlen = dstm->m_len;
1639                 }
1640         }
1641 }
1642
1643 /*
1644  * feed the key generator, must be called at splimp() or higher.
1645  */
1646 static int
1647 ubsec_feed2(struct ubsec_softc *sc)
1648 {
1649         struct ubsec_q2 *q;
1650
1651         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1652                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1653                         break;
1654                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1655
1656                 ubsec_dma_sync(&q->q_mcr,
1657                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1658                 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
1659
1660                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1661                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next);
1662                 --sc->sc_nqueue2;
1663                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1664         }
1665         return (0);
1666 }
1667
1668 /*
1669  * Callback for handling random numbers
1670  */
1671 static void
1672 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1673 {
1674         struct cryptkop *krp;
1675         struct ubsec_ctx_keyop *ctx;
1676
1677         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1678         ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
1679
1680         switch (q->q_type) {
1681 #ifndef UBSEC_NO_RNG
1682         case UBS_CTXOP_RNGBYPASS: {
1683                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1684
1685                 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
1686                 (*sc->sc_harvest)(sc->sc_rndtest,
1687                         rng->rng_buf.dma_vaddr,
1688                         UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
1689                 rng->rng_used = 0;
1690                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1691                 break;
1692         }
1693 #endif
1694         case UBS_CTXOP_MODEXP: {
1695                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1696                 u_int rlen, clen;
1697
1698                 krp = me->me_krp;
1699                 rlen = (me->me_modbits + 7) / 8;
1700                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1701
1702                 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
1703                 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
1704                 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
1705                 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
1706
1707                 if (clen < rlen)
1708                         krp->krp_status = E2BIG;
1709                 else {
1710                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1711                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
1712                                     (krp->krp_param[krp->krp_iparams].crp_nbits
1713                                         + 7) / 8);
1714                                 bcopy(me->me_C.dma_vaddr,
1715                                     krp->krp_param[krp->krp_iparams].crp_p,
1716                                     (me->me_modbits + 7) / 8);
1717                         } else
1718                                 ubsec_kshift_l(me->me_shiftbits,
1719                                     me->me_C.dma_vaddr, me->me_normbits,
1720                                     krp->krp_param[krp->krp_iparams].crp_p,
1721                                     krp->krp_param[krp->krp_iparams].crp_nbits);
1722                 }
1723
1724                 crypto_kdone(krp);
1725
1726                 /* bzero all potentially sensitive data */
1727                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1728                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1729                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1730                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1731
1732                 /* Can't free here, so put us on the free list. */
1733                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1734                 break;
1735         }
1736         case UBS_CTXOP_RSAPRIV: {
1737                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1738                 u_int len;
1739
1740                 krp = rp->rpr_krp;
1741                 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
1742                 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
1743
1744                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1745                 bcopy(rp->rpr_msgout.dma_vaddr,
1746                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1747
1748                 crypto_kdone(krp);
1749
1750                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1751                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1752                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1753
1754                 /* Can't free here, so put us on the free list. */
1755                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1756                 break;
1757         }
1758         default:
1759                 device_printf(sc->sc_dev, "unknown ctx op: %x\n",
1760                     letoh16(ctx->ctx_op));
1761                 break;
1762         }
1763 }
1764
1765 #ifndef UBSEC_NO_RNG
1766 static void
1767 ubsec_rng(void *vsc)
1768 {
1769         struct ubsec_softc *sc = vsc;
1770         struct ubsec_q2_rng *rng = &sc->sc_rng;
1771         struct ubsec_mcr *mcr;
1772         struct ubsec_ctx_rngbypass *ctx;
1773         int s;
1774
1775         s = splimp();
1776         if (rng->rng_used) {
1777                 splx(s);
1778                 return;
1779         }
1780         sc->sc_nqueue2++;
1781         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1782                 goto out;
1783
1784         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1785         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1786
1787         mcr->mcr_pkts = htole16(1);
1788         mcr->mcr_flags = 0;
1789         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1790         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1791         mcr->mcr_ipktbuf.pb_len = 0;
1792         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1793         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1794         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1795             UBS_PKTBUF_LEN);
1796         mcr->mcr_opktbuf.pb_next = 0;
1797
1798         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1799         ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
1800         rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
1801
1802         ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
1803
1804         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1805         rng->rng_used = 1;
1806         ubsec_feed2(sc);
1807         ubsecstats.hst_rng++;
1808         splx(s);
1809
1810         return;
1811
1812 out:
1813         /*
1814          * Something weird happened, generate our own call back.
1815          */
1816         sc->sc_nqueue2--;
1817         splx(s);
1818         callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1819 }
1820 #endif /* UBSEC_NO_RNG */
1821
1822 static void
1823 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1824 {
1825         bus_addr_t *paddr = (bus_addr_t*) arg;
1826         *paddr = segs->ds_addr;
1827 }
1828
1829 static int
1830 ubsec_dma_malloc(
1831         struct ubsec_softc *sc,
1832         bus_size_t size,
1833         struct ubsec_dma_alloc *dma,
1834         int mapflags
1835 )
1836 {
1837         int r;
1838
1839         /* XXX could specify sc_dmat as parent but that just adds overhead */
1840         r = bus_dma_tag_create(NULL,                    /* parent */
1841                                1, 0,                    /* alignment, bounds */
1842                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1843                                BUS_SPACE_MAXADDR,       /* highaddr */
1844                                NULL, NULL,              /* filter, filterarg */
1845                                size,                    /* maxsize */
1846                                1,                       /* nsegments */
1847                                size,                    /* maxsegsize */
1848                                BUS_DMA_ALLOCNOW,        /* flags */
1849                                &dma->dma_tag);
1850         if (r != 0) {
1851                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1852                         "bus_dma_tag_create failed; error %u\n", r);
1853                 goto fail_0;
1854         }
1855
1856         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1857         if (r != 0) {
1858                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1859                         "bus_dmamap_create failed; error %u\n", r);
1860                 goto fail_1;
1861         }
1862
1863         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1864                              BUS_DMA_NOWAIT, &dma->dma_map);
1865         if (r != 0) {
1866                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1867                         "bus_dmammem_alloc failed; size %u, error %u\n",
1868                         size, r);
1869                 goto fail_2;
1870         }
1871
1872         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1873                             size,
1874                             ubsec_dmamap_cb,
1875                             &dma->dma_paddr,
1876                             mapflags | BUS_DMA_NOWAIT);
1877         if (r != 0) {
1878                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1879                         "bus_dmamap_load failed; error %u\n", r);
1880                 goto fail_3;
1881         }
1882
1883         dma->dma_size = size;
1884         return (0);
1885
1886 fail_3:
1887         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1888 fail_2:
1889         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1890 fail_1:
1891         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1892         bus_dma_tag_destroy(dma->dma_tag);
1893 fail_0:
1894         dma->dma_map = NULL;
1895         dma->dma_tag = NULL;
1896         return (r);
1897 }
1898
1899 static void
1900 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1901 {
1902         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1903         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1904         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1905         bus_dma_tag_destroy(dma->dma_tag);
1906 }
1907
1908 /*
1909  * Resets the board.  Values in the regesters are left as is
1910  * from the reset (i.e. initial values are assigned elsewhere).
1911  */
1912 static void
1913 ubsec_reset_board(struct ubsec_softc *sc)
1914 {
1915     volatile u_int32_t ctrl;
1916
1917     ctrl = READ_REG(sc, BS_CTRL);
1918     ctrl |= BS_CTRL_RESET;
1919     WRITE_REG(sc, BS_CTRL, ctrl);
1920
1921     /*
1922      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1923      */
1924     DELAY(10);
1925 }
1926
1927 /*
1928  * Init Broadcom registers
1929  */
1930 static void
1931 ubsec_init_board(struct ubsec_softc *sc)
1932 {
1933         u_int32_t ctrl;
1934
1935         ctrl = READ_REG(sc, BS_CTRL);
1936         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1937         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1938
1939         if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
1940                 ctrl |= BS_CTRL_MCR2INT;
1941         else
1942                 ctrl &= ~BS_CTRL_MCR2INT;
1943
1944         if (sc->sc_flags & UBS_FLAGS_HWNORM)
1945                 ctrl &= ~BS_CTRL_SWNORM;
1946
1947         WRITE_REG(sc, BS_CTRL, ctrl);
1948 }
1949
1950 /*
1951  * Init Broadcom PCI registers
1952  */
1953 static void
1954 ubsec_init_pciregs(device_t dev)
1955 {
1956 #if 0
1957         u_int32_t misc;
1958
1959         misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
1960         misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
1961             | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
1962         misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
1963             | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
1964         pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
1965 #endif
1966
1967         /*
1968          * This will set the cache line size to 1, this will
1969          * force the BCM58xx chip just to do burst read/writes.
1970          * Cache line read/writes are to slow
1971          */
1972         pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
1973 }
1974
1975 /*
1976  * Clean up after a chip crash.
1977  * It is assumed that the caller in splimp()
1978  */
1979 static void
1980 ubsec_cleanchip(struct ubsec_softc *sc)
1981 {
1982         struct ubsec_q *q;
1983
1984         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1985                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
1986                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
1987                 ubsec_free_q(sc, q);
1988         }
1989         sc->sc_nqchip = 0;
1990 }
1991
1992 /*
1993  * free a ubsec_q
1994  * It is assumed that the caller is within spimp()
1995  */
1996 static int
1997 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
1998 {
1999         struct ubsec_q *q2;
2000         struct cryptop *crp;
2001         int npkts;
2002         int i;
2003
2004         npkts = q->q_nstacked_mcrs;
2005
2006         for (i = 0; i < npkts; i++) {
2007                 if(q->q_stacked_mcr[i]) {
2008                         q2 = q->q_stacked_mcr[i];
2009
2010                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2011                                 m_freem(q2->q_dst_m);
2012
2013                         crp = (struct cryptop *)q2->q_crp;
2014
2015                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2016
2017                         crp->crp_etype = EFAULT;
2018                         crypto_done(crp);
2019                 } else {
2020                         break;
2021                 }
2022         }
2023
2024         /*
2025          * Free header MCR
2026          */
2027         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2028                 m_freem(q->q_dst_m);
2029
2030         crp = (struct cryptop *)q->q_crp;
2031
2032         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2033
2034         crp->crp_etype = EFAULT;
2035         crypto_done(crp);
2036         return(0);
2037 }
2038
2039 /*
2040  * Routine to reset the chip and clean up.
2041  * It is assumed that the caller is in splimp()
2042  */
2043 static void
2044 ubsec_totalreset(struct ubsec_softc *sc)
2045 {
2046         ubsec_reset_board(sc);
2047         ubsec_init_board(sc);
2048         ubsec_cleanchip(sc);
2049 }
2050
2051 static int
2052 ubsec_dmamap_aligned(struct ubsec_operand *op)
2053 {
2054         int i;
2055
2056         for (i = 0; i < op->nsegs; i++) {
2057                 if (op->segs[i].ds_addr & 3)
2058                         return (0);
2059                 if ((i != (op->nsegs - 1)) &&
2060                     (op->segs[i].ds_len & 3))
2061                         return (0);
2062         }
2063         return (1);
2064 }
2065
2066 static void
2067 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2068 {
2069         switch (q->q_type) {
2070         case UBS_CTXOP_MODEXP: {
2071                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2072
2073                 ubsec_dma_free(sc, &me->me_q.q_mcr);
2074                 ubsec_dma_free(sc, &me->me_q.q_ctx);
2075                 ubsec_dma_free(sc, &me->me_M);
2076                 ubsec_dma_free(sc, &me->me_E);
2077                 ubsec_dma_free(sc, &me->me_C);
2078                 ubsec_dma_free(sc, &me->me_epb);
2079                 free(me, M_DEVBUF);
2080                 break;
2081         }
2082         case UBS_CTXOP_RSAPRIV: {
2083                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2084
2085                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2086                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2087                 ubsec_dma_free(sc, &rp->rpr_msgin);
2088                 ubsec_dma_free(sc, &rp->rpr_msgout);
2089                 free(rp, M_DEVBUF);
2090                 break;
2091         }
2092         default:
2093                 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
2094                 break;
2095         }
2096 }
2097
2098 static int
2099 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint)
2100 {
2101         struct ubsec_softc *sc = arg;
2102         int r;
2103
2104         if (krp == NULL || krp->krp_callback == NULL)
2105                 return (EINVAL);
2106
2107         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2108                 struct ubsec_q2 *q;
2109
2110                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
2111                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next);
2112                 ubsec_kfree(sc, q);
2113         }
2114
2115         switch (krp->krp_op) {
2116         case CRK_MOD_EXP:
2117                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2118                         r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2119                 else
2120                         r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2121                 break;
2122         case CRK_MOD_EXP_CRT:
2123                 return (ubsec_kprocess_rsapriv(sc, krp, hint));
2124         default:
2125                 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
2126                     krp->krp_op);
2127                 krp->krp_status = EOPNOTSUPP;
2128                 crypto_kdone(krp);
2129                 return (0);
2130         }
2131         return (0);                     /* silence compiler */
2132 }
2133
2134 /*
2135  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2136  */
2137 static int
2138 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2139 {
2140         struct ubsec_q2_modexp *me;
2141         struct ubsec_mcr *mcr;
2142         struct ubsec_ctx_modexp *ctx;
2143         struct ubsec_pktbuf *epb;
2144         int s, err = 0;
2145         u_int nbits, normbits, mbits, shiftbits, ebits;
2146
2147         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2148         if (me == NULL) {
2149                 err = ENOMEM;
2150                 goto errout;
2151         }
2152         bzero(me, sizeof *me);
2153         me->me_krp = krp;
2154         me->me_q.q_type = UBS_CTXOP_MODEXP;
2155
2156         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2157         if (nbits <= 512)
2158                 normbits = 512;
2159         else if (nbits <= 768)
2160                 normbits = 768;
2161         else if (nbits <= 1024)
2162                 normbits = 1024;
2163         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2164                 normbits = 1536;
2165         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2166                 normbits = 2048;
2167         else {
2168                 err = E2BIG;
2169                 goto errout;
2170         }
2171
2172         shiftbits = normbits - nbits;
2173
2174         me->me_modbits = nbits;
2175         me->me_shiftbits = shiftbits;
2176         me->me_normbits = normbits;
2177
2178         /* Sanity check: result bits must be >= true modulus bits. */
2179         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2180                 err = ERANGE;
2181                 goto errout;
2182         }
2183
2184         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2185             &me->me_q.q_mcr, 0)) {
2186                 err = ENOMEM;
2187                 goto errout;
2188         }
2189         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2190
2191         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2192             &me->me_q.q_ctx, 0)) {
2193                 err = ENOMEM;
2194                 goto errout;
2195         }
2196
2197         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2198         if (mbits > nbits) {
2199                 err = E2BIG;
2200                 goto errout;
2201         }
2202         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2203                 err = ENOMEM;
2204                 goto errout;
2205         }
2206         ubsec_kshift_r(shiftbits,
2207             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2208             me->me_M.dma_vaddr, normbits);
2209
2210         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2211                 err = ENOMEM;
2212                 goto errout;
2213         }
2214         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2215
2216         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2217         if (ebits > nbits) {
2218                 err = E2BIG;
2219                 goto errout;
2220         }
2221         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2222                 err = ENOMEM;
2223                 goto errout;
2224         }
2225         ubsec_kshift_r(shiftbits,
2226             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2227             me->me_E.dma_vaddr, normbits);
2228
2229         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2230             &me->me_epb, 0)) {
2231                 err = ENOMEM;
2232                 goto errout;
2233         }
2234         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2235         epb->pb_addr = htole32(me->me_E.dma_paddr);
2236         epb->pb_next = 0;
2237         epb->pb_len = htole32(normbits / 8);
2238
2239 #ifdef UBSEC_DEBUG
2240         if (ubsec_debug) {
2241                 printf("Epb ");
2242                 ubsec_dump_pb(epb);
2243         }
2244 #endif
2245
2246         mcr->mcr_pkts = htole16(1);
2247         mcr->mcr_flags = 0;
2248         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2249         mcr->mcr_reserved = 0;
2250         mcr->mcr_pktlen = 0;
2251
2252         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2253         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2254         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2255
2256         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2257         mcr->mcr_opktbuf.pb_next = 0;
2258         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2259
2260 #ifdef DIAGNOSTIC
2261         /* Misaligned output buffer will hang the chip. */
2262         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2263                 panic("%s: modexp invalid addr 0x%x\n",
2264                     device_get_nameunit(sc->sc_dev),
2265                     letoh32(mcr->mcr_opktbuf.pb_addr));
2266         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2267                 panic("%s: modexp invalid len 0x%x\n",
2268                     device_get_nameunit(sc->sc_dev),
2269                     letoh32(mcr->mcr_opktbuf.pb_len));
2270 #endif
2271
2272         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2273         bzero(ctx, sizeof(*ctx));
2274         ubsec_kshift_r(shiftbits,
2275             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2276             ctx->me_N, normbits);
2277         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2278         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2279         ctx->me_E_len = htole16(nbits);
2280         ctx->me_N_len = htole16(nbits);
2281
2282 #ifdef UBSEC_DEBUG
2283         if (ubsec_debug) {
2284                 ubsec_dump_mcr(mcr);
2285                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2286         }
2287 #endif
2288
2289         /*
2290          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2291          * everything else.
2292          */
2293         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2294         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2295         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2296         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2297
2298         /* Enqueue and we're done... */
2299         s = splimp();
2300         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2301         ubsec_feed2(sc);
2302         ubsecstats.hst_modexp++;
2303         splx(s);
2304
2305         return (0);
2306
2307 errout:
2308         if (me != NULL) {
2309                 if (me->me_q.q_mcr.dma_map != NULL)
2310                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2311                 if (me->me_q.q_ctx.dma_map != NULL) {
2312                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2313                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2314                 }
2315                 if (me->me_M.dma_map != NULL) {
2316                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2317                         ubsec_dma_free(sc, &me->me_M);
2318                 }
2319                 if (me->me_E.dma_map != NULL) {
2320                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2321                         ubsec_dma_free(sc, &me->me_E);
2322                 }
2323                 if (me->me_C.dma_map != NULL) {
2324                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2325                         ubsec_dma_free(sc, &me->me_C);
2326                 }
2327                 if (me->me_epb.dma_map != NULL)
2328                         ubsec_dma_free(sc, &me->me_epb);
2329                 free(me, M_DEVBUF);
2330         }
2331         krp->krp_status = err;
2332         crypto_kdone(krp);
2333         return (0);
2334 }
2335
2336 /*
2337  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2338  */
2339 static int
2340 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2341 {
2342         struct ubsec_q2_modexp *me;
2343         struct ubsec_mcr *mcr;
2344         struct ubsec_ctx_modexp *ctx;
2345         struct ubsec_pktbuf *epb;
2346         int s, err = 0;
2347         u_int nbits, normbits, mbits, shiftbits, ebits;
2348
2349         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2350         if (me == NULL) {
2351                 err = ENOMEM;
2352                 goto errout;
2353         }
2354         bzero(me, sizeof *me);
2355         me->me_krp = krp;
2356         me->me_q.q_type = UBS_CTXOP_MODEXP;
2357
2358         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2359         if (nbits <= 512)
2360                 normbits = 512;
2361         else if (nbits <= 768)
2362                 normbits = 768;
2363         else if (nbits <= 1024)
2364                 normbits = 1024;
2365         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2366                 normbits = 1536;
2367         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2368                 normbits = 2048;
2369         else {
2370                 err = E2BIG;
2371                 goto errout;
2372         }
2373
2374         shiftbits = normbits - nbits;
2375
2376         /* XXX ??? */
2377         me->me_modbits = nbits;
2378         me->me_shiftbits = shiftbits;
2379         me->me_normbits = normbits;
2380
2381         /* Sanity check: result bits must be >= true modulus bits. */
2382         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2383                 err = ERANGE;
2384                 goto errout;
2385         }
2386
2387         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2388             &me->me_q.q_mcr, 0)) {
2389                 err = ENOMEM;
2390                 goto errout;
2391         }
2392         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2393
2394         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2395             &me->me_q.q_ctx, 0)) {
2396                 err = ENOMEM;
2397                 goto errout;
2398         }
2399
2400         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2401         if (mbits > nbits) {
2402                 err = E2BIG;
2403                 goto errout;
2404         }
2405         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2406                 err = ENOMEM;
2407                 goto errout;
2408         }
2409         bzero(me->me_M.dma_vaddr, normbits / 8);
2410         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2411             me->me_M.dma_vaddr, (mbits + 7) / 8);
2412
2413         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2414                 err = ENOMEM;
2415                 goto errout;
2416         }
2417         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2418
2419         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2420         if (ebits > nbits) {
2421                 err = E2BIG;
2422                 goto errout;
2423         }
2424         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2425                 err = ENOMEM;
2426                 goto errout;
2427         }
2428         bzero(me->me_E.dma_vaddr, normbits / 8);
2429         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2430             me->me_E.dma_vaddr, (ebits + 7) / 8);
2431
2432         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2433             &me->me_epb, 0)) {
2434                 err = ENOMEM;
2435                 goto errout;
2436         }
2437         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2438         epb->pb_addr = htole32(me->me_E.dma_paddr);
2439         epb->pb_next = 0;
2440         epb->pb_len = htole32((ebits + 7) / 8);
2441
2442 #ifdef UBSEC_DEBUG
2443         if (ubsec_debug) {
2444                 printf("Epb ");
2445                 ubsec_dump_pb(epb);
2446         }
2447 #endif
2448
2449         mcr->mcr_pkts = htole16(1);
2450         mcr->mcr_flags = 0;
2451         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2452         mcr->mcr_reserved = 0;
2453         mcr->mcr_pktlen = 0;
2454
2455         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2456         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2457         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2458
2459         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2460         mcr->mcr_opktbuf.pb_next = 0;
2461         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2462
2463 #ifdef DIAGNOSTIC
2464         /* Misaligned output buffer will hang the chip. */
2465         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2466                 panic("%s: modexp invalid addr 0x%x\n",
2467                     device_get_nameunit(sc->sc_dev),
2468                     letoh32(mcr->mcr_opktbuf.pb_addr));
2469         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2470                 panic("%s: modexp invalid len 0x%x\n",
2471                     device_get_nameunit(sc->sc_dev),
2472                     letoh32(mcr->mcr_opktbuf.pb_len));
2473 #endif
2474
2475         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2476         bzero(ctx, sizeof(*ctx));
2477         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2478             (nbits + 7) / 8);
2479         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2480         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2481         ctx->me_E_len = htole16(ebits);
2482         ctx->me_N_len = htole16(nbits);
2483
2484 #ifdef UBSEC_DEBUG
2485         if (ubsec_debug) {
2486                 ubsec_dump_mcr(mcr);
2487                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2488         }
2489 #endif
2490
2491         /*
2492          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2493          * everything else.
2494          */
2495         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2496         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2497         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2498         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2499
2500         /* Enqueue and we're done... */
2501         s = splimp();
2502         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2503         ubsec_feed2(sc);
2504         splx(s);
2505
2506         return (0);
2507
2508 errout:
2509         if (me != NULL) {
2510                 if (me->me_q.q_mcr.dma_map != NULL)
2511                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2512                 if (me->me_q.q_ctx.dma_map != NULL) {
2513                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2514                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2515                 }
2516                 if (me->me_M.dma_map != NULL) {
2517                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2518                         ubsec_dma_free(sc, &me->me_M);
2519                 }
2520                 if (me->me_E.dma_map != NULL) {
2521                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2522                         ubsec_dma_free(sc, &me->me_E);
2523                 }
2524                 if (me->me_C.dma_map != NULL) {
2525                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2526                         ubsec_dma_free(sc, &me->me_C);
2527                 }
2528                 if (me->me_epb.dma_map != NULL)
2529                         ubsec_dma_free(sc, &me->me_epb);
2530                 free(me, M_DEVBUF);
2531         }
2532         krp->krp_status = err;
2533         crypto_kdone(krp);
2534         return (0);
2535 }
2536
2537 static int
2538 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2539 {
2540         struct ubsec_q2_rsapriv *rp = NULL;
2541         struct ubsec_mcr *mcr;
2542         struct ubsec_ctx_rsapriv *ctx;
2543         int s, err = 0;
2544         u_int padlen, msglen;
2545
2546         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2547         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2548         if (msglen > padlen)
2549                 padlen = msglen;
2550
2551         if (padlen <= 256)
2552                 padlen = 256;
2553         else if (padlen <= 384)
2554                 padlen = 384;
2555         else if (padlen <= 512)
2556                 padlen = 512;
2557         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2558                 padlen = 768;
2559         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2560                 padlen = 1024;
2561         else {
2562                 err = E2BIG;
2563                 goto errout;
2564         }
2565
2566         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2567                 err = E2BIG;
2568                 goto errout;
2569         }
2570
2571         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2572                 err = E2BIG;
2573                 goto errout;
2574         }
2575
2576         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2577                 err = E2BIG;
2578                 goto errout;
2579         }
2580
2581         rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
2582         if (rp == NULL)
2583                 return (ENOMEM);
2584         bzero(rp, sizeof *rp);
2585         rp->rpr_krp = krp;
2586         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2587
2588         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2589             &rp->rpr_q.q_mcr, 0)) {
2590                 err = ENOMEM;
2591                 goto errout;
2592         }
2593         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2594
2595         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2596             &rp->rpr_q.q_ctx, 0)) {
2597                 err = ENOMEM;
2598                 goto errout;
2599         }
2600         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2601         bzero(ctx, sizeof *ctx);
2602
2603         /* Copy in p */
2604         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2605             &ctx->rpr_buf[0 * (padlen / 8)],
2606             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2607
2608         /* Copy in q */
2609         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2610             &ctx->rpr_buf[1 * (padlen / 8)],
2611             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2612
2613         /* Copy in dp */
2614         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2615             &ctx->rpr_buf[2 * (padlen / 8)],
2616             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2617
2618         /* Copy in dq */
2619         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2620             &ctx->rpr_buf[3 * (padlen / 8)],
2621             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2622
2623         /* Copy in pinv */
2624         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2625             &ctx->rpr_buf[4 * (padlen / 8)],
2626             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2627
2628         msglen = padlen * 2;
2629
2630         /* Copy in input message (aligned buffer/length). */
2631         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2632                 /* Is this likely? */
2633                 err = E2BIG;
2634                 goto errout;
2635         }
2636         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2637                 err = ENOMEM;
2638                 goto errout;
2639         }
2640         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2641         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2642             rp->rpr_msgin.dma_vaddr,
2643             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2644
2645         /* Prepare space for output message (aligned buffer/length). */
2646         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2647                 /* Is this likely? */
2648                 err = E2BIG;
2649                 goto errout;
2650         }
2651         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2652                 err = ENOMEM;
2653                 goto errout;
2654         }
2655         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2656
2657         mcr->mcr_pkts = htole16(1);
2658         mcr->mcr_flags = 0;
2659         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2660         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2661         mcr->mcr_ipktbuf.pb_next = 0;
2662         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2663         mcr->mcr_reserved = 0;
2664         mcr->mcr_pktlen = htole16(msglen);
2665         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2666         mcr->mcr_opktbuf.pb_next = 0;
2667         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2668
2669 #ifdef DIAGNOSTIC
2670         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2671                 panic("%s: rsapriv: invalid msgin %x(0x%x)",
2672                     device_get_nameunit(sc->sc_dev),
2673                     rp->rpr_msgin.dma_paddr, rp->rpr_msgin.dma_size);
2674         }
2675         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2676                 panic("%s: rsapriv: invalid msgout %x(0x%x)",
2677                     device_get_nameunit(sc->sc_dev),
2678                     rp->rpr_msgout.dma_paddr, rp->rpr_msgout.dma_size);
2679         }
2680 #endif
2681
2682         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2683         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2684         ctx->rpr_q_len = htole16(padlen);
2685         ctx->rpr_p_len = htole16(padlen);
2686
2687         /*
2688          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2689          * everything else.
2690          */
2691         ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
2692         ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
2693
2694         /* Enqueue and we're done... */
2695         s = splimp();
2696         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2697         ubsec_feed2(sc);
2698         ubsecstats.hst_modexpcrt++;
2699         splx(s);
2700         return (0);
2701
2702 errout:
2703         if (rp != NULL) {
2704                 if (rp->rpr_q.q_mcr.dma_map != NULL)
2705                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2706                 if (rp->rpr_msgin.dma_map != NULL) {
2707                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2708                         ubsec_dma_free(sc, &rp->rpr_msgin);
2709                 }
2710                 if (rp->rpr_msgout.dma_map != NULL) {
2711                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2712                         ubsec_dma_free(sc, &rp->rpr_msgout);
2713                 }
2714                 free(rp, M_DEVBUF);
2715         }
2716         krp->krp_status = err;
2717         crypto_kdone(krp);
2718         return (0);
2719 }
2720
2721 #ifdef UBSEC_DEBUG
2722 static void
2723 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
2724 {
2725         printf("addr 0x%x (0x%x) next 0x%x\n",
2726             pb->pb_addr, pb->pb_len, pb->pb_next);
2727 }
2728
2729 static void
2730 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2731 {
2732         printf("CTX (0x%x):\n", c->ctx_len);
2733         switch (letoh16(c->ctx_op)) {
2734         case UBS_CTXOP_RNGBYPASS:
2735         case UBS_CTXOP_RNGSHA1:
2736                 break;
2737         case UBS_CTXOP_MODEXP:
2738         {
2739                 struct ubsec_ctx_modexp *cx = (void *)c;
2740                 int i, len;
2741
2742                 printf(" Elen %u, Nlen %u\n",
2743                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2744                 len = (cx->me_N_len + 7)/8;
2745                 for (i = 0; i < len; i++)
2746                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2747                 printf("\n");
2748                 break;
2749         }
2750         default:
2751                 printf("unknown context: %x\n", c->ctx_op);
2752         }
2753         printf("END CTX\n");
2754 }
2755
2756 static void
2757 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2758 {
2759         volatile struct ubsec_mcr_add *ma;
2760         int i;
2761
2762         printf("MCR:\n");
2763         printf(" pkts: %u, flags 0x%x\n",
2764             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2765         ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2766         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2767                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2768                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2769                     letoh16(ma->mcr_reserved));
2770                 printf(" %d: ipkt ", i);
2771                 ubsec_dump_pb(&ma->mcr_ipktbuf);
2772                 printf(" %d: opkt ", i);
2773                 ubsec_dump_pb(&ma->mcr_opktbuf);
2774                 ma++;
2775         }
2776         printf("END MCR\n");
2777 }
2778 #endif /* UBSEC_DEBUG */
2779
2780 /*
2781  * Return the number of significant bits of a big number.
2782  */
2783 static int
2784 ubsec_ksigbits(struct crparam *cr)
2785 {
2786         u_int plen = (cr->crp_nbits + 7) / 8;
2787         int i, sig = plen * 8;
2788         u_int8_t c, *p = cr->crp_p;
2789
2790         for (i = plen - 1; i >= 0; i--) {
2791                 c = p[i];
2792                 if (c != 0) {
2793                         while ((c & 0x80) == 0) {
2794                                 sig--;
2795                                 c <<= 1;
2796                         }
2797                         break;
2798                 }
2799                 sig -= 8;
2800         }
2801         return (sig);
2802 }
2803
2804 static void
2805 ubsec_kshift_r(
2806         u_int shiftbits,
2807         u_int8_t *src, u_int srcbits,
2808         u_int8_t *dst, u_int dstbits)
2809 {
2810         u_int slen, dlen;
2811         int i, si, di, n;
2812
2813         slen = (srcbits + 7) / 8;
2814         dlen = (dstbits + 7) / 8;
2815
2816         for (i = 0; i < slen; i++)
2817                 dst[i] = src[i];
2818         for (i = 0; i < dlen - slen; i++)
2819                 dst[slen + i] = 0;
2820
2821         n = shiftbits / 8;
2822         if (n != 0) {
2823                 si = dlen - n - 1;
2824                 di = dlen - 1;
2825                 while (si >= 0)
2826                         dst[di--] = dst[si--];
2827                 while (di >= 0)
2828                         dst[di--] = 0;
2829         }
2830
2831         n = shiftbits % 8;
2832         if (n != 0) {
2833                 for (i = dlen - 1; i > 0; i--)
2834                         dst[i] = (dst[i] << n) |
2835                             (dst[i - 1] >> (8 - n));
2836                 dst[0] = dst[0] << n;
2837         }
2838 }
2839
2840 static void
2841 ubsec_kshift_l(
2842         u_int shiftbits,
2843         u_int8_t *src, u_int srcbits,
2844         u_int8_t *dst, u_int dstbits)
2845 {
2846         int slen, dlen, i, n;
2847
2848         slen = (srcbits + 7) / 8;
2849         dlen = (dstbits + 7) / 8;
2850
2851         n = shiftbits / 8;
2852         for (i = 0; i < slen; i++)
2853                 dst[i] = src[i + n];
2854         for (i = 0; i < dlen - slen; i++)
2855                 dst[slen + i] = 0;
2856
2857         n = shiftbits % 8;
2858         if (n != 0) {
2859                 for (i = 0; i < (dlen - 1); i++)
2860                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2861                 dst[dlen - 1] = dst[dlen - 1] >> n;
2862         }
2863 }