sys/dev/crypto/ubsec/ubsec.c

   1 /* $FreeBSD: src/sys/dev/ubsec/ubsec.c,v 1.6.2.12 2003/06/04 17:56:59 sam Exp $ */
   2 /*      $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $       */
   3
   4 /*
   5  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
   6  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
   7  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
   8  *
   9  * All rights reserved.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. All advertising materials mentioning features or use of this software
  20  *    must display the following acknowledgement:
  21  *      This product includes software developed by Jason L. Wright
  22  * 4. The name of the author may not be used to endorse or promote products
  23  *    derived from this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  27  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  28  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  29  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  30  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  31  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  33  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  34  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35  * POSSIBILITY OF SUCH DAMAGE.
  36  *
  37  * Effort sponsored in part by the Defense Advanced Research Projects
  38  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  39  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  40  *
  41  */
  42
  43 /*
  44  * uBsec 5[56]01, 58xx hardware crypto accelerator
  45  */
  46
  47 #include "opt_ubsec.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/proc.h>
  52 #include <sys/errno.h>
  53 #include <sys/malloc.h>
  54 #include <sys/kernel.h>
  55 #include <sys/mbuf.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/endian.h>
  58
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61
  62 #include <machine/clock.h>
  63 #include <machine/bus.h>
  64 #include <machine/resource.h>
  65 #include <sys/bus.h>
  66 #include <sys/rman.h>
  67
  68 #include <crypto/sha1.h>
  69 #include <opencrypto/cryptodev.h>
  70 #include <opencrypto/cryptosoft.h>
  71 #include <sys/md5.h>
  72 #include <sys/random.h>
  73
  74 #include <pci/pcivar.h>
  75 #include <pci/pcireg.h>
  76
  77 /* grr, #defines for gratuitous incompatibility in queue.h */
  78 #define SIMPLEQ_HEAD            STAILQ_HEAD
  79 #define SIMPLEQ_ENTRY           STAILQ_ENTRY
  80 #define SIMPLEQ_INIT            STAILQ_INIT
  81 #define SIMPLEQ_INSERT_TAIL     STAILQ_INSERT_TAIL
  82 #define SIMPLEQ_EMPTY           STAILQ_EMPTY
  83 #define SIMPLEQ_FIRST           STAILQ_FIRST
  84 #define SIMPLEQ_REMOVE_HEAD     STAILQ_REMOVE_HEAD_UNTIL
  85 #define SIMPLEQ_FOREACH         STAILQ_FOREACH
  86 /* ditto for endian.h */
  87 #define letoh16(x)              le16toh(x)
  88 #define letoh32(x)              le32toh(x)
  89
  90 #ifdef UBSEC_RNDTEST
  91 #include <dev/rndtest/rndtest.h>
  92 #endif
  93 #include <dev/ubsec/ubsecreg.h>
  94 #include <dev/ubsec/ubsecvar.h>
  95
  96 /*
  97  * Prototypes and count for the pci_device structure
  98  */
  99 static  int ubsec_probe(device_t);
 100 static  int ubsec_attach(device_t);
 101 static  int ubsec_detach(device_t);
 102 static  int ubsec_suspend(device_t);
 103 static  int ubsec_resume(device_t);
 104 static  void ubsec_shutdown(device_t);
 105
 106 static device_method_t ubsec_methods[] = {
 107         /* Device interface */
 108         DEVMETHOD(device_probe,         ubsec_probe),
 109         DEVMETHOD(device_attach,        ubsec_attach),
 110         DEVMETHOD(device_detach,        ubsec_detach),
 111         DEVMETHOD(device_suspend,       ubsec_suspend),
 112         DEVMETHOD(device_resume,        ubsec_resume),
 113         DEVMETHOD(device_shutdown,      ubsec_shutdown),
 114
 115         /* bus interface */
 116         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 117         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 118
 119         { 0, 0 }
 120 };
 121 static driver_t ubsec_driver = {
 122         "ubsec",
 123         ubsec_methods,
 124         sizeof (struct ubsec_softc)
 125 };
 126 static devclass_t ubsec_devclass;
 127
 128 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 129 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 130 #ifdef UBSEC_RNDTEST
 131 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 132 #endif
 133
 134 static  void ubsec_intr(void *);
 135 static  int ubsec_newsession(void *, u_int32_t *, struct cryptoini *);
 136 static  int ubsec_freesession(void *, u_int64_t);
 137 static  int ubsec_process(void *, struct cryptop *, int);
 138 static  void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 139 static  void ubsec_feed(struct ubsec_softc *);
 140 static  void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 141 static  void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 142 static  int ubsec_feed2(struct ubsec_softc *);
 143 static  void ubsec_rng(void *);
 144 static  int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 145                              struct ubsec_dma_alloc *, int);
 146 #define ubsec_dma_sync(_dma, _flags) \
 147         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 148 static  void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 149 static  int ubsec_dmamap_aligned(struct ubsec_operand *op);
 150
 151 static  void ubsec_reset_board(struct ubsec_softc *sc);
 152 static  void ubsec_init_board(struct ubsec_softc *sc);
 153 static  void ubsec_init_pciregs(device_t dev);
 154 static  void ubsec_totalreset(struct ubsec_softc *sc);
 155
 156 static  int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 157
 158 static  int ubsec_kprocess(void*, struct cryptkop *, int);
 159 static  int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 160 static  int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 161 static  int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 162 static  void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 163 static  int ubsec_ksigbits(struct crparam *);
 164 static  void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 165 static  void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 166
 167 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters");
 168
 169 #ifdef UBSEC_DEBUG
 170 static  void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 171 static  void ubsec_dump_mcr(struct ubsec_mcr *);
 172 static  void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 173
 174 static  int ubsec_debug = 0;
 175 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 176             0, "control debugging msgs");
 177 #endif
 178
 179 #define READ_REG(sc,r) \
 180         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 181
 182 #define WRITE_REG(sc,reg,val) \
 183         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 184
 185 #define SWAP32(x) (x) = htole32(ntohl((x)))
 186 #define HTOLE32(x) (x) = htole32(x)
 187
 188
 189 struct ubsec_stats ubsecstats;
 190 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 191             ubsec_stats, "driver statistics");
 192
 193 static int
 194 ubsec_probe(device_t dev)
 195 {
 196         if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 197             (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 198              pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 199                 return (0);
 200         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 201             (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 202              pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 203                 return (0);
 204         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 205             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 206              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 207              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 208              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 209              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 210              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 211              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823
 212              ))
 213                 return (0);
 214         return (ENXIO);
 215 }
 216
 217 static const char*
 218 ubsec_partname(struct ubsec_softc *sc)
 219 {
 220         /* XXX sprintf numbers when not decoded */
 221         switch (pci_get_vendor(sc->sc_dev)) {
 222         case PCI_VENDOR_BROADCOM:
 223                 switch (pci_get_device(sc->sc_dev)) {
 224                 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
 225                 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
 226                 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
 227                 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
 228                 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
 229                 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
 230                 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
 231                 }
 232                 return "Broadcom unknown-part";
 233         case PCI_VENDOR_BLUESTEEL:
 234                 switch (pci_get_device(sc->sc_dev)) {
 235                 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 236                 }
 237                 return "Bluesteel unknown-part";
 238         case PCI_VENDOR_SUN:
 239                 switch (pci_get_device(sc->sc_dev)) {
 240                 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 241                 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 242                 }
 243                 return "Sun unknown-part";
 244         }
 245         return "Unknown-vendor unknown-part";
 246 }
 247
 248 static void
 249 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 250 {
 251         u_int32_t *p = (u_int32_t *)buf;
 252         for (count /= sizeof (u_int32_t); count; count--)
 253                 add_true_randomness(*p++);
 254 }
 255
 256 static int
 257 ubsec_attach(device_t dev)
 258 {
 259         struct ubsec_softc *sc = device_get_softc(dev);
 260         struct ubsec_dma *dmap;
 261         u_int32_t cmd, i;
 262         int rid;
 263
 264         KASSERT(sc != NULL, ("ubsec_attach: null software carrier!"));
 265         bzero(sc, sizeof (*sc));
 266         sc->sc_dev = dev;
 267
 268         SIMPLEQ_INIT(&sc->sc_queue);
 269         SIMPLEQ_INIT(&sc->sc_qchip);
 270         SIMPLEQ_INIT(&sc->sc_queue2);
 271         SIMPLEQ_INIT(&sc->sc_qchip2);
 272         SIMPLEQ_INIT(&sc->sc_q2free);
 273
 274         /* XXX handle power management */
 275
 276         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 277
 278         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 279             pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 280                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 281
 282         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 283             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 284              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 285                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 286
 287         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 288             pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 289                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 290                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 291
 292         if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 293              (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 294               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 295               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) ||
 296             (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 297              (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 298               pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 299                 /* NB: the 5821/5822 defines some additional status bits */
 300                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 301                     BS_STAT_MCR2_ALLEMPTY;
 302                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 303                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 304         }
 305
 306         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 307         cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN;
 308         pci_write_config(dev, PCIR_COMMAND, cmd, 4);
 309         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 310
 311         if (!(cmd & PCIM_CMD_MEMEN)) {
 312                 device_printf(dev, "failed to enable memory mapping\n");
 313                 goto bad;
 314         }
 315
 316         if (!(cmd & PCIM_CMD_BUSMASTEREN)) {
 317                 device_printf(dev, "failed to enable bus mastering\n");
 318                 goto bad;
 319         }
 320
 321         /*
 322          * Setup memory-mapping of PCI registers.
 323          */
 324         rid = BS_BAR;
 325         sc->sc_sr = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid,
 326                                        0, ~0, 1, RF_ACTIVE);
 327         if (sc->sc_sr == NULL) {
 328                 device_printf(dev, "cannot map register space\n");
 329                 goto bad;
 330         }
 331         sc->sc_st = rman_get_bustag(sc->sc_sr);
 332         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 333
 334         /*
 335          * Arrange interrupt line.
 336          */
 337         rid = 0;
 338         sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid,
 339                                         0, ~0, 1, RF_SHAREABLE|RF_ACTIVE);
 340         if (sc->sc_irq == NULL) {
 341                 device_printf(dev, "could not map interrupt\n");
 342                 goto bad1;
 343         }
 344         /*
 345          * NB: Network code assumes we are blocked with splimp()
 346          *     so make sure the IRQ is mapped appropriately.
 347          */
 348         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET,
 349                            ubsec_intr, sc, &sc->sc_ih)) {
 350                 device_printf(dev, "could not establish interrupt\n");
 351                 goto bad2;
 352         }
 353
 354         sc->sc_cid = crypto_get_driverid(0);
 355         if (sc->sc_cid < 0) {
 356                 device_printf(dev, "could not get crypto driver id\n");
 357                 goto bad3;
 358         }
 359
 360         /*
 361          * Setup DMA descriptor area.
 362          */
 363         if (bus_dma_tag_create(NULL,                    /* parent */
 364                                1, 0,                    /* alignment, bounds */
 365                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 366                                BUS_SPACE_MAXADDR,       /* highaddr */
 367                                NULL, NULL,              /* filter, filterarg */
 368                                0x3ffff,                 /* maxsize */
 369                                UBS_MAX_SCATTER,         /* nsegments */
 370                                0xffff,                  /* maxsegsize */
 371                                BUS_DMA_ALLOCNOW,        /* flags */
 372                                &sc->sc_dmat)) {
 373                 device_printf(dev, "cannot allocate DMA tag\n");
 374                 goto bad4;
 375         }
 376         SIMPLEQ_INIT(&sc->sc_freequeue);
 377         dmap = sc->sc_dmaa;
 378         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 379                 struct ubsec_q *q;
 380
 381                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 382                     M_DEVBUF, M_NOWAIT);
 383                 if (q == NULL) {
 384                         device_printf(dev, "cannot allocate queue buffers\n");
 385                         break;
 386                 }
 387
 388                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 389                     &dmap->d_alloc, 0)) {
 390                         device_printf(dev, "cannot allocate dma buffers\n");
 391                         free(q, M_DEVBUF);
 392                         break;
 393                 }
 394                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 395
 396                 q->q_dma = dmap;
 397                 sc->sc_queuea[i] = q;
 398
 399                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 400         }
 401
 402         device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 403
 404         crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
 405             ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 406         crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
 407              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 408         crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
 409              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 410         crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
 411              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 412
 413         /*
 414          * Reset Broadcom chip
 415          */
 416         ubsec_reset_board(sc);
 417
 418         /*
 419          * Init Broadcom specific PCI settings
 420          */
 421         ubsec_init_pciregs(dev);
 422
 423         /*
 424          * Init Broadcom chip
 425          */
 426         ubsec_init_board(sc);
 427
 428 #ifndef UBSEC_NO_RNG
 429         if (sc->sc_flags & UBS_FLAGS_RNG) {
 430                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 431 #ifdef UBSEC_RNDTEST
 432                 sc->sc_rndtest = rndtest_attach(dev);
 433                 if (sc->sc_rndtest)
 434                         sc->sc_harvest = rndtest_harvest;
 435                 else
 436                         sc->sc_harvest = default_harvest;
 437 #else
 438                 sc->sc_harvest = default_harvest;
 439 #endif
 440
 441                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 442                     &sc->sc_rng.rng_q.q_mcr, 0))
 443                         goto skip_rng;
 444
 445                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 446                     &sc->sc_rng.rng_q.q_ctx, 0)) {
 447                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 448                         goto skip_rng;
 449                 }
 450
 451                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 452                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 453                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 454                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 455                         goto skip_rng;
 456                 }
 457
 458                 if (hz >= 100)
 459                         sc->sc_rnghz = hz / 100;
 460                 else
 461                         sc->sc_rnghz = 1;
 462                 callout_init(&sc->sc_rngto);
 463                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 464 skip_rng:
 465         ;
 466         }
 467 #endif /* UBSEC_NO_RNG */
 468
 469         if (sc->sc_flags & UBS_FLAGS_KEY) {
 470                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 471
 472                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0,
 473                         ubsec_kprocess, sc);
 474 #if 0
 475                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0,
 476                         ubsec_kprocess, sc);
 477 #endif
 478         }
 479         return (0);
 480 bad4:
 481         crypto_unregister_all(sc->sc_cid);
 482 bad3:
 483         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 484 bad2:
 485         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 486 bad1:
 487         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 488 bad:
 489         return (ENXIO);
 490 }
 491
 492 /*
 493  * Detach a device that successfully probed.
 494  */
 495 static int
 496 ubsec_detach(device_t dev)
 497 {
 498         struct ubsec_softc *sc = device_get_softc(dev);
 499         int s;
 500
 501         KASSERT(sc != NULL, ("ubsec_detach: null software carrier"));
 502
 503         /* XXX wait/abort active ops */
 504
 505         s = splimp();
 506
 507         callout_stop(&sc->sc_rngto);
 508
 509         crypto_unregister_all(sc->sc_cid);
 510
 511 #ifdef UBSEC_RNDTEST
 512         if (sc->sc_rndtest)
 513                 rndtest_detach(sc->sc_rndtest);
 514 #endif
 515
 516         while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 517                 struct ubsec_q *q;
 518
 519                 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 520                 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
 521                 ubsec_dma_free(sc, &q->q_dma->d_alloc);
 522                 free(q, M_DEVBUF);
 523         }
 524 #ifndef UBSEC_NO_RNG
 525         if (sc->sc_flags & UBS_FLAGS_RNG) {
 526                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 527                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 528                 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 529         }
 530 #endif /* UBSEC_NO_RNG */
 531
 532         bus_generic_detach(dev);
 533         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 534         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 535
 536         bus_dma_tag_destroy(sc->sc_dmat);
 537         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 538
 539         splx(s);
 540
 541         return (0);
 542 }
 543
 544 /*
 545  * Stop all chip i/o so that the kernel's probe routines don't
 546  * get confused by errant DMAs when rebooting.
 547  */
 548 static void
 549 ubsec_shutdown(device_t dev)
 550 {
 551 #ifdef notyet
 552         ubsec_stop(device_get_softc(dev));
 553 #endif
 554 }
 555
 556 /*
 557  * Device suspend routine.
 558  */
 559 static int
 560 ubsec_suspend(device_t dev)
 561 {
 562         struct ubsec_softc *sc = device_get_softc(dev);
 563
 564         KASSERT(sc != NULL, ("ubsec_suspend: null software carrier"));
 565 #ifdef notyet
 566         /* XXX stop the device and save PCI settings */
 567 #endif
 568         sc->sc_suspended = 1;
 569
 570         return (0);
 571 }
 572
 573 static int
 574 ubsec_resume(device_t dev)
 575 {
 576         struct ubsec_softc *sc = device_get_softc(dev);
 577
 578         KASSERT(sc != NULL, ("ubsec_resume: null software carrier"));
 579 #ifdef notyet
 580         /* XXX retore PCI settings and start the device */
 581 #endif
 582         sc->sc_suspended = 0;
 583         return (0);
 584 }
 585
 586 /*
 587  * UBSEC Interrupt routine
 588  */
 589 static void
 590 ubsec_intr(void *arg)
 591 {
 592         struct ubsec_softc *sc = arg;
 593         volatile u_int32_t stat;
 594         struct ubsec_q *q;
 595         struct ubsec_dma *dmap;
 596         int npkts = 0, i;
 597
 598         stat = READ_REG(sc, BS_STAT);
 599         stat &= sc->sc_statmask;
 600         if (stat == 0) {
 601                 return;
 602         }
 603
 604         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
 605
 606         /*
 607          * Check to see if we have any packets waiting for us
 608          */
 609         if ((stat & BS_STAT_MCR1_DONE)) {
 610                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 611                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
 612                         dmap = q->q_dma;
 613
 614                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 615                                 break;
 616
 617                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
 618
 619                         npkts = q->q_nstacked_mcrs;
 620                         sc->sc_nqchip -= 1+npkts;
 621                         /*
 622                          * search for further sc_qchip ubsec_q's that share
 623                          * the same MCR, and complete them too, they must be
 624                          * at the top.
 625                          */
 626                         for (i = 0; i < npkts; i++) {
 627                                 if(q->q_stacked_mcr[i]) {
 628                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
 629                                 } else {
 630                                         break;
 631                                 }
 632                         }
 633                         ubsec_callback(sc, q);
 634                 }
 635
 636                 /*
 637                  * Don't send any more packet to chip if there has been
 638                  * a DMAERR.
 639                  */
 640                 if (!(stat & BS_STAT_DMAERR))
 641                         ubsec_feed(sc);
 642         }
 643
 644         /*
 645          * Check to see if we have any key setups/rng's waiting for us
 646          */
 647         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 648             (stat & BS_STAT_MCR2_DONE)) {
 649                 struct ubsec_q2 *q2;
 650                 struct ubsec_mcr *mcr;
 651
 652                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 653                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 654
 655                         ubsec_dma_sync(&q2->q_mcr,
 656                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 657
 658                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 659                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 660                                 ubsec_dma_sync(&q2->q_mcr,
 661                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 662                                 break;
 663                         }
 664                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next);
 665                         ubsec_callback2(sc, q2);
 666                         /*
 667                          * Don't send any more packet to chip if there has been
 668                          * a DMAERR.
 669                          */
 670                         if (!(stat & BS_STAT_DMAERR))
 671                                 ubsec_feed2(sc);
 672                 }
 673         }
 674
 675         /*
 676          * Check to see if we got any DMA Error
 677          */
 678         if (stat & BS_STAT_DMAERR) {
 679 #ifdef UBSEC_DEBUG
 680                 if (ubsec_debug) {
 681                         volatile u_int32_t a = READ_REG(sc, BS_ERR);
 682
 683                         printf("dmaerr %s@%08x\n",
 684                             (a & BS_ERR_READ) ? "read" : "write",
 685                             a & BS_ERR_ADDR);
 686                 }
 687 #endif /* UBSEC_DEBUG */
 688                 ubsecstats.hst_dmaerr++;
 689                 ubsec_totalreset(sc);
 690                 ubsec_feed(sc);
 691         }
 692
 693         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 694                 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 695 #ifdef UBSEC_DEBUG
 696                 if (ubsec_debug)
 697                         device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 698                                 sc->sc_needwakeup);
 699 #endif /* UBSEC_DEBUG */
 700                 sc->sc_needwakeup &= ~wakeup;
 701                 crypto_unblock(sc->sc_cid, wakeup);
 702         }
 703 }
 704
 705 /*
 706  * ubsec_feed() - aggregate and post requests to chip
 707  */
 708 static void
 709 ubsec_feed(struct ubsec_softc *sc)
 710 {
 711         struct ubsec_q *q, *q2;
 712         int npkts, i;
 713         void *v;
 714         u_int32_t stat;
 715
 716         /*
 717          * Decide how many ops to combine in a single MCR.  We cannot
 718          * aggregate more than UBS_MAX_AGGR because this is the number
 719          * of slots defined in the data structure.  Note that
 720          * aggregation only happens if ops are marked batch'able.
 721          * Aggregating ops reduces the number of interrupts to the host
 722          * but also (potentially) increases the latency for processing
 723          * completed ops as we only get an interrupt when all aggregated
 724          * ops have completed.
 725          */
 726         if (sc->sc_nqueue == 0)
 727                 return;
 728         if (sc->sc_nqueue > 1) {
 729                 npkts = 0;
 730                 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 731                         npkts++;
 732                         if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 733                                 break;
 734                 }
 735         } else
 736                 npkts = 1;
 737         /*
 738          * Check device status before going any further.
 739          */
 740         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 741                 if (stat & BS_STAT_DMAERR) {
 742                         ubsec_totalreset(sc);
 743                         ubsecstats.hst_dmaerr++;
 744                 } else
 745                         ubsecstats.hst_mcr1full++;
 746                 return;
 747         }
 748         if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 749                 ubsecstats.hst_maxqueue = sc->sc_nqueue;
 750         if (npkts > UBS_MAX_AGGR)
 751                 npkts = UBS_MAX_AGGR;
 752         if (npkts < 2)                          /* special case 1 op */
 753                 goto feed1;
 754
 755         ubsecstats.hst_totbatch += npkts-1;
 756 #ifdef UBSEC_DEBUG
 757         if (ubsec_debug)
 758                 printf("merging %d records\n", npkts);
 759 #endif /* UBSEC_DEBUG */
 760
 761         q = SIMPLEQ_FIRST(&sc->sc_queue);
 762         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 763         --sc->sc_nqueue;
 764
 765         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 766         if (q->q_dst_map != NULL)
 767                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 768
 769         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
 770
 771         for (i = 0; i < q->q_nstacked_mcrs; i++) {
 772                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 773                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 774                     BUS_DMASYNC_PREWRITE);
 775                 if (q2->q_dst_map != NULL)
 776                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 777                             BUS_DMASYNC_PREREAD);
 778                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next);
 779                 --sc->sc_nqueue;
 780
 781                 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 782                     sizeof(struct ubsec_mcr_add));
 783                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 784                 q->q_stacked_mcr[i] = q2;
 785         }
 786         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 787         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 788         sc->sc_nqchip += npkts;
 789         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 790                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 791         ubsec_dma_sync(&q->q_dma->d_alloc,
 792             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 793         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 794             offsetof(struct ubsec_dmachunk, d_mcr));
 795         return;
 796
 797 feed1:
 798         q = SIMPLEQ_FIRST(&sc->sc_queue);
 799
 800         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 801         if (q->q_dst_map != NULL)
 802                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 803         ubsec_dma_sync(&q->q_dma->d_alloc,
 804             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 805
 806         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 807             offsetof(struct ubsec_dmachunk, d_mcr));
 808 #ifdef UBSEC_DEBUG
 809         if (ubsec_debug)
 810                 printf("feed1: q->chip %p %08x stat %08x\n",
 811                       q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 812                       stat);
 813 #endif /* UBSEC_DEBUG */
 814         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 815         --sc->sc_nqueue;
 816         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 817         sc->sc_nqchip++;
 818         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 819                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 820         return;
 821 }
 822
 823 /*
 824  * Allocate a new 'session' and return an encoded session id.  'sidp'
 825  * contains our registration id, and should contain an encoded session
 826  * id on successful allocation.
 827  */
 828 static int
 829 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
 830 {
 831         struct cryptoini *c, *encini = NULL, *macini = NULL;
 832         struct ubsec_softc *sc = arg;
 833         struct ubsec_session *ses = NULL;
 834         MD5_CTX md5ctx;
 835         SHA1_CTX sha1ctx;
 836         int i, sesn;
 837
 838         KASSERT(sc != NULL, ("ubsec_newsession: null softc"));
 839         if (sidp == NULL || cri == NULL || sc == NULL)
 840                 return (EINVAL);
 841
 842         for (c = cri; c != NULL; c = c->cri_next) {
 843                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 844                     c->cri_alg == CRYPTO_SHA1_HMAC) {
 845                         if (macini)
 846                                 return (EINVAL);
 847                         macini = c;
 848                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 849                     c->cri_alg == CRYPTO_3DES_CBC) {
 850                         if (encini)
 851                                 return (EINVAL);
 852                         encini = c;
 853                 } else
 854                         return (EINVAL);
 855         }
 856         if (encini == NULL && macini == NULL)
 857                 return (EINVAL);
 858
 859         if (sc->sc_sessions == NULL) {
 860                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 861                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 862                 if (ses == NULL)
 863                         return (ENOMEM);
 864                 sesn = 0;
 865                 sc->sc_nsessions = 1;
 866         } else {
 867                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 868                         if (sc->sc_sessions[sesn].ses_used == 0) {
 869                                 ses = &sc->sc_sessions[sesn];
 870                                 break;
 871                         }
 872                 }
 873
 874                 if (ses == NULL) {
 875                         sesn = sc->sc_nsessions;
 876                         ses = (struct ubsec_session *)malloc((sesn + 1) *
 877                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 878                         if (ses == NULL)
 879                                 return (ENOMEM);
 880                         bcopy(sc->sc_sessions, ses, sesn *
 881                             sizeof(struct ubsec_session));
 882                         bzero(sc->sc_sessions, sesn *
 883                             sizeof(struct ubsec_session));
 884                         free(sc->sc_sessions, M_DEVBUF);
 885                         sc->sc_sessions = ses;
 886                         ses = &sc->sc_sessions[sesn];
 887                         sc->sc_nsessions++;
 888                 }
 889         }
 890
 891         bzero(ses, sizeof(struct ubsec_session));
 892         ses->ses_used = 1;
 893         if (encini) {
 894                 /* get an IV, network byte order */
 895                 /* XXX may read fewer than requested */
 896                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 897
 898                 /* Go ahead and compute key in ubsec's byte order */
 899                 if (encini->cri_alg == CRYPTO_DES_CBC) {
 900                         bcopy(encini->cri_key, &ses->ses_deskey[0], 8);
 901                         bcopy(encini->cri_key, &ses->ses_deskey[2], 8);
 902                         bcopy(encini->cri_key, &ses->ses_deskey[4], 8);
 903                 } else
 904                         bcopy(encini->cri_key, ses->ses_deskey, 24);
 905
 906                 SWAP32(ses->ses_deskey[0]);
 907                 SWAP32(ses->ses_deskey[1]);
 908                 SWAP32(ses->ses_deskey[2]);
 909                 SWAP32(ses->ses_deskey[3]);
 910                 SWAP32(ses->ses_deskey[4]);
 911                 SWAP32(ses->ses_deskey[5]);
 912         }
 913
 914         if (macini) {
 915                 for (i = 0; i < macini->cri_klen / 8; i++)
 916                         macini->cri_key[i] ^= HMAC_IPAD_VAL;
 917
 918                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 919                         MD5Init(&md5ctx);
 920                         MD5Update(&md5ctx, macini->cri_key,
 921                             macini->cri_klen / 8);
 922                         MD5Update(&md5ctx, hmac_ipad_buffer,
 923                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 924                         bcopy(md5ctx.state, ses->ses_hminner,
 925                             sizeof(md5ctx.state));
 926                 } else {
 927                         SHA1Init(&sha1ctx);
 928                         SHA1Update(&sha1ctx, macini->cri_key,
 929                             macini->cri_klen / 8);
 930                         SHA1Update(&sha1ctx, hmac_ipad_buffer,
 931                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 932                         bcopy(sha1ctx.h.b32, ses->ses_hminner,
 933                             sizeof(sha1ctx.h.b32));
 934                 }
 935
 936                 for (i = 0; i < macini->cri_klen / 8; i++)
 937                         macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 938
 939                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 940                         MD5Init(&md5ctx);
 941                         MD5Update(&md5ctx, macini->cri_key,
 942                             macini->cri_klen / 8);
 943                         MD5Update(&md5ctx, hmac_opad_buffer,
 944                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 945                         bcopy(md5ctx.state, ses->ses_hmouter,
 946                             sizeof(md5ctx.state));
 947                 } else {
 948                         SHA1Init(&sha1ctx);
 949                         SHA1Update(&sha1ctx, macini->cri_key,
 950                             macini->cri_klen / 8);
 951                         SHA1Update(&sha1ctx, hmac_opad_buffer,
 952                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 953                         bcopy(sha1ctx.h.b32, ses->ses_hmouter,
 954                             sizeof(sha1ctx.h.b32));
 955                 }
 956
 957                 for (i = 0; i < macini->cri_klen / 8; i++)
 958                         macini->cri_key[i] ^= HMAC_OPAD_VAL;
 959         }
 960
 961         *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
 962         return (0);
 963 }
 964
 965 /*
 966  * Deallocate a session.
 967  */
 968 static int
 969 ubsec_freesession(void *arg, u_int64_t tid)
 970 {
 971         struct ubsec_softc *sc = arg;
 972         int session;
 973         u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 974
 975         KASSERT(sc != NULL, ("ubsec_freesession: null softc"));
 976         if (sc == NULL)
 977                 return (EINVAL);
 978
 979         session = UBSEC_SESSION(sid);
 980         if (session >= sc->sc_nsessions)
 981                 return (EINVAL);
 982
 983         bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
 984         return (0);
 985 }
 986
 987 static void
 988 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
 989 {
 990         struct ubsec_operand *op = arg;
 991
 992         KASSERT(nsegs <= UBS_MAX_SCATTER,
 993                 ("Too many DMA segments returned when mapping operand"));
 994 #ifdef UBSEC_DEBUG
 995         if (ubsec_debug)
 996                 printf("ubsec_op_cb: mapsize %u nsegs %d\n",
 997                         (u_int) mapsize, nsegs);
 998 #endif
 999         op->mapsize = mapsize;
1000         op->nsegs = nsegs;
1001         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1002 }
1003
1004 static int
1005 ubsec_process(void *arg, struct cryptop *crp, int hint)
1006 {
1007         struct ubsec_q *q = NULL;
1008         int err = 0, i, j, s, nicealign;
1009         struct ubsec_softc *sc = arg;
1010         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1011         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1012         int sskip, dskip, stheend, dtheend;
1013         int16_t coffset;
1014         struct ubsec_session *ses;
1015         struct ubsec_pktctx ctx;
1016         struct ubsec_dma *dmap = NULL;
1017
1018         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1019                 ubsecstats.hst_invalid++;
1020                 return (EINVAL);
1021         }
1022         if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1023                 ubsecstats.hst_badsession++;
1024                 return (EINVAL);
1025         }
1026
1027         s = splimp();
1028
1029         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1030                 ubsecstats.hst_queuefull++;
1031                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1032                 splx(s);
1033                 return (ERESTART);
1034         }
1035         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1036         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
1037         splx(s);
1038
1039         dmap = q->q_dma; /* Save dma pointer */
1040         bzero(q, sizeof(struct ubsec_q));
1041         bzero(&ctx, sizeof(ctx));
1042
1043         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1044         q->q_dma = dmap;
1045         ses = &sc->sc_sessions[q->q_sesn];
1046
1047         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1048                 q->q_src_m = (struct mbuf *)crp->crp_buf;
1049                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1050         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1051                 q->q_src_io = (struct uio *)crp->crp_buf;
1052                 q->q_dst_io = (struct uio *)crp->crp_buf;
1053         } else {
1054                 ubsecstats.hst_badflags++;
1055                 err = EINVAL;
1056                 goto errout;    /* XXX we don't handle contiguous blocks! */
1057         }
1058
1059         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1060
1061         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1062         dmap->d_dma->d_mcr.mcr_flags = 0;
1063         q->q_crp = crp;
1064
1065         crd1 = crp->crp_desc;
1066         if (crd1 == NULL) {
1067                 ubsecstats.hst_nodesc++;
1068                 err = EINVAL;
1069                 goto errout;
1070         }
1071         crd2 = crd1->crd_next;
1072
1073         if (crd2 == NULL) {
1074                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1075                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1076                         maccrd = crd1;
1077                         enccrd = NULL;
1078                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1079                     crd1->crd_alg == CRYPTO_3DES_CBC) {
1080                         maccrd = NULL;
1081                         enccrd = crd1;
1082                 } else {
1083                         ubsecstats.hst_badalg++;
1084                         err = EINVAL;
1085                         goto errout;
1086                 }
1087         } else {
1088                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1089                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1090                     (crd2->crd_alg == CRYPTO_DES_CBC ||
1091                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
1092                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1093                         maccrd = crd1;
1094                         enccrd = crd2;
1095                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1096                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
1097                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
1098                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1099                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
1100                         enccrd = crd1;
1101                         maccrd = crd2;
1102                 } else {
1103                         /*
1104                          * We cannot order the ubsec as requested
1105                          */
1106                         ubsecstats.hst_badalg++;
1107                         err = EINVAL;
1108                         goto errout;
1109                 }
1110         }
1111
1112         if (enccrd) {
1113                 encoffset = enccrd->crd_skip;
1114                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
1115
1116                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1117                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1118
1119                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1120                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1121                         else {
1122                                 ctx.pc_iv[0] = ses->ses_iv[0];
1123                                 ctx.pc_iv[1] = ses->ses_iv[1];
1124                         }
1125
1126                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1127                                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1128                                         m_copyback(q->q_src_m,
1129                                             enccrd->crd_inject,
1130                                             8, (caddr_t)ctx.pc_iv);
1131                                 else if (crp->crp_flags & CRYPTO_F_IOV)
1132                                         cuio_copyback(q->q_src_io,
1133                                             enccrd->crd_inject,
1134                                             8, (caddr_t)ctx.pc_iv);
1135                         }
1136                 } else {
1137                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
1138
1139                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1140                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1141                         else if (crp->crp_flags & CRYPTO_F_IMBUF)
1142                                 m_copydata(q->q_src_m, enccrd->crd_inject,
1143                                     8, (caddr_t)ctx.pc_iv);
1144                         else if (crp->crp_flags & CRYPTO_F_IOV)
1145                                 cuio_copydata(q->q_src_io,
1146                                     enccrd->crd_inject, 8,
1147                                     (caddr_t)ctx.pc_iv);
1148                 }
1149
1150                 ctx.pc_deskey[0] = ses->ses_deskey[0];
1151                 ctx.pc_deskey[1] = ses->ses_deskey[1];
1152                 ctx.pc_deskey[2] = ses->ses_deskey[2];
1153                 ctx.pc_deskey[3] = ses->ses_deskey[3];
1154                 ctx.pc_deskey[4] = ses->ses_deskey[4];
1155                 ctx.pc_deskey[5] = ses->ses_deskey[5];
1156                 SWAP32(ctx.pc_iv[0]);
1157                 SWAP32(ctx.pc_iv[1]);
1158         }
1159
1160         if (maccrd) {
1161                 macoffset = maccrd->crd_skip;
1162
1163                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1164                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1165                 else
1166                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1167
1168                 for (i = 0; i < 5; i++) {
1169                         ctx.pc_hminner[i] = ses->ses_hminner[i];
1170                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1171
1172                         HTOLE32(ctx.pc_hminner[i]);
1173                         HTOLE32(ctx.pc_hmouter[i]);
1174                 }
1175         }
1176
1177         if (enccrd && maccrd) {
1178                 /*
1179                  * ubsec cannot handle packets where the end of encryption
1180                  * and authentication are not the same, or where the
1181                  * encrypted part begins before the authenticated part.
1182                  */
1183                 if ((encoffset + enccrd->crd_len) !=
1184                     (macoffset + maccrd->crd_len)) {
1185                         ubsecstats.hst_lenmismatch++;
1186                         err = EINVAL;
1187                         goto errout;
1188                 }
1189                 if (enccrd->crd_skip < maccrd->crd_skip) {
1190                         ubsecstats.hst_skipmismatch++;
1191                         err = EINVAL;
1192                         goto errout;
1193                 }
1194                 sskip = maccrd->crd_skip;
1195                 cpskip = dskip = enccrd->crd_skip;
1196                 stheend = maccrd->crd_len;
1197                 dtheend = enccrd->crd_len;
1198                 coffset = enccrd->crd_skip - maccrd->crd_skip;
1199                 cpoffset = cpskip + dtheend;
1200 #ifdef UBSEC_DEBUG
1201                 if (ubsec_debug) {
1202                         printf("mac: skip %d, len %d, inject %d\n",
1203                             maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1204                         printf("enc: skip %d, len %d, inject %d\n",
1205                             enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1206                         printf("src: skip %d, len %d\n", sskip, stheend);
1207                         printf("dst: skip %d, len %d\n", dskip, dtheend);
1208                         printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1209                             coffset, stheend, cpskip, cpoffset);
1210                 }
1211 #endif
1212         } else {
1213                 cpskip = dskip = sskip = macoffset + encoffset;
1214                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1215                 cpoffset = cpskip + dtheend;
1216                 coffset = 0;
1217         }
1218         ctx.pc_offset = htole16(coffset >> 2);
1219
1220         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
1221                 ubsecstats.hst_nomap++;
1222                 err = ENOMEM;
1223                 goto errout;
1224         }
1225         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1226                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1227                     q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1228                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1229                         q->q_src_map = NULL;
1230                         ubsecstats.hst_noload++;
1231                         err = ENOMEM;
1232                         goto errout;
1233                 }
1234         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1235                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1236                     q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1237                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1238                         q->q_src_map = NULL;
1239                         ubsecstats.hst_noload++;
1240                         err = ENOMEM;
1241                         goto errout;
1242                 }
1243         }
1244         nicealign = ubsec_dmamap_aligned(&q->q_src);
1245
1246         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1247
1248 #ifdef UBSEC_DEBUG
1249         if (ubsec_debug)
1250                 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1251 #endif
1252         for (i = j = 0; i < q->q_src_nsegs; i++) {
1253                 struct ubsec_pktbuf *pb;
1254                 bus_size_t packl = q->q_src_segs[i].ds_len;
1255                 bus_addr_t packp = q->q_src_segs[i].ds_addr;
1256
1257                 if (sskip >= packl) {
1258                         sskip -= packl;
1259                         continue;
1260                 }
1261
1262                 packl -= sskip;
1263                 packp += sskip;
1264                 sskip = 0;
1265
1266                 if (packl > 0xfffc) {
1267                         err = EIO;
1268                         goto errout;
1269                 }
1270
1271                 if (j == 0)
1272                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1273                 else
1274                         pb = &dmap->d_dma->d_sbuf[j - 1];
1275
1276                 pb->pb_addr = htole32(packp);
1277
1278                 if (stheend) {
1279                         if (packl > stheend) {
1280                                 pb->pb_len = htole32(stheend);
1281                                 stheend = 0;
1282                         } else {
1283                                 pb->pb_len = htole32(packl);
1284                                 stheend -= packl;
1285                         }
1286                 } else
1287                         pb->pb_len = htole32(packl);
1288
1289                 if ((i + 1) == q->q_src_nsegs)
1290                         pb->pb_next = 0;
1291                 else
1292                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1293                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1294                 j++;
1295         }
1296
1297         if (enccrd == NULL && maccrd != NULL) {
1298                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1299                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1300                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1301                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1302 #ifdef UBSEC_DEBUG
1303                 if (ubsec_debug)
1304                         printf("opkt: %x %x %x\n",
1305                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1306                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1307                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1308 #endif
1309         } else {
1310                 if (crp->crp_flags & CRYPTO_F_IOV) {
1311                         if (!nicealign) {
1312                                 ubsecstats.hst_iovmisaligned++;
1313                                 err = EINVAL;
1314                                 goto errout;
1315                         }
1316                         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
1317                              &q->q_dst_map)) {
1318                                 ubsecstats.hst_nomap++;
1319                                 err = ENOMEM;
1320                                 goto errout;
1321                         }
1322                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1323                             q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
1324                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1325                                 q->q_dst_map = NULL;
1326                                 ubsecstats.hst_noload++;
1327                                 err = ENOMEM;
1328                                 goto errout;
1329                         }
1330                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1331                         if (nicealign) {
1332                                 q->q_dst = q->q_src;
1333                         } else {
1334                                 int totlen, len;
1335                                 struct mbuf *m, *top, **mp;
1336
1337                                 ubsecstats.hst_unaligned++;
1338                                 totlen = q->q_src_mapsize;
1339                                 if (q->q_src_m->m_flags & M_PKTHDR) {
1340                                         len = MHLEN;
1341                                         MGETHDR(m, M_DONTWAIT, MT_DATA);
1342                                         if (m && !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) {
1343                                                 m_free(m);
1344                                                 m = NULL;
1345                                         }
1346                                 } else {
1347                                         len = MLEN;
1348                                         MGET(m, M_DONTWAIT, MT_DATA);
1349                                 }
1350                                 if (m == NULL) {
1351                                         ubsecstats.hst_nombuf++;
1352                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1353                                         goto errout;
1354                                 }
1355                                 if (totlen >= MINCLSIZE) {
1356                                         MCLGET(m, M_DONTWAIT);
1357                                         if ((m->m_flags & M_EXT) == 0) {
1358                                                 m_free(m);
1359                                                 ubsecstats.hst_nomcl++;
1360                                                 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1361                                                 goto errout;
1362                                         }
1363                                         len = MCLBYTES;
1364                                 }
1365                                 m->m_len = len;
1366                                 top = NULL;
1367                                 mp = &top;
1368
1369                                 while (totlen > 0) {
1370                                         if (top) {
1371                                                 MGET(m, M_DONTWAIT, MT_DATA);
1372                                                 if (m == NULL) {
1373                                                         m_freem(top);
1374                                                         ubsecstats.hst_nombuf++;
1375                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1376                                                         goto errout;
1377                                                 }
1378                                                 len = MLEN;
1379                                         }
1380                                         if (top && totlen >= MINCLSIZE) {
1381                                                 MCLGET(m, M_DONTWAIT);
1382                                                 if ((m->m_flags & M_EXT) == 0) {
1383                                                         *mp = m;
1384                                                         m_freem(top);
1385                                                         ubsecstats.hst_nomcl++;
1386                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1387                                                         goto errout;
1388                                                 }
1389                                                 len = MCLBYTES;
1390                                         }
1391                                         m->m_len = len = min(totlen, len);
1392                                         totlen -= len;
1393                                         *mp = m;
1394                                         mp = &m->m_next;
1395                                 }
1396                                 q->q_dst_m = top;
1397                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1398                                     cpskip, cpoffset);
1399                                 if (bus_dmamap_create(sc->sc_dmat,
1400                                     BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
1401                                         ubsecstats.hst_nomap++;
1402                                         err = ENOMEM;
1403                                         goto errout;
1404                                 }
1405                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1406                                     q->q_dst_map, q->q_dst_m,
1407                                     ubsec_op_cb, &q->q_dst,
1408                                     BUS_DMA_NOWAIT) != 0) {
1409                                         bus_dmamap_destroy(sc->sc_dmat,
1410                                         q->q_dst_map);
1411                                         q->q_dst_map = NULL;
1412                                         ubsecstats.hst_noload++;
1413                                         err = ENOMEM;
1414                                         goto errout;
1415                                 }
1416                         }
1417                 } else {
1418                         ubsecstats.hst_badflags++;
1419                         err = EINVAL;
1420                         goto errout;
1421                 }
1422
1423 #ifdef UBSEC_DEBUG
1424                 if (ubsec_debug)
1425                         printf("dst skip: %d\n", dskip);
1426 #endif
1427                 for (i = j = 0; i < q->q_dst_nsegs; i++) {
1428                         struct ubsec_pktbuf *pb;
1429                         bus_size_t packl = q->q_dst_segs[i].ds_len;
1430                         bus_addr_t packp = q->q_dst_segs[i].ds_addr;
1431
1432                         if (dskip >= packl) {
1433                                 dskip -= packl;
1434                                 continue;
1435                         }
1436
1437                         packl -= dskip;
1438                         packp += dskip;
1439                         dskip = 0;
1440
1441                         if (packl > 0xfffc) {
1442                                 err = EIO;
1443                                 goto errout;
1444                         }
1445
1446                         if (j == 0)
1447                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1448                         else
1449                                 pb = &dmap->d_dma->d_dbuf[j - 1];
1450
1451                         pb->pb_addr = htole32(packp);
1452
1453                         if (dtheend) {
1454                                 if (packl > dtheend) {
1455                                         pb->pb_len = htole32(dtheend);
1456                                         dtheend = 0;
1457                                 } else {
1458                                         pb->pb_len = htole32(packl);
1459                                         dtheend -= packl;
1460                                 }
1461                         } else
1462                                 pb->pb_len = htole32(packl);
1463
1464                         if ((i + 1) == q->q_dst_nsegs) {
1465                                 if (maccrd)
1466                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1467                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1468                                 else
1469                                         pb->pb_next = 0;
1470                         } else
1471                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1472                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1473                         j++;
1474                 }
1475         }
1476
1477         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1478             offsetof(struct ubsec_dmachunk, d_ctx));
1479
1480         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1481                 struct ubsec_pktctx_long *ctxl;
1482
1483                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1484                     offsetof(struct ubsec_dmachunk, d_ctx));
1485
1486                 /* transform small context into long context */
1487                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1488                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1489                 ctxl->pc_flags = ctx.pc_flags;
1490                 ctxl->pc_offset = ctx.pc_offset;
1491                 for (i = 0; i < 6; i++)
1492                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1493                 for (i = 0; i < 5; i++)
1494                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1495                 for (i = 0; i < 5; i++)
1496                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1497                 ctxl->pc_iv[0] = ctx.pc_iv[0];
1498                 ctxl->pc_iv[1] = ctx.pc_iv[1];
1499         } else
1500                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1501                     offsetof(struct ubsec_dmachunk, d_ctx),
1502                     sizeof(struct ubsec_pktctx));
1503
1504         s = splimp();
1505         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1506         sc->sc_nqueue++;
1507         ubsecstats.hst_ipackets++;
1508         ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
1509         if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
1510                 ubsec_feed(sc);
1511         splx(s);
1512         return (0);
1513
1514 errout:
1515         if (q != NULL) {
1516                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1517                         m_freem(q->q_dst_m);
1518
1519                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1520                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1521                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1522                 }
1523                 if (q->q_src_map != NULL) {
1524                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1525                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1526                 }
1527
1528                 s = splimp();
1529                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1530                 splx(s);
1531         }
1532         if (err != ERESTART) {
1533                 crp->crp_etype = err;
1534                 crypto_done(crp);
1535         } else {
1536                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1537         }
1538         return (err);
1539 }
1540
1541 static void
1542 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1543 {
1544         struct cryptop *crp = (struct cryptop *)q->q_crp;
1545         struct cryptodesc *crd;
1546         struct ubsec_dma *dmap = q->q_dma;
1547
1548         ubsecstats.hst_opackets++;
1549         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1550
1551         ubsec_dma_sync(&dmap->d_alloc,
1552             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1553         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1554                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1555                     BUS_DMASYNC_POSTREAD);
1556                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1557                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1558         }
1559         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
1560         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1561         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1562
1563         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1564                 m_freem(q->q_src_m);
1565                 crp->crp_buf = (caddr_t)q->q_dst_m;
1566         }
1567         ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len;
1568
1569         /* copy out IV for future use */
1570         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1571                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1572                         if (crd->crd_alg != CRYPTO_DES_CBC &&
1573                             crd->crd_alg != CRYPTO_3DES_CBC)
1574                                 continue;
1575                         if (crp->crp_flags & CRYPTO_F_IMBUF)
1576                                 m_copydata((struct mbuf *)crp->crp_buf,
1577                                     crd->crd_skip + crd->crd_len - 8, 8,
1578                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1579                         else if (crp->crp_flags & CRYPTO_F_IOV) {
1580                                 cuio_copydata((struct uio *)crp->crp_buf,
1581                                     crd->crd_skip + crd->crd_len - 8, 8,
1582                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1583                         }
1584                         break;
1585                 }
1586         }
1587
1588         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1589                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1590                     crd->crd_alg != CRYPTO_SHA1_HMAC)
1591                         continue;
1592                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1593                         m_copyback((struct mbuf *)crp->crp_buf,
1594                             crd->crd_inject, 12,
1595                             (caddr_t)dmap->d_dma->d_macbuf);
1596                 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1597                         bcopy((caddr_t)dmap->d_dma->d_macbuf,
1598                             crp->crp_mac, 12);
1599                 break;
1600         }
1601         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1602         crypto_done(crp);
1603 }
1604
1605 static void
1606 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1607 {
1608         int i, j, dlen, slen;
1609         caddr_t dptr, sptr;
1610
1611         j = 0;
1612         sptr = srcm->m_data;
1613         slen = srcm->m_len;
1614         dptr = dstm->m_data;
1615         dlen = dstm->m_len;
1616
1617         while (1) {
1618                 for (i = 0; i < min(slen, dlen); i++) {
1619                         if (j < hoffset || j >= toffset)
1620                                 *dptr++ = *sptr++;
1621                         slen--;
1622                         dlen--;
1623                         j++;
1624                 }
1625                 if (slen == 0) {
1626                         srcm = srcm->m_next;
1627                         if (srcm == NULL)
1628                                 return;
1629                         sptr = srcm->m_data;
1630                         slen = srcm->m_len;
1631                 }
1632                 if (dlen == 0) {
1633                         dstm = dstm->m_next;
1634                         if (dstm == NULL)
1635                                 return;
1636                         dptr = dstm->m_data;
1637                         dlen = dstm->m_len;
1638                 }
1639         }
1640 }
1641
1642 /*
1643  * feed the key generator, must be called at splimp() or higher.
1644  */
1645 static int
1646 ubsec_feed2(struct ubsec_softc *sc)
1647 {
1648         struct ubsec_q2 *q;
1649
1650         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1651                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1652                         break;
1653                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1654
1655                 ubsec_dma_sync(&q->q_mcr,
1656                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1657                 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
1658
1659                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1660                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next);
1661                 --sc->sc_nqueue2;
1662                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1663         }
1664         return (0);
1665 }
1666
1667 /*
1668  * Callback for handling random numbers
1669  */
1670 static void
1671 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1672 {
1673         struct cryptkop *krp;
1674         struct ubsec_ctx_keyop *ctx;
1675
1676         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1677         ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
1678
1679         switch (q->q_type) {
1680 #ifndef UBSEC_NO_RNG
1681         case UBS_CTXOP_RNGBYPASS: {
1682                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1683
1684                 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
1685                 (*sc->sc_harvest)(sc->sc_rndtest,
1686                         rng->rng_buf.dma_vaddr,
1687                         UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
1688                 rng->rng_used = 0;
1689                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1690                 break;
1691         }
1692 #endif
1693         case UBS_CTXOP_MODEXP: {
1694                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1695                 u_int rlen, clen;
1696
1697                 krp = me->me_krp;
1698                 rlen = (me->me_modbits + 7) / 8;
1699                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1700
1701                 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
1702                 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
1703                 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
1704                 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
1705
1706                 if (clen < rlen)
1707                         krp->krp_status = E2BIG;
1708                 else {
1709                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1710                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
1711                                     (krp->krp_param[krp->krp_iparams].crp_nbits
1712                                         + 7) / 8);
1713                                 bcopy(me->me_C.dma_vaddr,
1714                                     krp->krp_param[krp->krp_iparams].crp_p,
1715                                     (me->me_modbits + 7) / 8);
1716                         } else
1717                                 ubsec_kshift_l(me->me_shiftbits,
1718                                     me->me_C.dma_vaddr, me->me_normbits,
1719                                     krp->krp_param[krp->krp_iparams].crp_p,
1720                                     krp->krp_param[krp->krp_iparams].crp_nbits);
1721                 }
1722
1723                 crypto_kdone(krp);
1724
1725                 /* bzero all potentially sensitive data */
1726                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1727                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1728                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1729                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1730
1731                 /* Can't free here, so put us on the free list. */
1732                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1733                 break;
1734         }
1735         case UBS_CTXOP_RSAPRIV: {
1736                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1737                 u_int len;
1738
1739                 krp = rp->rpr_krp;
1740                 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
1741                 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
1742
1743                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1744                 bcopy(rp->rpr_msgout.dma_vaddr,
1745                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1746
1747                 crypto_kdone(krp);
1748
1749                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1750                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1751                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1752
1753                 /* Can't free here, so put us on the free list. */
1754                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1755                 break;
1756         }
1757         default:
1758                 device_printf(sc->sc_dev, "unknown ctx op: %x\n",
1759                     letoh16(ctx->ctx_op));
1760                 break;
1761         }
1762 }
1763
1764 #ifndef UBSEC_NO_RNG
1765 static void
1766 ubsec_rng(void *vsc)
1767 {
1768         struct ubsec_softc *sc = vsc;
1769         struct ubsec_q2_rng *rng = &sc->sc_rng;
1770         struct ubsec_mcr *mcr;
1771         struct ubsec_ctx_rngbypass *ctx;
1772         int s;
1773
1774         s = splimp();
1775         if (rng->rng_used) {
1776                 splx(s);
1777                 return;
1778         }
1779         sc->sc_nqueue2++;
1780         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1781                 goto out;
1782
1783         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1784         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1785
1786         mcr->mcr_pkts = htole16(1);
1787         mcr->mcr_flags = 0;
1788         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1789         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1790         mcr->mcr_ipktbuf.pb_len = 0;
1791         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1792         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1793         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1794             UBS_PKTBUF_LEN);
1795         mcr->mcr_opktbuf.pb_next = 0;
1796
1797         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1798         ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
1799         rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
1800
1801         ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
1802
1803         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1804         rng->rng_used = 1;
1805         ubsec_feed2(sc);
1806         ubsecstats.hst_rng++;
1807         splx(s);
1808
1809         return;
1810
1811 out:
1812         /*
1813          * Something weird happened, generate our own call back.
1814          */
1815         sc->sc_nqueue2--;
1816         splx(s);
1817         callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1818 }
1819 #endif /* UBSEC_NO_RNG */
1820
1821 static void
1822 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1823 {
1824         bus_addr_t *paddr = (bus_addr_t*) arg;
1825         *paddr = segs->ds_addr;
1826 }
1827
1828 static int
1829 ubsec_dma_malloc(
1830         struct ubsec_softc *sc,
1831         bus_size_t size,
1832         struct ubsec_dma_alloc *dma,
1833         int mapflags
1834 )
1835 {
1836         int r;
1837
1838         /* XXX could specify sc_dmat as parent but that just adds overhead */
1839         r = bus_dma_tag_create(NULL,                    /* parent */
1840                                1, 0,                    /* alignment, bounds */
1841                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1842                                BUS_SPACE_MAXADDR,       /* highaddr */
1843                                NULL, NULL,              /* filter, filterarg */
1844                                size,                    /* maxsize */
1845                                1,                       /* nsegments */
1846                                size,                    /* maxsegsize */
1847                                BUS_DMA_ALLOCNOW,        /* flags */
1848                                &dma->dma_tag);
1849         if (r != 0) {
1850                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1851                         "bus_dma_tag_create failed; error %u\n", r);
1852                 goto fail_0;
1853         }
1854
1855         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1856         if (r != 0) {
1857                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1858                         "bus_dmamap_create failed; error %u\n", r);
1859                 goto fail_1;
1860         }
1861
1862         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1863                              BUS_DMA_NOWAIT, &dma->dma_map);
1864         if (r != 0) {
1865                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1866                         "bus_dmammem_alloc failed; size %u, error %u\n",
1867                         size, r);
1868                 goto fail_2;
1869         }
1870
1871         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1872                             size,
1873                             ubsec_dmamap_cb,
1874                             &dma->dma_paddr,
1875                             mapflags | BUS_DMA_NOWAIT);
1876         if (r != 0) {
1877                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1878                         "bus_dmamap_load failed; error %u\n", r);
1879                 goto fail_3;
1880         }
1881
1882         dma->dma_size = size;
1883         return (0);
1884
1885 fail_3:
1886         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1887 fail_2:
1888         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1889 fail_1:
1890         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1891         bus_dma_tag_destroy(dma->dma_tag);
1892 fail_0:
1893         dma->dma_map = NULL;
1894         dma->dma_tag = NULL;
1895         return (r);
1896 }
1897
1898 static void
1899 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1900 {
1901         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1902         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1903         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1904         bus_dma_tag_destroy(dma->dma_tag);
1905 }
1906
1907 /*
1908  * Resets the board.  Values in the regesters are left as is
1909  * from the reset (i.e. initial values are assigned elsewhere).
1910  */
1911 static void
1912 ubsec_reset_board(struct ubsec_softc *sc)
1913 {
1914     volatile u_int32_t ctrl;
1915
1916     ctrl = READ_REG(sc, BS_CTRL);
1917     ctrl |= BS_CTRL_RESET;
1918     WRITE_REG(sc, BS_CTRL, ctrl);
1919
1920     /*
1921      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1922      */
1923     DELAY(10);
1924 }
1925
1926 /*
1927  * Init Broadcom registers
1928  */
1929 static void
1930 ubsec_init_board(struct ubsec_softc *sc)
1931 {
1932         u_int32_t ctrl;
1933
1934         ctrl = READ_REG(sc, BS_CTRL);
1935         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1936         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1937
1938         if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
1939                 ctrl |= BS_CTRL_MCR2INT;
1940         else
1941                 ctrl &= ~BS_CTRL_MCR2INT;
1942
1943         if (sc->sc_flags & UBS_FLAGS_HWNORM)
1944                 ctrl &= ~BS_CTRL_SWNORM;
1945
1946         WRITE_REG(sc, BS_CTRL, ctrl);
1947 }
1948
1949 /*
1950  * Init Broadcom PCI registers
1951  */
1952 static void
1953 ubsec_init_pciregs(device_t dev)
1954 {
1955 #if 0
1956         u_int32_t misc;
1957
1958         misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
1959         misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
1960             | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
1961         misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
1962             | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
1963         pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
1964 #endif
1965
1966         /*
1967          * This will set the cache line size to 1, this will
1968          * force the BCM58xx chip just to do burst read/writes.
1969          * Cache line read/writes are to slow
1970          */
1971         pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
1972 }
1973
1974 /*
1975  * Clean up after a chip crash.
1976  * It is assumed that the caller in splimp()
1977  */
1978 static void
1979 ubsec_cleanchip(struct ubsec_softc *sc)
1980 {
1981         struct ubsec_q *q;
1982
1983         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1984                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
1985                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
1986                 ubsec_free_q(sc, q);
1987         }
1988         sc->sc_nqchip = 0;
1989 }
1990
1991 /*
1992  * free a ubsec_q
1993  * It is assumed that the caller is within spimp()
1994  */
1995 static int
1996 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
1997 {
1998         struct ubsec_q *q2;
1999         struct cryptop *crp;
2000         int npkts;
2001         int i;
2002
2003         npkts = q->q_nstacked_mcrs;
2004
2005         for (i = 0; i < npkts; i++) {
2006                 if(q->q_stacked_mcr[i]) {
2007                         q2 = q->q_stacked_mcr[i];
2008
2009                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2010                                 m_freem(q2->q_dst_m);
2011
2012                         crp = (struct cryptop *)q2->q_crp;
2013
2014                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2015
2016                         crp->crp_etype = EFAULT;
2017                         crypto_done(crp);
2018                 } else {
2019                         break;
2020                 }
2021         }
2022
2023         /*
2024          * Free header MCR
2025          */
2026         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2027                 m_freem(q->q_dst_m);
2028
2029         crp = (struct cryptop *)q->q_crp;
2030
2031         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2032
2033         crp->crp_etype = EFAULT;
2034         crypto_done(crp);
2035         return(0);
2036 }
2037
2038 /*
2039  * Routine to reset the chip and clean up.
2040  * It is assumed that the caller is in splimp()
2041  */
2042 static void
2043 ubsec_totalreset(struct ubsec_softc *sc)
2044 {
2045         ubsec_reset_board(sc);
2046         ubsec_init_board(sc);
2047         ubsec_cleanchip(sc);
2048 }
2049
2050 static int
2051 ubsec_dmamap_aligned(struct ubsec_operand *op)
2052 {
2053         int i;
2054
2055         for (i = 0; i < op->nsegs; i++) {
2056                 if (op->segs[i].ds_addr & 3)
2057                         return (0);
2058                 if ((i != (op->nsegs - 1)) &&
2059                     (op->segs[i].ds_len & 3))
2060                         return (0);
2061         }
2062         return (1);
2063 }
2064
2065 static void
2066 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2067 {
2068         switch (q->q_type) {
2069         case UBS_CTXOP_MODEXP: {
2070                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2071
2072                 ubsec_dma_free(sc, &me->me_q.q_mcr);
2073                 ubsec_dma_free(sc, &me->me_q.q_ctx);
2074                 ubsec_dma_free(sc, &me->me_M);
2075                 ubsec_dma_free(sc, &me->me_E);
2076                 ubsec_dma_free(sc, &me->me_C);
2077                 ubsec_dma_free(sc, &me->me_epb);
2078                 free(me, M_DEVBUF);
2079                 break;
2080         }
2081         case UBS_CTXOP_RSAPRIV: {
2082                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2083
2084                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2085                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2086                 ubsec_dma_free(sc, &rp->rpr_msgin);
2087                 ubsec_dma_free(sc, &rp->rpr_msgout);
2088                 free(rp, M_DEVBUF);
2089                 break;
2090         }
2091         default:
2092                 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
2093                 break;
2094         }
2095 }
2096
2097 static int
2098 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint)
2099 {
2100         struct ubsec_softc *sc = arg;
2101         int r;
2102
2103         if (krp == NULL || krp->krp_callback == NULL)
2104                 return (EINVAL);
2105
2106         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2107                 struct ubsec_q2 *q;
2108
2109                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
2110                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next);
2111                 ubsec_kfree(sc, q);
2112         }
2113
2114         switch (krp->krp_op) {
2115         case CRK_MOD_EXP:
2116                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2117                         r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2118                 else
2119                         r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2120                 break;
2121         case CRK_MOD_EXP_CRT:
2122                 return (ubsec_kprocess_rsapriv(sc, krp, hint));
2123         default:
2124                 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
2125                     krp->krp_op);
2126                 krp->krp_status = EOPNOTSUPP;
2127                 crypto_kdone(krp);
2128                 return (0);
2129         }
2130         return (0);                     /* silence compiler */
2131 }
2132
2133 /*
2134  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2135  */
2136 static int
2137 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2138 {
2139         struct ubsec_q2_modexp *me;
2140         struct ubsec_mcr *mcr;
2141         struct ubsec_ctx_modexp *ctx;
2142         struct ubsec_pktbuf *epb;
2143         int s, err = 0;
2144         u_int nbits, normbits, mbits, shiftbits, ebits;
2145
2146         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2147         if (me == NULL) {
2148                 err = ENOMEM;
2149                 goto errout;
2150         }
2151         bzero(me, sizeof *me);
2152         me->me_krp = krp;
2153         me->me_q.q_type = UBS_CTXOP_MODEXP;
2154
2155         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2156         if (nbits <= 512)
2157                 normbits = 512;
2158         else if (nbits <= 768)
2159                 normbits = 768;
2160         else if (nbits <= 1024)
2161                 normbits = 1024;
2162         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2163                 normbits = 1536;
2164         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2165                 normbits = 2048;
2166         else {
2167                 err = E2BIG;
2168                 goto errout;
2169         }
2170
2171         shiftbits = normbits - nbits;
2172
2173         me->me_modbits = nbits;
2174         me->me_shiftbits = shiftbits;
2175         me->me_normbits = normbits;
2176
2177         /* Sanity check: result bits must be >= true modulus bits. */
2178         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2179                 err = ERANGE;
2180                 goto errout;
2181         }
2182
2183         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2184             &me->me_q.q_mcr, 0)) {
2185                 err = ENOMEM;
2186                 goto errout;
2187         }
2188         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2189
2190         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2191             &me->me_q.q_ctx, 0)) {
2192                 err = ENOMEM;
2193                 goto errout;
2194         }
2195
2196         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2197         if (mbits > nbits) {
2198                 err = E2BIG;
2199                 goto errout;
2200         }
2201         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2202                 err = ENOMEM;
2203                 goto errout;
2204         }
2205         ubsec_kshift_r(shiftbits,
2206             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2207             me->me_M.dma_vaddr, normbits);
2208
2209         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2210                 err = ENOMEM;
2211                 goto errout;
2212         }
2213         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2214
2215         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2216         if (ebits > nbits) {
2217                 err = E2BIG;
2218                 goto errout;
2219         }
2220         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2221                 err = ENOMEM;
2222                 goto errout;
2223         }
2224         ubsec_kshift_r(shiftbits,
2225             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2226             me->me_E.dma_vaddr, normbits);
2227
2228         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2229             &me->me_epb, 0)) {
2230                 err = ENOMEM;
2231                 goto errout;
2232         }
2233         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2234         epb->pb_addr = htole32(me->me_E.dma_paddr);
2235         epb->pb_next = 0;
2236         epb->pb_len = htole32(normbits / 8);
2237
2238 #ifdef UBSEC_DEBUG
2239         if (ubsec_debug) {
2240                 printf("Epb ");
2241                 ubsec_dump_pb(epb);
2242         }
2243 #endif
2244
2245         mcr->mcr_pkts = htole16(1);
2246         mcr->mcr_flags = 0;
2247         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2248         mcr->mcr_reserved = 0;
2249         mcr->mcr_pktlen = 0;
2250
2251         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2252         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2253         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2254
2255         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2256         mcr->mcr_opktbuf.pb_next = 0;
2257         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2258
2259 #ifdef DIAGNOSTIC
2260         /* Misaligned output buffer will hang the chip. */
2261         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2262                 panic("%s: modexp invalid addr 0x%x\n",
2263                     device_get_nameunit(sc->sc_dev),
2264                     letoh32(mcr->mcr_opktbuf.pb_addr));
2265         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2266                 panic("%s: modexp invalid len 0x%x\n",
2267                     device_get_nameunit(sc->sc_dev),
2268                     letoh32(mcr->mcr_opktbuf.pb_len));
2269 #endif
2270
2271         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2272         bzero(ctx, sizeof(*ctx));
2273         ubsec_kshift_r(shiftbits,
2274             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2275             ctx->me_N, normbits);
2276         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2277         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2278         ctx->me_E_len = htole16(nbits);
2279         ctx->me_N_len = htole16(nbits);
2280
2281 #ifdef UBSEC_DEBUG
2282         if (ubsec_debug) {
2283                 ubsec_dump_mcr(mcr);
2284                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2285         }
2286 #endif
2287
2288         /*
2289          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2290          * everything else.
2291          */
2292         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2293         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2294         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2295         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2296
2297         /* Enqueue and we're done... */
2298         s = splimp();
2299         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2300         ubsec_feed2(sc);
2301         ubsecstats.hst_modexp++;
2302         splx(s);
2303
2304         return (0);
2305
2306 errout:
2307         if (me != NULL) {
2308                 if (me->me_q.q_mcr.dma_map != NULL)
2309                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2310                 if (me->me_q.q_ctx.dma_map != NULL) {
2311                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2312                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2313                 }
2314                 if (me->me_M.dma_map != NULL) {
2315                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2316                         ubsec_dma_free(sc, &me->me_M);
2317                 }
2318                 if (me->me_E.dma_map != NULL) {
2319                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2320                         ubsec_dma_free(sc, &me->me_E);
2321                 }
2322                 if (me->me_C.dma_map != NULL) {
2323                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2324                         ubsec_dma_free(sc, &me->me_C);
2325                 }
2326                 if (me->me_epb.dma_map != NULL)
2327                         ubsec_dma_free(sc, &me->me_epb);
2328                 free(me, M_DEVBUF);
2329         }
2330         krp->krp_status = err;
2331         crypto_kdone(krp);
2332         return (0);
2333 }
2334
2335 /*
2336  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2337  */
2338 static int
2339 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2340 {
2341         struct ubsec_q2_modexp *me;
2342         struct ubsec_mcr *mcr;
2343         struct ubsec_ctx_modexp *ctx;
2344         struct ubsec_pktbuf *epb;
2345         int s, err = 0;
2346         u_int nbits, normbits, mbits, shiftbits, ebits;
2347
2348         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2349         if (me == NULL) {
2350                 err = ENOMEM;
2351                 goto errout;
2352         }
2353         bzero(me, sizeof *me);
2354         me->me_krp = krp;
2355         me->me_q.q_type = UBS_CTXOP_MODEXP;
2356
2357         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2358         if (nbits <= 512)
2359                 normbits = 512;
2360         else if (nbits <= 768)
2361                 normbits = 768;
2362         else if (nbits <= 1024)
2363                 normbits = 1024;
2364         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2365                 normbits = 1536;
2366         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2367                 normbits = 2048;
2368         else {
2369                 err = E2BIG;
2370                 goto errout;
2371         }
2372
2373         shiftbits = normbits - nbits;
2374
2375         /* XXX ??? */
2376         me->me_modbits = nbits;
2377         me->me_shiftbits = shiftbits;
2378         me->me_normbits = normbits;
2379
2380         /* Sanity check: result bits must be >= true modulus bits. */
2381         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2382                 err = ERANGE;
2383                 goto errout;
2384         }
2385
2386         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2387             &me->me_q.q_mcr, 0)) {
2388                 err = ENOMEM;
2389                 goto errout;
2390         }
2391         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2392
2393         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2394             &me->me_q.q_ctx, 0)) {
2395                 err = ENOMEM;
2396                 goto errout;
2397         }
2398
2399         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2400         if (mbits > nbits) {
2401                 err = E2BIG;
2402                 goto errout;
2403         }
2404         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2405                 err = ENOMEM;
2406                 goto errout;
2407         }
2408         bzero(me->me_M.dma_vaddr, normbits / 8);
2409         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2410             me->me_M.dma_vaddr, (mbits + 7) / 8);
2411
2412         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2413                 err = ENOMEM;
2414                 goto errout;
2415         }
2416         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2417
2418         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2419         if (ebits > nbits) {
2420                 err = E2BIG;
2421                 goto errout;
2422         }
2423         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2424                 err = ENOMEM;
2425                 goto errout;
2426         }
2427         bzero(me->me_E.dma_vaddr, normbits / 8);
2428         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2429             me->me_E.dma_vaddr, (ebits + 7) / 8);
2430
2431         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2432             &me->me_epb, 0)) {
2433                 err = ENOMEM;
2434                 goto errout;
2435         }
2436         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2437         epb->pb_addr = htole32(me->me_E.dma_paddr);
2438         epb->pb_next = 0;
2439         epb->pb_len = htole32((ebits + 7) / 8);
2440
2441 #ifdef UBSEC_DEBUG
2442         if (ubsec_debug) {
2443                 printf("Epb ");
2444                 ubsec_dump_pb(epb);
2445         }
2446 #endif
2447
2448         mcr->mcr_pkts = htole16(1);
2449         mcr->mcr_flags = 0;
2450         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2451         mcr->mcr_reserved = 0;
2452         mcr->mcr_pktlen = 0;
2453
2454         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2455         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2456         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2457
2458         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2459         mcr->mcr_opktbuf.pb_next = 0;
2460         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2461
2462 #ifdef DIAGNOSTIC
2463         /* Misaligned output buffer will hang the chip. */
2464         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2465                 panic("%s: modexp invalid addr 0x%x\n",
2466                     device_get_nameunit(sc->sc_dev),
2467                     letoh32(mcr->mcr_opktbuf.pb_addr));
2468         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2469                 panic("%s: modexp invalid len 0x%x\n",
2470                     device_get_nameunit(sc->sc_dev),
2471                     letoh32(mcr->mcr_opktbuf.pb_len));
2472 #endif
2473
2474         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2475         bzero(ctx, sizeof(*ctx));
2476         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2477             (nbits + 7) / 8);
2478         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2479         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2480         ctx->me_E_len = htole16(ebits);
2481         ctx->me_N_len = htole16(nbits);
2482
2483 #ifdef UBSEC_DEBUG
2484         if (ubsec_debug) {
2485                 ubsec_dump_mcr(mcr);
2486                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2487         }
2488 #endif
2489
2490         /*
2491          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2492          * everything else.
2493          */
2494         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2495         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2496         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2497         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2498
2499         /* Enqueue and we're done... */
2500         s = splimp();
2501         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2502         ubsec_feed2(sc);
2503         splx(s);
2504
2505         return (0);
2506
2507 errout:
2508         if (me != NULL) {
2509                 if (me->me_q.q_mcr.dma_map != NULL)
2510                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2511                 if (me->me_q.q_ctx.dma_map != NULL) {
2512                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2513                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2514                 }
2515                 if (me->me_M.dma_map != NULL) {
2516                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2517                         ubsec_dma_free(sc, &me->me_M);
2518                 }
2519                 if (me->me_E.dma_map != NULL) {
2520                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2521                         ubsec_dma_free(sc, &me->me_E);
2522                 }
2523                 if (me->me_C.dma_map != NULL) {
2524                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2525                         ubsec_dma_free(sc, &me->me_C);
2526                 }
2527                 if (me->me_epb.dma_map != NULL)
2528                         ubsec_dma_free(sc, &me->me_epb);
2529                 free(me, M_DEVBUF);
2530         }
2531         krp->krp_status = err;
2532         crypto_kdone(krp);
2533         return (0);
2534 }
2535
2536 static int
2537 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2538 {
2539         struct ubsec_q2_rsapriv *rp = NULL;
2540         struct ubsec_mcr *mcr;
2541         struct ubsec_ctx_rsapriv *ctx;
2542         int s, err = 0;
2543         u_int padlen, msglen;
2544
2545         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2546         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2547         if (msglen > padlen)
2548                 padlen = msglen;
2549
2550         if (padlen <= 256)
2551                 padlen = 256;
2552         else if (padlen <= 384)
2553                 padlen = 384;
2554         else if (padlen <= 512)
2555                 padlen = 512;
2556         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2557                 padlen = 768;
2558         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2559                 padlen = 1024;
2560         else {
2561                 err = E2BIG;
2562                 goto errout;
2563         }
2564
2565         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2566                 err = E2BIG;
2567                 goto errout;
2568         }
2569
2570         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2571                 err = E2BIG;
2572                 goto errout;
2573         }
2574
2575         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2576                 err = E2BIG;
2577                 goto errout;
2578         }
2579
2580         rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
2581         if (rp == NULL)
2582                 return (ENOMEM);
2583         bzero(rp, sizeof *rp);
2584         rp->rpr_krp = krp;
2585         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2586
2587         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2588             &rp->rpr_q.q_mcr, 0)) {
2589                 err = ENOMEM;
2590                 goto errout;
2591         }
2592         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2593
2594         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2595             &rp->rpr_q.q_ctx, 0)) {
2596                 err = ENOMEM;
2597                 goto errout;
2598         }
2599         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2600         bzero(ctx, sizeof *ctx);
2601
2602         /* Copy in p */
2603         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2604             &ctx->rpr_buf[0 * (padlen / 8)],
2605             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2606
2607         /* Copy in q */
2608         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2609             &ctx->rpr_buf[1 * (padlen / 8)],
2610             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2611
2612         /* Copy in dp */
2613         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2614             &ctx->rpr_buf[2 * (padlen / 8)],
2615             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2616
2617         /* Copy in dq */
2618         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2619             &ctx->rpr_buf[3 * (padlen / 8)],
2620             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2621
2622         /* Copy in pinv */
2623         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2624             &ctx->rpr_buf[4 * (padlen / 8)],
2625             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2626
2627         msglen = padlen * 2;
2628
2629         /* Copy in input message (aligned buffer/length). */
2630         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2631                 /* Is this likely? */
2632                 err = E2BIG;
2633                 goto errout;
2634         }
2635         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2636                 err = ENOMEM;
2637                 goto errout;
2638         }
2639         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2640         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2641             rp->rpr_msgin.dma_vaddr,
2642             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2643
2644         /* Prepare space for output message (aligned buffer/length). */
2645         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2646                 /* Is this likely? */
2647                 err = E2BIG;
2648                 goto errout;
2649         }
2650         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2651                 err = ENOMEM;
2652                 goto errout;
2653         }
2654         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2655
2656         mcr->mcr_pkts = htole16(1);
2657         mcr->mcr_flags = 0;
2658         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2659         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2660         mcr->mcr_ipktbuf.pb_next = 0;
2661         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2662         mcr->mcr_reserved = 0;
2663         mcr->mcr_pktlen = htole16(msglen);
2664         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2665         mcr->mcr_opktbuf.pb_next = 0;
2666         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2667
2668 #ifdef DIAGNOSTIC
2669         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2670                 panic("%s: rsapriv: invalid msgin %x(0x%x)",
2671                     device_get_nameunit(sc->sc_dev),
2672                     rp->rpr_msgin.dma_paddr, rp->rpr_msgin.dma_size);
2673         }
2674         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2675                 panic("%s: rsapriv: invalid msgout %x(0x%x)",
2676                     device_get_nameunit(sc->sc_dev),
2677                     rp->rpr_msgout.dma_paddr, rp->rpr_msgout.dma_size);
2678         }
2679 #endif
2680
2681         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2682         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2683         ctx->rpr_q_len = htole16(padlen);
2684         ctx->rpr_p_len = htole16(padlen);
2685
2686         /*
2687          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2688          * everything else.
2689          */
2690         ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
2691         ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
2692
2693         /* Enqueue and we're done... */
2694         s = splimp();
2695         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2696         ubsec_feed2(sc);
2697         ubsecstats.hst_modexpcrt++;
2698         splx(s);
2699         return (0);
2700
2701 errout:
2702         if (rp != NULL) {
2703                 if (rp->rpr_q.q_mcr.dma_map != NULL)
2704                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2705                 if (rp->rpr_msgin.dma_map != NULL) {
2706                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2707                         ubsec_dma_free(sc, &rp->rpr_msgin);
2708                 }
2709                 if (rp->rpr_msgout.dma_map != NULL) {
2710                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2711                         ubsec_dma_free(sc, &rp->rpr_msgout);
2712                 }
2713                 free(rp, M_DEVBUF);
2714         }
2715         krp->krp_status = err;
2716         crypto_kdone(krp);
2717         return (0);
2718 }
2719
2720 #ifdef UBSEC_DEBUG
2721 static void
2722 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
2723 {
2724         printf("addr 0x%x (0x%x) next 0x%x\n",
2725             pb->pb_addr, pb->pb_len, pb->pb_next);
2726 }
2727
2728 static void
2729 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2730 {
2731         printf("CTX (0x%x):\n", c->ctx_len);
2732         switch (letoh16(c->ctx_op)) {
2733         case UBS_CTXOP_RNGBYPASS:
2734         case UBS_CTXOP_RNGSHA1:
2735                 break;
2736         case UBS_CTXOP_MODEXP:
2737         {
2738                 struct ubsec_ctx_modexp *cx = (void *)c;
2739                 int i, len;
2740
2741                 printf(" Elen %u, Nlen %u\n",
2742                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2743                 len = (cx->me_N_len + 7)/8;
2744                 for (i = 0; i < len; i++)
2745                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2746                 printf("\n");
2747                 break;
2748         }
2749         default:
2750                 printf("unknown context: %x\n", c->ctx_op);
2751         }
2752         printf("END CTX\n");
2753 }
2754
2755 static void
2756 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2757 {
2758         volatile struct ubsec_mcr_add *ma;
2759         int i;
2760
2761         printf("MCR:\n");
2762         printf(" pkts: %u, flags 0x%x\n",
2763             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2764         ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2765         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2766                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2767                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2768                     letoh16(ma->mcr_reserved));
2769                 printf(" %d: ipkt ", i);
2770                 ubsec_dump_pb(&ma->mcr_ipktbuf);
2771                 printf(" %d: opkt ", i);
2772                 ubsec_dump_pb(&ma->mcr_opktbuf);
2773                 ma++;
2774         }
2775         printf("END MCR\n");
2776 }
2777 #endif /* UBSEC_DEBUG */
2778
2779 /*
2780  * Return the number of significant bits of a big number.
2781  */
2782 static int
2783 ubsec_ksigbits(struct crparam *cr)
2784 {
2785         u_int plen = (cr->crp_nbits + 7) / 8;
2786         int i, sig = plen * 8;
2787         u_int8_t c, *p = cr->crp_p;
2788
2789         for (i = plen - 1; i >= 0; i--) {
2790                 c = p[i];
2791                 if (c != 0) {
2792                         while ((c & 0x80) == 0) {
2793                                 sig--;
2794                                 c <<= 1;
2795                         }
2796                         break;
2797                 }
2798                 sig -= 8;
2799         }
2800         return (sig);
2801 }
2802
2803 static void
2804 ubsec_kshift_r(
2805         u_int shiftbits,
2806         u_int8_t *src, u_int srcbits,
2807         u_int8_t *dst, u_int dstbits)
2808 {
2809         u_int slen, dlen;
2810         int i, si, di, n;
2811
2812         slen = (srcbits + 7) / 8;
2813         dlen = (dstbits + 7) / 8;
2814
2815         for (i = 0; i < slen; i++)
2816                 dst[i] = src[i];
2817         for (i = 0; i < dlen - slen; i++)
2818                 dst[slen + i] = 0;
2819
2820         n = shiftbits / 8;
2821         if (n != 0) {
2822                 si = dlen - n - 1;
2823                 di = dlen - 1;
2824                 while (si >= 0)
2825                         dst[di--] = dst[si--];
2826                 while (di >= 0)
2827                         dst[di--] = 0;
2828         }
2829
2830         n = shiftbits % 8;
2831         if (n != 0) {
2832                 for (i = dlen - 1; i > 0; i--)
2833                         dst[i] = (dst[i] << n) |
2834                             (dst[i - 1] >> (8 - n));
2835                 dst[0] = dst[0] << n;
2836         }
2837 }
2838
2839 static void
2840 ubsec_kshift_l(
2841         u_int shiftbits,
2842         u_int8_t *src, u_int srcbits,
2843         u_int8_t *dst, u_int dstbits)
2844 {
2845         int slen, dlen, i, n;
2846
2847         slen = (srcbits + 7) / 8;
2848         dlen = (dstbits + 7) / 8;
2849
2850         n = shiftbits / 8;
2851         for (i = 0; i < slen; i++)
2852                 dst[i] = src[i + n];
2853         for (i = 0; i < dlen - slen; i++)
2854                 dst[slen + i] = 0;
2855
2856         n = shiftbits % 8;
2857         if (n != 0) {
2858                 for (i = 0; i < (dlen - 1); i++)
2859                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2860                 dst[dlen - 1] = dst[dlen - 1] >> n;
2861         }
2862 }