sys/dev/netif/mxge/if_mxge.c

   1 /******************************************************************************
   2
   3 Copyright (c) 2006-2013, Myricom Inc.
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12  2. Neither the name of the Myricom Inc, nor the names of its
  13     contributors may be used to endorse or promote products derived from
  14     this software without specific prior written permission.
  15
  16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  26 POSSIBILITY OF SUCH DAMAGE.
  27
  28 $FreeBSD: head/sys/dev/mxge/if_mxge.c 254263 2013-08-12 23:30:01Z scottl $
  29
  30 ***************************************************************************/
  31
  32 #include "opt_inet.h"
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/linker.h>
  37 #include <sys/firmware.h>
  38 #include <sys/endian.h>
  39 #include <sys/in_cksum.h>
  40 #include <sys/sockio.h>
  41 #include <sys/mbuf.h>
  42 #include <sys/malloc.h>
  43 #include <sys/kernel.h>
  44 #include <sys/module.h>
  45 #include <sys/serialize.h>
  46 #include <sys/socket.h>
  47 #include <sys/sysctl.h>
  48
  49 #include <net/if.h>
  50 #include <net/if_arp.h>
  51 #include <net/ifq_var.h>
  52 #include <net/ethernet.h>
  53 #include <net/if_dl.h>
  54 #include <net/if_media.h>
  55
  56 #include <net/bpf.h>
  57
  58 #include <net/if_types.h>
  59 #include <net/vlan/if_vlan_var.h>
  60 #include <net/zlib.h>
  61
  62 #include <netinet/in_systm.h>
  63 #include <netinet/in.h>
  64 #include <netinet/ip.h>
  65 #include <netinet/tcp.h>
  66
  67 #include <sys/bus.h>
  68 #include <sys/rman.h>
  69
  70 #include <bus/pci/pcireg.h>
  71 #include <bus/pci/pcivar.h>
  72 #include <bus/pci/pci_private.h> /* XXX for pci_cfg_restore */
  73
  74 #include <vm/vm.h>              /* for pmap_mapdev() */
  75 #include <vm/pmap.h>
  76
  77 #if defined(__i386__) || defined(__x86_64__)
  78 #include <machine/specialreg.h>
  79 #endif
  80
  81 #include <dev/netif/mxge/mxge_mcp.h>
  82 #include <dev/netif/mxge/mcp_gen_header.h>
  83 #include <dev/netif/mxge/if_mxge_var.h>
  84
  85 /* tunable params */
  86 static int mxge_nvidia_ecrc_enable = 1;
  87 static int mxge_force_firmware = 0;
  88 static int mxge_intr_coal_delay = 30;
  89 static int mxge_deassert_wait = 1;
  90 static int mxge_flow_control = 1;
  91 static int mxge_verbose = 0;
  92 static int mxge_ticks;
  93 static int mxge_max_slices = 1;
  94 static int mxge_rss_hash_type = MXGEFW_RSS_HASH_TYPE_TCP_IPV4;
  95 static int mxge_always_promisc = 0;
  96 static int mxge_throttle = 0;
  97 static char *mxge_fw_unaligned = "mxge_ethp_z8e";
  98 static char *mxge_fw_aligned = "mxge_eth_z8e";
  99 static char *mxge_fw_rss_aligned = "mxge_rss_eth_z8e";
 100 static char *mxge_fw_rss_unaligned = "mxge_rss_ethp_z8e";
 101
 102 static int mxge_probe(device_t dev);
 103 static int mxge_attach(device_t dev);
 104 static int mxge_detach(device_t dev);
 105 static int mxge_shutdown(device_t dev);
 106 static void mxge_intr(void *arg);
 107
 108 static device_method_t mxge_methods[] = {
 109         /* Device interface */
 110         DEVMETHOD(device_probe, mxge_probe),
 111         DEVMETHOD(device_attach, mxge_attach),
 112         DEVMETHOD(device_detach, mxge_detach),
 113         DEVMETHOD(device_shutdown, mxge_shutdown),
 114         DEVMETHOD_END
 115 };
 116
 117 static driver_t mxge_driver = {
 118         "mxge",
 119         mxge_methods,
 120         sizeof(mxge_softc_t),
 121 };
 122
 123 static devclass_t mxge_devclass;
 124
 125 /* Declare ourselves to be a child of the PCI bus.*/
 126 DRIVER_MODULE(mxge, pci, mxge_driver, mxge_devclass, NULL, NULL);
 127 MODULE_DEPEND(mxge, firmware, 1, 1, 1);
 128 MODULE_DEPEND(mxge, zlib, 1, 1, 1);
 129
 130 static int mxge_load_firmware(mxge_softc_t *sc, int adopt);
 131 static int mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data);
 132 static int mxge_close(mxge_softc_t *sc, int down);
 133 static int mxge_open(mxge_softc_t *sc);
 134 static void mxge_tick(void *arg);
 135
 136 static int
 137 mxge_probe(device_t dev)
 138 {
 139         int rev;
 140
 141         if ((pci_get_vendor(dev) == MXGE_PCI_VENDOR_MYRICOM) &&
 142             ((pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E) ||
 143              (pci_get_device(dev) == MXGE_PCI_DEVICE_Z8E_9))) {
 144                 rev = pci_get_revid(dev);
 145                 switch (rev) {
 146                 case MXGE_PCI_REV_Z8E:
 147                         device_set_desc(dev, "Myri10G-PCIE-8A");
 148                         break;
 149                 case MXGE_PCI_REV_Z8ES:
 150                         device_set_desc(dev, "Myri10G-PCIE-8B");
 151                         break;
 152                 default:
 153                         device_set_desc(dev, "Myri10G-PCIE-8??");
 154                         device_printf(dev, "Unrecognized rev %d NIC\n",
 155                                       rev);
 156                         break;
 157                 }
 158                 return 0;
 159         }
 160         return ENXIO;
 161 }
 162
 163 static void
 164 mxge_enable_wc(mxge_softc_t *sc)
 165 {
 166 #if defined(__i386__) || defined(__x86_64__)
 167         vm_offset_t len;
 168
 169         sc->wc = 1;
 170         len = rman_get_size(sc->mem_res);
 171         pmap_change_attr((vm_offset_t) sc->sram, len / PAGE_SIZE,
 172             PAT_WRITE_COMBINING);
 173 #endif
 174 }
 175
 176 /* callback to get our DMA address */
 177 static void
 178 mxge_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 179 {
 180         if (error == 0) {
 181                 *(bus_addr_t *) arg = segs->ds_addr;
 182         }
 183 }
 184
 185 static int
 186 mxge_dma_alloc(mxge_softc_t *sc, mxge_dma_t *dma, size_t bytes,
 187     bus_size_t alignment)
 188 {
 189         int err;
 190         device_t dev = sc->dev;
 191         bus_size_t boundary, maxsegsize;
 192
 193         if (bytes > 4096 && alignment == 4096) {
 194                 boundary = 0;
 195                 maxsegsize = bytes;
 196         } else {
 197                 boundary = 4096;
 198                 maxsegsize = 4096;
 199         }
 200
 201         /* allocate DMAable memory tags */
 202         err = bus_dma_tag_create(sc->parent_dmat,       /* parent */
 203                                  alignment,             /* alignment */
 204                                  boundary,              /* boundary */
 205                                  BUS_SPACE_MAXADDR,     /* low */
 206                                  BUS_SPACE_MAXADDR,     /* high */
 207                                  NULL, NULL,            /* filter */
 208                                  bytes,                 /* maxsize */
 209                                  1,                     /* num segs */
 210                                  maxsegsize,            /* maxsegsize */
 211                                  BUS_DMA_COHERENT,      /* flags */
 212                                  &dma->dmat);           /* tag */
 213         if (err != 0) {
 214                 device_printf(dev, "couldn't alloc tag (err = %d)\n", err);
 215                 return err;
 216         }
 217
 218         /* allocate DMAable memory & map */
 219         err = bus_dmamem_alloc(dma->dmat, &dma->addr,
 220                                (BUS_DMA_WAITOK | BUS_DMA_COHERENT
 221                                 | BUS_DMA_ZERO),  &dma->map);
 222         if (err != 0) {
 223                 device_printf(dev, "couldn't alloc mem (err = %d)\n", err);
 224                 goto abort_with_dmat;
 225         }
 226
 227         /* load the memory */
 228         err = bus_dmamap_load(dma->dmat, dma->map, dma->addr, bytes,
 229                               mxge_dmamap_callback,
 230                               (void *)&dma->bus_addr, 0);
 231         if (err != 0) {
 232                 device_printf(dev, "couldn't load map (err = %d)\n", err);
 233                 goto abort_with_mem;
 234         }
 235         return 0;
 236
 237 abort_with_mem:
 238         bus_dmamem_free(dma->dmat, dma->addr, dma->map);
 239 abort_with_dmat:
 240         (void)bus_dma_tag_destroy(dma->dmat);
 241         return err;
 242 }
 243
 244 static void
 245 mxge_dma_free(mxge_dma_t *dma)
 246 {
 247         bus_dmamap_unload(dma->dmat, dma->map);
 248         bus_dmamem_free(dma->dmat, dma->addr, dma->map);
 249         (void)bus_dma_tag_destroy(dma->dmat);
 250 }
 251
 252 /*
 253  * The eeprom strings on the lanaiX have the format
 254  * SN=x\0
 255  * MAC=x:x:x:x:x:x\0
 256  * PC=text\0
 257  */
 258 static int
 259 mxge_parse_strings(mxge_softc_t *sc)
 260 {
 261         char *ptr;
 262         int i, found_mac, found_sn2;
 263         char *endptr;
 264
 265         ptr = sc->eeprom_strings;
 266         found_mac = 0;
 267         found_sn2 = 0;
 268         while (*ptr != '\0') {
 269                 if (strncmp(ptr, "MAC=", 4) == 0) {
 270                         ptr += 4;
 271                         for (i = 0;;) {
 272                                 sc->mac_addr[i] = strtoul(ptr, &endptr, 16);
 273                                 if (endptr - ptr != 2)
 274                                         goto abort;
 275                                 ptr = endptr;
 276                                 if (++i == 6)
 277                                         break;
 278                                 if (*ptr++ != ':')
 279                                         goto abort;
 280                         }
 281                         found_mac = 1;
 282                 } else if (strncmp(ptr, "PC=", 3) == 0) {
 283                         ptr += 3;
 284                         strlcpy(sc->product_code_string, ptr,
 285                             sizeof(sc->product_code_string));
 286                 } else if (!found_sn2 && (strncmp(ptr, "SN=", 3) == 0)) {
 287                         ptr += 3;
 288                         strlcpy(sc->serial_number_string, ptr,
 289                             sizeof(sc->serial_number_string));
 290                 } else if (strncmp(ptr, "SN2=", 4) == 0) {
 291                         /* SN2 takes precedence over SN */
 292                         ptr += 4;
 293                         found_sn2 = 1;
 294                         strlcpy(sc->serial_number_string, ptr,
 295                             sizeof(sc->serial_number_string));
 296                 }
 297                 while (*ptr++ != '\0') {}
 298         }
 299
 300         if (found_mac)
 301                 return 0;
 302
 303 abort:
 304         device_printf(sc->dev, "failed to parse eeprom_strings\n");
 305         return ENXIO;
 306 }
 307
 308 #if defined(__i386__) || defined(__x86_64__)
 309
 310 static void
 311 mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
 312 {
 313         uint32_t val;
 314         unsigned long base, off;
 315         char *va, *cfgptr;
 316         device_t pdev, mcp55;
 317         uint16_t vendor_id, device_id, word;
 318         uintptr_t bus, slot, func, ivend, idev;
 319         uint32_t *ptr32;
 320
 321
 322         if (!mxge_nvidia_ecrc_enable)
 323                 return;
 324
 325         pdev = device_get_parent(device_get_parent(sc->dev));
 326         if (pdev == NULL) {
 327                 device_printf(sc->dev, "could not find parent?\n");
 328                 return;
 329         }
 330         vendor_id = pci_read_config(pdev, PCIR_VENDOR, 2);
 331         device_id = pci_read_config(pdev, PCIR_DEVICE, 2);
 332
 333         if (vendor_id != 0x10de)
 334                 return;
 335
 336         base = 0;
 337
 338         if (device_id == 0x005d) {
 339                 /* ck804, base address is magic */
 340                 base = 0xe0000000UL;
 341         } else if (device_id >= 0x0374 && device_id <= 0x378) {
 342                 /* mcp55, base address stored in chipset */
 343                 mcp55 = pci_find_bsf(0, 0, 0);
 344                 if (mcp55 &&
 345                     0x10de == pci_read_config(mcp55, PCIR_VENDOR, 2) &&
 346                     0x0369 == pci_read_config(mcp55, PCIR_DEVICE, 2)) {
 347                         word = pci_read_config(mcp55, 0x90, 2);
 348                         base = ((unsigned long)word & 0x7ffeU) << 25;
 349                 }
 350         }
 351         if (!base)
 352                 return;
 353
 354         /* XXXX
 355            Test below is commented because it is believed that doing
 356            config read/write beyond 0xff will access the config space
 357            for the next larger function.  Uncomment this and remove
 358            the hacky pmap_mapdev() way of accessing config space when
 359            FreeBSD grows support for extended pcie config space access
 360         */
 361 #if 0
 362         /* See if we can, by some miracle, access the extended
 363            config space */
 364         val = pci_read_config(pdev, 0x178, 4);
 365         if (val != 0xffffffff) {
 366                 val |= 0x40;
 367                 pci_write_config(pdev, 0x178, val, 4);
 368                 return;
 369         }
 370 #endif
 371         /* Rather than using normal pci config space writes, we must
 372          * map the Nvidia config space ourselves.  This is because on
 373          * opteron/nvidia class machine the 0xe000000 mapping is
 374          * handled by the nvidia chipset, that means the internal PCI
 375          * device (the on-chip northbridge), or the amd-8131 bridge
 376          * and things behind them are not visible by this method.
 377          */
 378
 379         BUS_READ_IVAR(device_get_parent(pdev), pdev,
 380                       PCI_IVAR_BUS, &bus);
 381         BUS_READ_IVAR(device_get_parent(pdev), pdev,
 382                       PCI_IVAR_SLOT, &slot);
 383         BUS_READ_IVAR(device_get_parent(pdev), pdev,
 384                       PCI_IVAR_FUNCTION, &func);
 385         BUS_READ_IVAR(device_get_parent(pdev), pdev,
 386                       PCI_IVAR_VENDOR, &ivend);
 387         BUS_READ_IVAR(device_get_parent(pdev), pdev,
 388                       PCI_IVAR_DEVICE, &idev);
 389
 390         off =  base
 391                 + 0x00100000UL * (unsigned long)bus
 392                 + 0x00001000UL * (unsigned long)(func
 393                                                  + 8 * slot);
 394
 395         /* map it into the kernel */
 396         va = pmap_mapdev(trunc_page((vm_paddr_t)off), PAGE_SIZE);
 397
 398
 399         if (va == NULL) {
 400                 device_printf(sc->dev, "pmap_kenter_temporary didn't\n");
 401                 return;
 402         }
 403         /* get a pointer to the config space mapped into the kernel */
 404         cfgptr = va + (off & PAGE_MASK);
 405
 406         /* make sure that we can really access it */
 407         vendor_id = *(uint16_t *)(cfgptr + PCIR_VENDOR);
 408         device_id = *(uint16_t *)(cfgptr + PCIR_DEVICE);
 409         if (! (vendor_id == ivend && device_id == idev)) {
 410                 device_printf(sc->dev, "mapping failed: 0x%x:0x%x\n",
 411                               vendor_id, device_id);
 412                 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
 413                 return;
 414         }
 415
 416         ptr32 = (uint32_t*)(cfgptr + 0x178);
 417         val = *ptr32;
 418
 419         if (val == 0xffffffff) {
 420                 device_printf(sc->dev, "extended mapping failed\n");
 421                 pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
 422                 return;
 423         }
 424         *ptr32 = val | 0x40;
 425         pmap_unmapdev((vm_offset_t)va, PAGE_SIZE);
 426         if (mxge_verbose)
 427                 device_printf(sc->dev,
 428                               "Enabled ECRC on upstream Nvidia bridge "
 429                               "at %d:%d:%d\n",
 430                               (int)bus, (int)slot, (int)func);
 431         return;
 432 }
 433
 434 #else   /* __i386__ || __x86_64__ */
 435
 436 static void
 437 mxge_enable_nvidia_ecrc(mxge_softc_t *sc)
 438 {
 439         device_printf(sc->dev,
 440                       "Nforce 4 chipset on non-x86/x86_64!?!?!\n");
 441         return;
 442 }
 443
 444 #endif
 445
 446 static int
 447 mxge_dma_test(mxge_softc_t *sc, int test_type)
 448 {
 449         mxge_cmd_t cmd;
 450         bus_addr_t dmatest_bus = sc->dmabench_dma.bus_addr;
 451         int status;
 452         uint32_t len;
 453         char *test = " ";
 454
 455         /* Run a small DMA test.
 456          * The magic multipliers to the length tell the firmware
 457          * to do DMA read, write, or read+write tests.  The
 458          * results are returned in cmd.data0.  The upper 16
 459          * bits of the return is the number of transfers completed.
 460          * The lower 16 bits is the time in 0.5us ticks that the
 461          * transfers took to complete.
 462          */
 463
 464         len = sc->tx_boundary;
 465
 466         cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
 467         cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
 468         cmd.data2 = len * 0x10000;
 469         status = mxge_send_cmd(sc, test_type, &cmd);
 470         if (status != 0) {
 471                 test = "read";
 472                 goto abort;
 473         }
 474         sc->read_dma = ((cmd.data0>>16) * len * 2) /
 475                 (cmd.data0 & 0xffff);
 476         cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
 477         cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
 478         cmd.data2 = len * 0x1;
 479         status = mxge_send_cmd(sc, test_type, &cmd);
 480         if (status != 0) {
 481                 test = "write";
 482                 goto abort;
 483         }
 484         sc->write_dma = ((cmd.data0>>16) * len * 2) /
 485                 (cmd.data0 & 0xffff);
 486
 487         cmd.data0 = MXGE_LOWPART_TO_U32(dmatest_bus);
 488         cmd.data1 = MXGE_HIGHPART_TO_U32(dmatest_bus);
 489         cmd.data2 = len * 0x10001;
 490         status = mxge_send_cmd(sc, test_type, &cmd);
 491         if (status != 0) {
 492                 test = "read/write";
 493                 goto abort;
 494         }
 495         sc->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) /
 496                 (cmd.data0 & 0xffff);
 497
 498 abort:
 499         if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST)
 500                 device_printf(sc->dev, "DMA %s benchmark failed: %d\n",
 501                               test, status);
 502
 503         return status;
 504 }
 505
 506 /*
 507  * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput
 508  * when the PCI-E Completion packets are aligned on an 8-byte
 509  * boundary.  Some PCI-E chip sets always align Completion packets; on
 510  * the ones that do not, the alignment can be enforced by enabling
 511  * ECRC generation (if supported).
 512  *
 513  * When PCI-E Completion packets are not aligned, it is actually more
 514  * efficient to limit Read-DMA transactions to 2KB, rather than 4KB.
 515  *
 516  * If the driver can neither enable ECRC nor verify that it has
 517  * already been enabled, then it must use a firmware image which works
 518  * around unaligned completion packets (ethp_z8e.dat), and it should
 519  * also ensure that it never gives the device a Read-DMA which is
 520  * larger than 2KB by setting the tx_boundary to 2KB.  If ECRC is
 521  * enabled, then the driver should use the aligned (eth_z8e.dat)
 522  * firmware image, and set tx_boundary to 4KB.
 523  */
 524 static int
 525 mxge_firmware_probe(mxge_softc_t *sc)
 526 {
 527         device_t dev = sc->dev;
 528         int reg, status;
 529         uint16_t pectl;
 530
 531         sc->tx_boundary = 4096;
 532         /*
 533          * Verify the max read request size was set to 4KB
 534          * before trying the test with 4KB.
 535          */
 536         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
 537                 pectl = pci_read_config(dev, reg + 0x8, 2);
 538                 if ((pectl & (5 << 12)) != (5 << 12)) {
 539                         device_printf(dev, "Max Read Req. size != 4k (0x%x\n",
 540                                       pectl);
 541                         sc->tx_boundary = 2048;
 542                 }
 543         }
 544
 545         /*
 546          * load the optimized firmware (which assumes aligned PCIe
 547          * completions) in order to see if it works on this host.
 548          */
 549         sc->fw_name = mxge_fw_aligned;
 550         status = mxge_load_firmware(sc, 1);
 551         if (status != 0) {
 552                 return status;
 553         }
 554
 555         /*
 556          * Enable ECRC if possible
 557          */
 558         mxge_enable_nvidia_ecrc(sc);
 559
 560         /*
 561          * Run a DMA test which watches for unaligned completions and
 562          * aborts on the first one seen.  Not required on Z8ES or newer.
 563          */
 564         if (pci_get_revid(sc->dev) >= MXGE_PCI_REV_Z8ES)
 565                 return 0;
 566
 567         status = mxge_dma_test(sc, MXGEFW_CMD_UNALIGNED_TEST);
 568         if (status == 0)
 569                 return 0; /* keep the aligned firmware */
 570
 571         if (status != E2BIG)
 572                 device_printf(dev, "DMA test failed: %d\n", status);
 573         if (status == ENOSYS)
 574                 device_printf(dev, "Falling back to ethp! "
 575                               "Please install up to date fw\n");
 576         return status;
 577 }
 578
 579 static int
 580 mxge_select_firmware(mxge_softc_t *sc)
 581 {
 582         int aligned = 0;
 583         int force_firmware = mxge_force_firmware;
 584
 585         if (sc->throttle)
 586                 force_firmware = sc->throttle;
 587
 588         if (force_firmware != 0) {
 589                 if (force_firmware == 1)
 590                         aligned = 1;
 591                 else
 592                         aligned = 0;
 593                 if (mxge_verbose)
 594                         device_printf(sc->dev,
 595                                       "Assuming %s completions (forced)\n",
 596                                       aligned ? "aligned" : "unaligned");
 597                 goto abort;
 598         }
 599
 600         /* if the PCIe link width is 4 or less, we can use the aligned
 601            firmware and skip any checks */
 602         if (sc->link_width != 0 && sc->link_width <= 4) {
 603                 device_printf(sc->dev,
 604                               "PCIe x%d Link, expect reduced performance\n",
 605                               sc->link_width);
 606                 aligned = 1;
 607                 goto abort;
 608         }
 609
 610         if (0 == mxge_firmware_probe(sc))
 611                 return 0;
 612
 613 abort:
 614         if (aligned) {
 615                 sc->fw_name = mxge_fw_aligned;
 616                 sc->tx_boundary = 4096;
 617         } else {
 618                 sc->fw_name = mxge_fw_unaligned;
 619                 sc->tx_boundary = 2048;
 620         }
 621         return (mxge_load_firmware(sc, 0));
 622 }
 623
 624 static int
 625 mxge_validate_firmware(mxge_softc_t *sc, const mcp_gen_header_t *hdr)
 626 {
 627
 628
 629         if (be32toh(hdr->mcp_type) != MCP_TYPE_ETH) {
 630                 device_printf(sc->dev, "Bad firmware type: 0x%x\n",
 631                               be32toh(hdr->mcp_type));
 632                 return EIO;
 633         }
 634
 635         /* save firmware version for sysctl */
 636         strncpy(sc->fw_version, hdr->version, sizeof(sc->fw_version));
 637         if (mxge_verbose)
 638                 device_printf(sc->dev, "firmware id: %s\n", hdr->version);
 639
 640         ksscanf(sc->fw_version, "%d.%d.%d", &sc->fw_ver_major,
 641                &sc->fw_ver_minor, &sc->fw_ver_tiny);
 642
 643         if (!(sc->fw_ver_major == MXGEFW_VERSION_MAJOR
 644               && sc->fw_ver_minor == MXGEFW_VERSION_MINOR)) {
 645                 device_printf(sc->dev, "Found firmware version %s\n",
 646                               sc->fw_version);
 647                 device_printf(sc->dev, "Driver needs %d.%d\n",
 648                               MXGEFW_VERSION_MAJOR, MXGEFW_VERSION_MINOR);
 649                 return EINVAL;
 650         }
 651         return 0;
 652
 653 }
 654
 655 static void *
 656 z_alloc(void *nil, u_int items, u_int size)
 657 {
 658         void *ptr;
 659
 660         ptr = kmalloc(items * size, M_TEMP, M_NOWAIT);
 661         return ptr;
 662 }
 663
 664 static void
 665 z_free(void *nil, void *ptr)
 666 {
 667         kfree(ptr, M_TEMP);
 668 }
 669
 670 static int
 671 mxge_load_firmware_helper(mxge_softc_t *sc, uint32_t *limit)
 672 {
 673         z_stream zs;
 674         char *inflate_buffer;
 675         const struct firmware *fw;
 676         const mcp_gen_header_t *hdr;
 677         unsigned hdr_offset;
 678         int status;
 679         unsigned int i;
 680         char dummy;
 681         size_t fw_len;
 682
 683         fw = firmware_get(sc->fw_name);
 684         if (fw == NULL) {
 685                 device_printf(sc->dev, "Could not find firmware image %s\n",
 686                               sc->fw_name);
 687                 return ENOENT;
 688         }
 689
 690
 691
 692         /* setup zlib and decompress f/w */
 693         bzero(&zs, sizeof (zs));
 694         zs.zalloc = z_alloc;
 695         zs.zfree = z_free;
 696         status = inflateInit(&zs);
 697         if (status != Z_OK) {
 698                 status = EIO;
 699                 goto abort_with_fw;
 700         }
 701
 702         /* the uncompressed size is stored as the firmware version,
 703            which would otherwise go unused */
 704         fw_len = (size_t) fw->version;
 705         inflate_buffer = kmalloc(fw_len, M_TEMP, M_NOWAIT);
 706         if (inflate_buffer == NULL)
 707                 goto abort_with_zs;
 708         zs.avail_in = fw->datasize;
 709         zs.next_in = __DECONST(char *, fw->data);
 710         zs.avail_out = fw_len;
 711         zs.next_out = inflate_buffer;
 712         status = inflate(&zs, Z_FINISH);
 713         if (status != Z_STREAM_END) {
 714                 device_printf(sc->dev, "zlib %d\n", status);
 715                 status = EIO;
 716                 goto abort_with_buffer;
 717         }
 718
 719         /* check id */
 720         hdr_offset = htobe32(*(const uint32_t *)
 721                              (inflate_buffer + MCP_HEADER_PTR_OFFSET));
 722         if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw_len) {
 723                 device_printf(sc->dev, "Bad firmware file");
 724                 status = EIO;
 725                 goto abort_with_buffer;
 726         }
 727         hdr = (const void*)(inflate_buffer + hdr_offset);
 728
 729         status = mxge_validate_firmware(sc, hdr);
 730         if (status != 0)
 731                 goto abort_with_buffer;
 732
 733         /* Copy the inflated firmware to NIC SRAM. */
 734         for (i = 0; i < fw_len; i += 256) {
 735                 mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i,
 736                               inflate_buffer + i,
 737                               min(256U, (unsigned)(fw_len - i)));
 738                 wmb();
 739                 dummy = *sc->sram;
 740                 wmb();
 741         }
 742
 743         *limit = fw_len;
 744         status = 0;
 745 abort_with_buffer:
 746         kfree(inflate_buffer, M_TEMP);
 747 abort_with_zs:
 748         inflateEnd(&zs);
 749 abort_with_fw:
 750         firmware_put(fw, FIRMWARE_UNLOAD);
 751         return status;
 752 }
 753
 754 /*
 755  * Enable or disable periodic RDMAs from the host to make certain
 756  * chipsets resend dropped PCIe messages
 757  */
 758 static void
 759 mxge_dummy_rdma(mxge_softc_t *sc, int enable)
 760 {
 761         char buf_bytes[72];
 762         volatile uint32_t *confirm;
 763         volatile char *submit;
 764         uint32_t *buf, dma_low, dma_high;
 765         int i;
 766
 767         buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
 768
 769         /* clear confirmation addr */
 770         confirm = (volatile uint32_t *)sc->cmd;
 771         *confirm = 0;
 772         wmb();
 773
 774         /* send an rdma command to the PCIe engine, and wait for the
 775            response in the confirmation address.  The firmware should
 776            write a -1 there to indicate it is alive and well
 777         */
 778
 779         dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
 780         dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
 781         buf[0] = htobe32(dma_high);             /* confirm addr MSW */
 782         buf[1] = htobe32(dma_low);              /* confirm addr LSW */
 783         buf[2] = htobe32(0xffffffff);           /* confirm data */
 784         dma_low = MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr);
 785         dma_high = MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr);
 786         buf[3] = htobe32(dma_high);             /* dummy addr MSW */
 787         buf[4] = htobe32(dma_low);              /* dummy addr LSW */
 788         buf[5] = htobe32(enable);                       /* enable? */
 789
 790
 791         submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA);
 792
 793         mxge_pio_copy(submit, buf, 64);
 794         wmb();
 795         DELAY(1000);
 796         wmb();
 797         i = 0;
 798         while (*confirm != 0xffffffff && i < 20) {
 799                 DELAY(1000);
 800                 i++;
 801         }
 802         if (*confirm != 0xffffffff) {
 803                 device_printf(sc->dev, "dummy rdma %s failed (%p = 0x%x)",
 804                               (enable ? "enable" : "disable"), confirm,
 805                               *confirm);
 806         }
 807         return;
 808 }
 809
 810 static int
 811 mxge_send_cmd(mxge_softc_t *sc, uint32_t cmd, mxge_cmd_t *data)
 812 {
 813         mcp_cmd_t *buf;
 814         char buf_bytes[sizeof(*buf) + 8];
 815         volatile mcp_cmd_response_t *response = sc->cmd;
 816         volatile char *cmd_addr = sc->sram + MXGEFW_ETH_CMD;
 817         uint32_t dma_low, dma_high;
 818         int err, sleep_total = 0;
 819
 820         /*
 821          * We may be called during attach, before if_serializer is available.
 822          * This is not a fast path, just check for NULL
 823          * XXX SERIALIZE
 824          */
 825         if (sc->ifp->if_serializer)
 826                 ASSERT_SERIALIZED(sc->ifp->if_serializer);
 827
 828         /* ensure buf is aligned to 8 bytes */
 829         buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
 830
 831         buf->data0 = htobe32(data->data0);
 832         buf->data1 = htobe32(data->data1);
 833         buf->data2 = htobe32(data->data2);
 834         buf->cmd = htobe32(cmd);
 835         dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
 836         dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
 837
 838         buf->response_addr.low = htobe32(dma_low);
 839         buf->response_addr.high = htobe32(dma_high);
 840
 841         response->result = 0xffffffff;
 842         wmb();
 843         mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf));
 844
 845         /* wait up to 20ms */
 846         err = EAGAIN;
 847         for (sleep_total = 0; sleep_total <  20; sleep_total++) {
 848                 bus_dmamap_sync(sc->cmd_dma.dmat,
 849                                 sc->cmd_dma.map, BUS_DMASYNC_POSTREAD);
 850                 wmb();
 851                 switch (be32toh(response->result)) {
 852                 case 0:
 853                         data->data0 = be32toh(response->data);
 854                         err = 0;
 855                         break;
 856                 case 0xffffffff:
 857                         DELAY(1000);
 858                         break;
 859                 case MXGEFW_CMD_UNKNOWN:
 860                         err = ENOSYS;
 861                         break;
 862                 case MXGEFW_CMD_ERROR_UNALIGNED:
 863                         err = E2BIG;
 864                         break;
 865                 case MXGEFW_CMD_ERROR_BUSY:
 866                         err = EBUSY;
 867                         break;
 868                 case MXGEFW_CMD_ERROR_I2C_ABSENT:
 869                         err = ENXIO;
 870                         break;
 871                 default:
 872                         device_printf(sc->dev,
 873                                       "mxge: command %d "
 874                                       "failed, result = %d\n",
 875                                       cmd, be32toh(response->result));
 876                         err = ENXIO;
 877                         break;
 878                 }
 879                 if (err != EAGAIN)
 880                         break;
 881         }
 882         if (err == EAGAIN)
 883                 device_printf(sc->dev, "mxge: command %d timed out"
 884                               "result = %d\n",
 885                               cmd, be32toh(response->result));
 886         return err;
 887 }
 888
 889 static int
 890 mxge_adopt_running_firmware(mxge_softc_t *sc)
 891 {
 892         struct mcp_gen_header *hdr;
 893         const size_t bytes = sizeof (struct mcp_gen_header);
 894         size_t hdr_offset;
 895         int status;
 896
 897         /* find running firmware header */
 898         hdr_offset = htobe32(*(volatile uint32_t *)
 899                              (sc->sram + MCP_HEADER_PTR_OFFSET));
 900
 901         if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > sc->sram_size) {
 902                 device_printf(sc->dev,
 903                               "Running firmware has bad header offset (%d)\n",
 904                               (int)hdr_offset);
 905                 return EIO;
 906         }
 907
 908         /* copy header of running firmware from SRAM to host memory to
 909          * validate firmware */
 910         hdr = kmalloc(bytes, M_DEVBUF, M_NOWAIT);
 911         if (hdr == NULL) {
 912                 device_printf(sc->dev, "could not kmalloc firmware hdr\n");
 913                 return ENOMEM;
 914         }
 915         bus_space_read_region_1(rman_get_bustag(sc->mem_res),
 916                                 rman_get_bushandle(sc->mem_res),
 917                                 hdr_offset, (char *)hdr, bytes);
 918         status = mxge_validate_firmware(sc, hdr);
 919         kfree(hdr, M_DEVBUF);
 920
 921         /*
 922          * check to see if adopted firmware has bug where adopting
 923          * it will cause broadcasts to be filtered unless the NIC
 924          * is kept in ALLMULTI mode
 925          */
 926         if (sc->fw_ver_major == 1 && sc->fw_ver_minor == 4 &&
 927             sc->fw_ver_tiny >= 4 && sc->fw_ver_tiny <= 11) {
 928                 sc->adopted_rx_filter_bug = 1;
 929                 device_printf(sc->dev, "Adopting fw %d.%d.%d: "
 930                               "working around rx filter bug\n",
 931                               sc->fw_ver_major, sc->fw_ver_minor,
 932                               sc->fw_ver_tiny);
 933         }
 934
 935         return status;
 936 }
 937
 938 static int
 939 mxge_load_firmware(mxge_softc_t *sc, int adopt)
 940 {
 941         volatile uint32_t *confirm;
 942         volatile char *submit;
 943         char buf_bytes[72];
 944         uint32_t *buf, size, dma_low, dma_high;
 945         int status, i;
 946
 947         buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
 948
 949         size = sc->sram_size;
 950         status = mxge_load_firmware_helper(sc, &size);
 951         if (status) {
 952                 if (!adopt)
 953                         return status;
 954                 /* Try to use the currently running firmware, if
 955                    it is new enough */
 956                 status = mxge_adopt_running_firmware(sc);
 957                 if (status) {
 958                         device_printf(sc->dev,
 959                                       "failed to adopt running firmware\n");
 960                         return status;
 961                 }
 962                 device_printf(sc->dev,
 963                               "Successfully adopted running firmware\n");
 964                 if (sc->tx_boundary == 4096) {
 965                         device_printf(sc->dev,
 966                                 "Using firmware currently running on NIC"
 967                                  ".  For optimal\n");
 968                         device_printf(sc->dev,
 969                                  "performance consider loading optimized "
 970                                  "firmware\n");
 971                 }
 972                 sc->fw_name = mxge_fw_unaligned;
 973                 sc->tx_boundary = 2048;
 974                 return 0;
 975         }
 976         /* clear confirmation addr */
 977         confirm = (volatile uint32_t *)sc->cmd;
 978         *confirm = 0;
 979         wmb();
 980         /* send a reload command to the bootstrap MCP, and wait for the
 981            response in the confirmation address.  The firmware should
 982            write a -1 there to indicate it is alive and well
 983         */
 984
 985         dma_low = MXGE_LOWPART_TO_U32(sc->cmd_dma.bus_addr);
 986         dma_high = MXGE_HIGHPART_TO_U32(sc->cmd_dma.bus_addr);
 987
 988         buf[0] = htobe32(dma_high);     /* confirm addr MSW */
 989         buf[1] = htobe32(dma_low);      /* confirm addr LSW */
 990         buf[2] = htobe32(0xffffffff);   /* confirm data */
 991
 992         /* FIX: All newest firmware should un-protect the bottom of
 993            the sram before handoff. However, the very first interfaces
 994            do not. Therefore the handoff copy must skip the first 8 bytes
 995         */
 996                                         /* where the code starts*/
 997         buf[3] = htobe32(MXGE_FW_OFFSET + 8);
 998         buf[4] = htobe32(size - 8);     /* length of code */
 999         buf[5] = htobe32(8);            /* where to copy to */
1000         buf[6] = htobe32(0);            /* where to jump to */
1001
1002         submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF);
1003         mxge_pio_copy(submit, buf, 64);
1004         wmb();
1005         DELAY(1000);
1006         wmb();
1007         i = 0;
1008         while (*confirm != 0xffffffff && i < 20) {
1009                 DELAY(1000*10);
1010                 i++;
1011                 bus_dmamap_sync(sc->cmd_dma.dmat,
1012                                 sc->cmd_dma.map, BUS_DMASYNC_POSTREAD);
1013         }
1014         if (*confirm != 0xffffffff) {
1015                 device_printf(sc->dev,"handoff failed (%p = 0x%x)",
1016                         confirm, *confirm);
1017
1018                 return ENXIO;
1019         }
1020         return 0;
1021 }
1022
1023 static int
1024 mxge_update_mac_address(mxge_softc_t *sc)
1025 {
1026         mxge_cmd_t cmd;
1027         uint8_t *addr = sc->mac_addr;
1028         int status;
1029
1030
1031         cmd.data0 = ((addr[0] << 24) | (addr[1] << 16)
1032                      | (addr[2] << 8) | addr[3]);
1033
1034         cmd.data1 = ((addr[4] << 8) | (addr[5]));
1035
1036         status = mxge_send_cmd(sc, MXGEFW_SET_MAC_ADDRESS, &cmd);
1037         return status;
1038 }
1039
1040 static int
1041 mxge_change_pause(mxge_softc_t *sc, int pause)
1042 {
1043         mxge_cmd_t cmd;
1044         int status;
1045
1046         if (pause)
1047                 status = mxge_send_cmd(sc, MXGEFW_ENABLE_FLOW_CONTROL,
1048                                        &cmd);
1049         else
1050                 status = mxge_send_cmd(sc, MXGEFW_DISABLE_FLOW_CONTROL,
1051                                        &cmd);
1052
1053         if (status) {
1054                 device_printf(sc->dev, "Failed to set flow control mode\n");
1055                 return ENXIO;
1056         }
1057         sc->pause = pause;
1058         return 0;
1059 }
1060
1061 static void
1062 mxge_change_promisc(mxge_softc_t *sc, int promisc)
1063 {
1064         mxge_cmd_t cmd;
1065         int status;
1066
1067         /* XXX SERIALIZE */
1068         if (sc->ifp->if_serializer)
1069                 ASSERT_SERIALIZED(sc->ifp->if_serializer);
1070
1071         if (mxge_always_promisc)
1072                 promisc = 1;
1073
1074         if (promisc)
1075                 status = mxge_send_cmd(sc, MXGEFW_ENABLE_PROMISC,
1076                                        &cmd);
1077         else
1078                 status = mxge_send_cmd(sc, MXGEFW_DISABLE_PROMISC,
1079                                        &cmd);
1080
1081         if (status) {
1082                 device_printf(sc->dev, "Failed to set promisc mode\n");
1083         }
1084 }
1085
1086 static void
1087 mxge_set_multicast_list(mxge_softc_t *sc)
1088 {
1089         mxge_cmd_t cmd;
1090         struct ifmultiaddr *ifma;
1091         struct ifnet *ifp = sc->ifp;
1092         int err;
1093
1094         /* XXX SERIALIZE */
1095         if (ifp->if_serializer)
1096                 ASSERT_SERIALIZED(ifp->if_serializer);
1097
1098         /* This firmware is known to not support multicast */
1099         if (!sc->fw_multicast_support)
1100                 return;
1101
1102         /* Disable multicast filtering while we play with the lists*/
1103         err = mxge_send_cmd(sc, MXGEFW_ENABLE_ALLMULTI, &cmd);
1104         if (err != 0) {
1105                 device_printf(sc->dev, "Failed MXGEFW_ENABLE_ALLMULTI,"
1106                        " error status: %d\n", err);
1107                 return;
1108         }
1109
1110         if (sc->adopted_rx_filter_bug)
1111                 return;
1112
1113         if (ifp->if_flags & IFF_ALLMULTI)
1114                 /* request to disable multicast filtering, so quit here */
1115                 return;
1116
1117         /* Flush all the filters */
1118
1119         err = mxge_send_cmd(sc, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, &cmd);
1120         if (err != 0) {
1121                 device_printf(sc->dev,
1122                               "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS"
1123                               ", error status: %d\n", err);
1124                 return;
1125         }
1126
1127         /* Walk the multicast list, and add each address */
1128
1129         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1130                 if (ifma->ifma_addr->sa_family != AF_LINK)
1131                         continue;
1132                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1133                       &cmd.data0, 4);
1134                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr) + 4,
1135                       &cmd.data1, 2);
1136                 cmd.data0 = htonl(cmd.data0);
1137                 cmd.data1 = htonl(cmd.data1);
1138                 err = mxge_send_cmd(sc, MXGEFW_JOIN_MULTICAST_GROUP, &cmd);
1139                 if (err != 0) {
1140                         device_printf(sc->dev, "Failed "
1141                                "MXGEFW_JOIN_MULTICAST_GROUP, error status:"
1142                                "%d\t", err);
1143                         /* abort, leaving multicast filtering off */
1144                         return;
1145                 }
1146         }
1147         /* Enable multicast filtering */
1148         err = mxge_send_cmd(sc, MXGEFW_DISABLE_ALLMULTI, &cmd);
1149         if (err != 0) {
1150                 device_printf(sc->dev, "Failed MXGEFW_DISABLE_ALLMULTI"
1151                        ", error status: %d\n", err);
1152         }
1153 }
1154
1155 #if 0
1156 static int
1157 mxge_max_mtu(mxge_softc_t *sc)
1158 {
1159         mxge_cmd_t cmd;
1160         int status;
1161
1162         if (MJUMPAGESIZE - MXGEFW_PAD >  MXGEFW_MAX_MTU)
1163                 return  MXGEFW_MAX_MTU - MXGEFW_PAD;
1164
1165         /* try to set nbufs to see if it we can
1166            use virtually contiguous jumbos */
1167         cmd.data0 = 0;
1168         status = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
1169                                &cmd);
1170         if (status == 0)
1171                 return  MXGEFW_MAX_MTU - MXGEFW_PAD;
1172
1173         /* otherwise, we're limited to MJUMPAGESIZE */
1174         return MJUMPAGESIZE - MXGEFW_PAD;
1175 }
1176 #endif
1177
1178 static int
1179 mxge_reset(mxge_softc_t *sc, int interrupts_setup)
1180 {
1181         struct mxge_slice_state *ss;
1182         mxge_rx_done_t *rx_done;
1183         volatile uint32_t *irq_claim;
1184         mxge_cmd_t cmd;
1185         int slice, status;
1186
1187         /* try to send a reset command to the card to see if it
1188            is alive */
1189         memset(&cmd, 0, sizeof (cmd));
1190         status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
1191         if (status != 0) {
1192                 device_printf(sc->dev, "failed reset\n");
1193                 return ENXIO;
1194         }
1195
1196         mxge_dummy_rdma(sc, 1);
1197
1198
1199         /* set the intrq size */
1200         cmd.data0 = sc->rx_ring_size;
1201         status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
1202
1203         /*
1204          * Even though we already know how many slices are supported
1205          * via mxge_slice_probe(), MXGEFW_CMD_GET_MAX_RSS_QUEUES
1206          * has magic side effects, and must be called after a reset.
1207          * It must be called prior to calling any RSS related cmds,
1208          * including assigning an interrupt queue for anything but
1209          * slice 0.  It must also be called *after*
1210          * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
1211          * the firmware to compute offsets.
1212          */
1213
1214         if (sc->num_slices > 1) {
1215                 /* ask the maximum number of slices it supports */
1216                 status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
1217                                            &cmd);
1218                 if (status != 0) {
1219                         device_printf(sc->dev,
1220                                       "failed to get number of slices\n");
1221                         return status;
1222                 }
1223                 /*
1224                  * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
1225                  * to setting up the interrupt queue DMA
1226                  */
1227                 cmd.data0 = sc->num_slices;
1228                 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE;
1229 #ifdef IFNET_BUF_RING
1230                 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
1231 #endif
1232                 status = mxge_send_cmd(sc, MXGEFW_CMD_ENABLE_RSS_QUEUES,
1233                                            &cmd);
1234                 if (status != 0) {
1235                         device_printf(sc->dev,
1236                                       "failed to set number of slices\n");
1237                         return status;
1238                 }
1239         }
1240
1241
1242         if (interrupts_setup) {
1243                 /* Now exchange information about interrupts  */
1244                 for (slice = 0; slice < sc->num_slices; slice++) {
1245                         rx_done = &sc->ss[slice].rx_done;
1246                         memset(rx_done->entry, 0, sc->rx_ring_size);
1247                         cmd.data0 = MXGE_LOWPART_TO_U32(rx_done->dma.bus_addr);
1248                         cmd.data1 = MXGE_HIGHPART_TO_U32(rx_done->dma.bus_addr);
1249                         cmd.data2 = slice;
1250                         status |= mxge_send_cmd(sc,
1251                                                 MXGEFW_CMD_SET_INTRQ_DMA,
1252                                                 &cmd);
1253                 }
1254         }
1255
1256         status |= mxge_send_cmd(sc,
1257                                 MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd);
1258
1259
1260         sc->intr_coal_delay_ptr = (volatile uint32_t *)(sc->sram + cmd.data0);
1261
1262         status |= mxge_send_cmd(sc, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
1263         irq_claim = (volatile uint32_t *)(sc->sram + cmd.data0);
1264
1265
1266         status |= mxge_send_cmd(sc,  MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET,
1267                                 &cmd);
1268         sc->irq_deassert = (volatile uint32_t *)(sc->sram + cmd.data0);
1269         if (status != 0) {
1270                 device_printf(sc->dev, "failed set interrupt parameters\n");
1271                 return status;
1272         }
1273
1274
1275         *sc->intr_coal_delay_ptr = htobe32(sc->intr_coal_delay);
1276
1277
1278         /* run a DMA benchmark */
1279         (void) mxge_dma_test(sc, MXGEFW_DMA_TEST);
1280
1281         for (slice = 0; slice < sc->num_slices; slice++) {
1282                 ss = &sc->ss[slice];
1283
1284                 ss->irq_claim = irq_claim + (2 * slice);
1285                 /* reset mcp/driver shared state back to 0 */
1286                 ss->rx_done.idx = 0;
1287                 ss->rx_done.cnt = 0;
1288                 ss->tx.req = 0;
1289                 ss->tx.done = 0;
1290                 ss->tx.pkt_done = 0;
1291                 ss->tx.queue_active = 0;
1292                 ss->tx.activate = 0;
1293                 ss->tx.deactivate = 0;
1294                 ss->tx.wake = 0;
1295                 ss->tx.defrag = 0;
1296                 ss->tx.stall = 0;
1297                 ss->rx_big.cnt = 0;
1298                 ss->rx_small.cnt = 0;
1299                 if (ss->fw_stats != NULL) {
1300                         bzero(ss->fw_stats, sizeof *ss->fw_stats);
1301                 }
1302         }
1303         sc->rdma_tags_available = 15;
1304         status = mxge_update_mac_address(sc);
1305         mxge_change_promisc(sc, sc->ifp->if_flags & IFF_PROMISC);
1306         mxge_change_pause(sc, sc->pause);
1307         mxge_set_multicast_list(sc);
1308         if (sc->throttle) {
1309                 cmd.data0 = sc->throttle;
1310                 if (mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR,
1311                                   &cmd)) {
1312                         device_printf(sc->dev,
1313                                       "can't enable throttle\n");
1314                 }
1315         }
1316         return status;
1317 }
1318
1319 static int
1320 mxge_change_throttle(SYSCTL_HANDLER_ARGS)
1321 {
1322         mxge_cmd_t cmd;
1323         mxge_softc_t *sc;
1324         int err;
1325         unsigned int throttle;
1326
1327         sc = arg1;
1328         throttle = sc->throttle;
1329         err = sysctl_handle_int(oidp, &throttle, arg2, req);
1330         if (err != 0) {
1331                 return err;
1332         }
1333
1334         if (throttle == sc->throttle)
1335                 return 0;
1336
1337         if (throttle < MXGE_MIN_THROTTLE || throttle > MXGE_MAX_THROTTLE)
1338                 return EINVAL;
1339
1340         /* XXX SERIALIZE */
1341         lwkt_serialize_enter(sc->ifp->if_serializer);
1342
1343         cmd.data0 = throttle;
1344         err = mxge_send_cmd(sc, MXGEFW_CMD_SET_THROTTLE_FACTOR, &cmd);
1345         if (err == 0)
1346                 sc->throttle = throttle;
1347
1348         lwkt_serialize_exit(sc->ifp->if_serializer);
1349         return err;
1350 }
1351
1352 static int
1353 mxge_change_intr_coal(SYSCTL_HANDLER_ARGS)
1354 {
1355         mxge_softc_t *sc;
1356         unsigned int intr_coal_delay;
1357         int err;
1358
1359         sc = arg1;
1360         intr_coal_delay = sc->intr_coal_delay;
1361         err = sysctl_handle_int(oidp, &intr_coal_delay, arg2, req);
1362         if (err != 0) {
1363                 return err;
1364         }
1365         if (intr_coal_delay == sc->intr_coal_delay)
1366                 return 0;
1367
1368         if (intr_coal_delay == 0 || intr_coal_delay > 1000*1000)
1369                 return EINVAL;
1370
1371         /* XXX SERIALIZE */
1372         lwkt_serialize_enter(sc->ifp->if_serializer);
1373
1374         *sc->intr_coal_delay_ptr = htobe32(intr_coal_delay);
1375         sc->intr_coal_delay = intr_coal_delay;
1376
1377         lwkt_serialize_exit(sc->ifp->if_serializer);
1378         return err;
1379 }
1380
1381 static int
1382 mxge_change_flow_control(SYSCTL_HANDLER_ARGS)
1383 {
1384         mxge_softc_t *sc;
1385         unsigned int enabled;
1386         int err;
1387
1388         sc = arg1;
1389         enabled = sc->pause;
1390         err = sysctl_handle_int(oidp, &enabled, arg2, req);
1391         if (err != 0) {
1392                 return err;
1393         }
1394         if (enabled == sc->pause)
1395                 return 0;
1396
1397         /* XXX SERIALIZE */
1398         lwkt_serialize_enter(sc->ifp->if_serializer);
1399         err = mxge_change_pause(sc, enabled);
1400         lwkt_serialize_exit(sc->ifp->if_serializer);
1401
1402         return err;
1403 }
1404
1405 static int
1406 mxge_handle_be32(SYSCTL_HANDLER_ARGS)
1407 {
1408         int err;
1409
1410         if (arg1 == NULL)
1411                 return EFAULT;
1412         arg2 = be32toh(*(int *)arg1);
1413         arg1 = NULL;
1414         err = sysctl_handle_int(oidp, arg1, arg2, req);
1415
1416         return err;
1417 }
1418
1419 static void
1420 mxge_rem_sysctls(mxge_softc_t *sc)
1421 {
1422         struct mxge_slice_state *ss;
1423         int slice;
1424
1425         if (sc->slice_sysctl_tree == NULL)
1426                 return;
1427
1428         for (slice = 0; slice < sc->num_slices; slice++) {
1429                 ss = &sc->ss[slice];
1430                 if (ss == NULL || ss->sysctl_tree == NULL)
1431                         continue;
1432                 sysctl_ctx_free(&ss->sysctl_ctx);
1433                 ss->sysctl_tree = NULL;
1434         }
1435         sysctl_ctx_free(&sc->slice_sysctl_ctx);
1436         sc->slice_sysctl_tree = NULL;
1437         sysctl_ctx_free(&sc->sysctl_ctx);
1438         sc->sysctl_tree = NULL;
1439 }
1440
1441 static void
1442 mxge_add_sysctls(mxge_softc_t *sc)
1443 {
1444         struct sysctl_ctx_list *ctx;
1445         struct sysctl_oid_list *children;
1446         mcp_irq_data_t *fw;
1447         struct mxge_slice_state *ss;
1448         int slice;
1449         char slice_num[8];
1450
1451         ctx = &sc->sysctl_ctx;
1452         sysctl_ctx_init(ctx);
1453         sc->sysctl_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_STATIC_CHILDREN(_hw),
1454                                           OID_AUTO,
1455                                           device_get_nameunit(sc->dev),
1456                                           CTLFLAG_RD, 0, "");
1457         if (sc->sysctl_tree == NULL) {
1458                 device_printf(sc->dev, "can't add sysctl node\n");
1459                 return;
1460         }
1461
1462         children = SYSCTL_CHILDREN(sc->sysctl_tree);
1463         fw = sc->ss[0].fw_stats;
1464
1465         /* random information */
1466         SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1467                        "firmware_version",
1468                        CTLFLAG_RD, &sc->fw_version,
1469                        0, "firmware version");
1470         SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1471                        "serial_number",
1472                        CTLFLAG_RD, &sc->serial_number_string,
1473                        0, "serial number");
1474         SYSCTL_ADD_STRING(ctx, children, OID_AUTO,
1475                        "product_code",
1476                        CTLFLAG_RD, &sc->product_code_string,
1477                        0, "product_code");
1478         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1479                        "pcie_link_width",
1480                        CTLFLAG_RD, &sc->link_width,
1481                        0, "tx_boundary");
1482         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1483                        "tx_boundary",
1484                        CTLFLAG_RD, &sc->tx_boundary,
1485                        0, "tx_boundary");
1486         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1487                        "write_combine",
1488                        CTLFLAG_RD, &sc->wc,
1489                        0, "write combining PIO?");
1490         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1491                        "read_dma_MBs",
1492                        CTLFLAG_RD, &sc->read_dma,
1493                        0, "DMA Read speed in MB/s");
1494         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1495                        "write_dma_MBs",
1496                        CTLFLAG_RD, &sc->write_dma,
1497                        0, "DMA Write speed in MB/s");
1498         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1499                        "read_write_dma_MBs",
1500                        CTLFLAG_RD, &sc->read_write_dma,
1501                        0, "DMA concurrent Read/Write speed in MB/s");
1502         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1503                        "watchdog_resets",
1504                        CTLFLAG_RD, &sc->watchdog_resets,
1505                        0, "Number of times NIC was reset");
1506
1507
1508         /* performance related tunables */
1509         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1510                         "intr_coal_delay",
1511                         CTLTYPE_INT|CTLFLAG_RW, sc,
1512                         0, mxge_change_intr_coal,
1513                         "I", "interrupt coalescing delay in usecs");
1514
1515         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1516                         "throttle",
1517                         CTLTYPE_INT|CTLFLAG_RW, sc,
1518                         0, mxge_change_throttle,
1519                         "I", "transmit throttling");
1520
1521         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1522                         "flow_control_enabled",
1523                         CTLTYPE_INT|CTLFLAG_RW, sc,
1524                         0, mxge_change_flow_control,
1525                         "I", "interrupt coalescing delay in usecs");
1526
1527         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1528                        "deassert_wait",
1529                        CTLFLAG_RW, &mxge_deassert_wait,
1530                        0, "Wait for IRQ line to go low in ihandler");
1531
1532         /* stats block from firmware is in network byte order.
1533            Need to swap it */
1534         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1535                         "link_up",
1536                         CTLTYPE_INT|CTLFLAG_RD, &fw->link_up,
1537                         0, mxge_handle_be32,
1538                         "I", "link up");
1539         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1540                         "rdma_tags_available",
1541                         CTLTYPE_INT|CTLFLAG_RD, &fw->rdma_tags_available,
1542                         0, mxge_handle_be32,
1543                         "I", "rdma_tags_available");
1544         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1545                         "dropped_bad_crc32",
1546                         CTLTYPE_INT|CTLFLAG_RD,
1547                         &fw->dropped_bad_crc32,
1548                         0, mxge_handle_be32,
1549                         "I", "dropped_bad_crc32");
1550         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1551                         "dropped_bad_phy",
1552                         CTLTYPE_INT|CTLFLAG_RD,
1553                         &fw->dropped_bad_phy,
1554                         0, mxge_handle_be32,
1555                         "I", "dropped_bad_phy");
1556         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1557                         "dropped_link_error_or_filtered",
1558                         CTLTYPE_INT|CTLFLAG_RD,
1559                         &fw->dropped_link_error_or_filtered,
1560                         0, mxge_handle_be32,
1561                         "I", "dropped_link_error_or_filtered");
1562         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1563                         "dropped_link_overflow",
1564                         CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_link_overflow,
1565                         0, mxge_handle_be32,
1566                         "I", "dropped_link_overflow");
1567         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1568                         "dropped_multicast_filtered",
1569                         CTLTYPE_INT|CTLFLAG_RD,
1570                         &fw->dropped_multicast_filtered,
1571                         0, mxge_handle_be32,
1572                         "I", "dropped_multicast_filtered");
1573         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1574                         "dropped_no_big_buffer",
1575                         CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_no_big_buffer,
1576                         0, mxge_handle_be32,
1577                         "I", "dropped_no_big_buffer");
1578         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1579                         "dropped_no_small_buffer",
1580                         CTLTYPE_INT|CTLFLAG_RD,
1581                         &fw->dropped_no_small_buffer,
1582                         0, mxge_handle_be32,
1583                         "I", "dropped_no_small_buffer");
1584         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1585                         "dropped_overrun",
1586                         CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_overrun,
1587                         0, mxge_handle_be32,
1588                         "I", "dropped_overrun");
1589         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1590                         "dropped_pause",
1591                         CTLTYPE_INT|CTLFLAG_RD,
1592                         &fw->dropped_pause,
1593                         0, mxge_handle_be32,
1594                         "I", "dropped_pause");
1595         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1596                         "dropped_runt",
1597                         CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_runt,
1598                         0, mxge_handle_be32,
1599                         "I", "dropped_runt");
1600
1601         SYSCTL_ADD_PROC(ctx, children, OID_AUTO,
1602                         "dropped_unicast_filtered",
1603                         CTLTYPE_INT|CTLFLAG_RD, &fw->dropped_unicast_filtered,
1604                         0, mxge_handle_be32,
1605                         "I", "dropped_unicast_filtered");
1606
1607         /* verbose printing? */
1608         SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1609                        "verbose",
1610                        CTLFLAG_RW, &mxge_verbose,
1611                        0, "verbose printing");
1612
1613         /* add counters exported for debugging from all slices */
1614         sysctl_ctx_init(&sc->slice_sysctl_ctx);
1615         sc->slice_sysctl_tree =
1616                 SYSCTL_ADD_NODE(&sc->slice_sysctl_ctx, children, OID_AUTO,
1617                                 "slice", CTLFLAG_RD, 0, "");
1618
1619         for (slice = 0; slice < sc->num_slices; slice++) {
1620                 ss = &sc->ss[slice];
1621                 sysctl_ctx_init(&ss->sysctl_ctx);
1622                 ctx = &ss->sysctl_ctx;
1623                 children = SYSCTL_CHILDREN(sc->slice_sysctl_tree);
1624                 ksprintf(slice_num, "%d", slice);
1625                 ss->sysctl_tree =
1626                         SYSCTL_ADD_NODE(ctx, children, OID_AUTO, slice_num,
1627                                         CTLFLAG_RD, 0, "");
1628                 children = SYSCTL_CHILDREN(ss->sysctl_tree);
1629                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1630                                "rx_small_cnt",
1631                                CTLFLAG_RD, &ss->rx_small.cnt,
1632                                0, "rx_small_cnt");
1633                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1634                                "rx_big_cnt",
1635                                CTLFLAG_RD, &ss->rx_big.cnt,
1636                                0, "rx_small_cnt");
1637
1638 #ifndef IFNET_BUF_RING
1639                 /* only transmit from slice 0 for now */
1640                 if (slice > 0)
1641                         continue;
1642 #endif
1643                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1644                                "tx_req",
1645                                CTLFLAG_RD, &ss->tx.req,
1646                                0, "tx_req");
1647
1648                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1649                                "tx_done",
1650                                CTLFLAG_RD, &ss->tx.done,
1651                                0, "tx_done");
1652                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1653                                "tx_pkt_done",
1654                                CTLFLAG_RD, &ss->tx.pkt_done,
1655                                0, "tx_done");
1656                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1657                                "tx_stall",
1658                                CTLFLAG_RD, &ss->tx.stall,
1659                                0, "tx_stall");
1660                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1661                                "tx_wake",
1662                                CTLFLAG_RD, &ss->tx.wake,
1663                                0, "tx_wake");
1664                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1665                                "tx_defrag",
1666                                CTLFLAG_RD, &ss->tx.defrag,
1667                                0, "tx_defrag");
1668                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1669                                "tx_queue_active",
1670                                CTLFLAG_RD, &ss->tx.queue_active,
1671                                0, "tx_queue_active");
1672                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1673                                "tx_activate",
1674                                CTLFLAG_RD, &ss->tx.activate,
1675                                0, "tx_activate");
1676                 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
1677                                "tx_deactivate",
1678                                CTLFLAG_RD, &ss->tx.deactivate,
1679                                0, "tx_deactivate");
1680         }
1681 }
1682
1683 /*
1684  * Copy an array of mcp_kreq_ether_send_t's to the mcp.  Copy
1685  * backwards one at a time and handle ring wraps
1686  */
1687 static inline void
1688 mxge_submit_req_backwards(mxge_tx_ring_t *tx,
1689     mcp_kreq_ether_send_t *src, int cnt)
1690 {
1691         int idx, starting_slot;
1692         starting_slot = tx->req;
1693         while (cnt > 1) {
1694                 cnt--;
1695                 idx = (starting_slot + cnt) & tx->mask;
1696                 mxge_pio_copy(&tx->lanai[idx],
1697                               &src[cnt], sizeof(*src));
1698                 wmb();
1699         }
1700 }
1701
1702 /*
1703  * Copy an array of mcp_kreq_ether_send_t's to the mcp.  Copy
1704  * at most 32 bytes at a time, so as to avoid involving the software
1705  * pio handler in the nic.   We re-write the first segment's flags
1706  * to mark them valid only after writing the entire chain
1707  */
1708 static inline void
1709 mxge_submit_req(mxge_tx_ring_t *tx, mcp_kreq_ether_send_t *src, int cnt)
1710 {
1711         int idx, i;
1712         uint32_t *src_ints;
1713         volatile uint32_t *dst_ints;
1714         mcp_kreq_ether_send_t *srcp;
1715         volatile mcp_kreq_ether_send_t *dstp, *dst;
1716         uint8_t last_flags;
1717
1718         idx = tx->req & tx->mask;
1719
1720         last_flags = src->flags;
1721         src->flags = 0;
1722         wmb();
1723         dst = dstp = &tx->lanai[idx];
1724         srcp = src;
1725
1726         if ((idx + cnt) < tx->mask) {
1727                 for (i = 0; i < (cnt - 1); i += 2) {
1728                         mxge_pio_copy(dstp, srcp, 2 * sizeof(*src));
1729                         wmb(); /* force write every 32 bytes */
1730                         srcp += 2;
1731                         dstp += 2;
1732                 }
1733         } else {
1734                 /* submit all but the first request, and ensure
1735                    that it is submitted below */
1736                 mxge_submit_req_backwards(tx, src, cnt);
1737                 i = 0;
1738         }
1739         if (i < cnt) {
1740                 /* submit the first request */
1741                 mxge_pio_copy(dstp, srcp, sizeof(*src));
1742                 wmb(); /* barrier before setting valid flag */
1743         }
1744
1745         /* re-write the last 32-bits with the valid flags */
1746         src->flags = last_flags;
1747         src_ints = (uint32_t *)src;
1748         src_ints+=3;
1749         dst_ints = (volatile uint32_t *)dst;
1750         dst_ints+=3;
1751         *dst_ints =  *src_ints;
1752         tx->req += cnt;
1753         wmb();
1754 }
1755
1756 static int
1757 mxge_pullup_tso(struct mbuf **mp)
1758 {
1759         int hoff, iphlen, thoff;
1760         struct mbuf *m;
1761
1762         m = *mp;
1763         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
1764
1765         iphlen = m->m_pkthdr.csum_iphlen;
1766         thoff = m->m_pkthdr.csum_thlen;
1767         hoff = m->m_pkthdr.csum_lhlen;
1768
1769         KASSERT(iphlen > 0, ("invalid ip hlen"));
1770         KASSERT(thoff > 0, ("invalid tcp hlen"));
1771         KASSERT(hoff > 0, ("invalid ether hlen"));
1772
1773         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
1774                 m = m_pullup(m, hoff + iphlen + thoff);
1775                 if (m == NULL) {
1776                         *mp = NULL;
1777                         return ENOBUFS;
1778                 }
1779                 *mp = m;
1780         }
1781         return 0;
1782 }
1783
1784 static void
1785 mxge_encap_tso(struct mxge_slice_state *ss, struct mbuf *m,
1786     int busdma_seg_cnt)
1787 {
1788         mxge_tx_ring_t *tx;
1789         mcp_kreq_ether_send_t *req;
1790         bus_dma_segment_t *seg;
1791         uint32_t low, high_swapped;
1792         int len, seglen, cum_len, cum_len_next;
1793         int next_is_first, chop, cnt, rdma_count, small;
1794         uint16_t pseudo_hdr_offset, cksum_offset, mss;
1795         uint8_t flags, flags_next;
1796         static int once;
1797
1798         mss = m->m_pkthdr.tso_segsz;
1799
1800         /* negative cum_len signifies to the
1801          * send loop that we are still in the
1802          * header portion of the TSO packet.
1803          */
1804         cum_len = -(m->m_pkthdr.csum_lhlen + m->m_pkthdr.csum_iphlen +
1805             m->m_pkthdr.csum_thlen);
1806
1807         /* TSO implies checksum offload on this hardware */
1808         cksum_offset = m->m_pkthdr.csum_lhlen + m->m_pkthdr.csum_iphlen;
1809         flags = MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST;
1810
1811         /* for TSO, pseudo_hdr_offset holds mss.
1812          * The firmware figures out where to put
1813          * the checksum by parsing the header. */
1814         pseudo_hdr_offset = htobe16(mss);
1815
1816         tx = &ss->tx;
1817         req = tx->req_list;
1818         seg = tx->seg_list;
1819         cnt = 0;
1820         rdma_count = 0;
1821         /* "rdma_count" is the number of RDMAs belonging to the
1822          * current packet BEFORE the current send request. For
1823          * non-TSO packets, this is equal to "count".
1824          * For TSO packets, rdma_count needs to be reset
1825          * to 0 after a segment cut.
1826          *
1827          * The rdma_count field of the send request is
1828          * the number of RDMAs of the packet starting at
1829          * that request. For TSO send requests with one ore more cuts
1830          * in the middle, this is the number of RDMAs starting
1831          * after the last cut in the request. All previous
1832          * segments before the last cut implicitly have 1 RDMA.
1833          *
1834          * Since the number of RDMAs is not known beforehand,
1835          * it must be filled-in retroactively - after each
1836          * segmentation cut or at the end of the entire packet.
1837          */
1838
1839         while (busdma_seg_cnt) {
1840                 /* Break the busdma segment up into pieces*/
1841                 low = MXGE_LOWPART_TO_U32(seg->ds_addr);
1842                 high_swapped =  htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
1843                 len = seg->ds_len;
1844
1845                 while (len) {
1846                         flags_next = flags & ~MXGEFW_FLAGS_FIRST;
1847                         seglen = len;
1848                         cum_len_next = cum_len + seglen;
1849                         (req-rdma_count)->rdma_count = rdma_count + 1;
1850                         if (__predict_true(cum_len >= 0)) {
1851                                 /* payload */
1852                                 chop = (cum_len_next > mss);
1853                                 cum_len_next = cum_len_next % mss;
1854                                 next_is_first = (cum_len_next == 0);
1855                                 flags |= chop * MXGEFW_FLAGS_TSO_CHOP;
1856                                 flags_next |= next_is_first *
1857                                         MXGEFW_FLAGS_FIRST;
1858                                 rdma_count |= -(chop | next_is_first);
1859                                 rdma_count += chop & !next_is_first;
1860                         } else if (cum_len_next >= 0) {
1861                                 /* header ends */
1862                                 rdma_count = -1;
1863                                 cum_len_next = 0;
1864                                 seglen = -cum_len;
1865                                 small = (mss <= MXGEFW_SEND_SMALL_SIZE);
1866                                 flags_next = MXGEFW_FLAGS_TSO_PLD |
1867                                         MXGEFW_FLAGS_FIRST |
1868                                         (small * MXGEFW_FLAGS_SMALL);
1869                         }
1870
1871                         req->addr_high = high_swapped;
1872                         req->addr_low = htobe32(low);
1873                         req->pseudo_hdr_offset = pseudo_hdr_offset;
1874                         req->pad = 0;
1875                         req->rdma_count = 1;
1876                         req->length = htobe16(seglen);
1877                         req->cksum_offset = cksum_offset;
1878                         req->flags = flags | ((cum_len & 1) *
1879                                               MXGEFW_FLAGS_ALIGN_ODD);
1880                         low += seglen;
1881                         len -= seglen;
1882                         cum_len = cum_len_next;
1883                         flags = flags_next;
1884                         req++;
1885                         cnt++;
1886                         rdma_count++;
1887                         if (__predict_false(cksum_offset > seglen))
1888                                 cksum_offset -= seglen;
1889                         else
1890                                 cksum_offset = 0;
1891                         if (__predict_false(cnt > tx->max_desc))
1892                                 goto drop;
1893                 }
1894                 busdma_seg_cnt--;
1895                 seg++;
1896         }
1897         (req-rdma_count)->rdma_count = rdma_count;
1898
1899         do {
1900                 req--;
1901                 req->flags |= MXGEFW_FLAGS_TSO_LAST;
1902         } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | MXGEFW_FLAGS_FIRST)));
1903
1904         tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
1905         mxge_submit_req(tx, tx->req_list, cnt);
1906 #ifdef IFNET_BUF_RING
1907         if ((ss->sc->num_slices > 1) && tx->queue_active == 0) {
1908                 /* tell the NIC to start polling this slice */
1909                 *tx->send_go = 1;
1910                 tx->queue_active = 1;
1911                 tx->activate++;
1912                 wmb();
1913         }
1914 #endif
1915         return;
1916
1917 drop:
1918         bus_dmamap_unload(tx->dmat, tx->info[tx->req & tx->mask].map);
1919         m_freem(m);
1920         ss->oerrors++;
1921         if (!once) {
1922                 kprintf("tx->max_desc exceeded via TSO!\n");
1923                 kprintf("mss = %d, %ld, %d!\n", mss,
1924                        (long)seg - (long)tx->seg_list, tx->max_desc);
1925                 once = 1;
1926         }
1927 }
1928
1929 static void
1930 mxge_encap(struct mxge_slice_state *ss, struct mbuf *m)
1931 {
1932         mxge_softc_t *sc;
1933         mcp_kreq_ether_send_t *req;
1934         bus_dma_segment_t *seg;
1935         mxge_tx_ring_t *tx;
1936         int cnt, cum_len, err, i, idx, odd_flag;
1937         uint16_t pseudo_hdr_offset;
1938         uint8_t flags, cksum_offset;
1939
1940         sc = ss->sc;
1941         tx = &ss->tx;
1942
1943         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1944                 if (mxge_pullup_tso(&m))
1945                         return;
1946         }
1947
1948         /* (try to) map the frame for DMA */
1949         idx = tx->req & tx->mask;
1950         err = bus_dmamap_load_mbuf_defrag(tx->dmat, tx->info[idx].map, &m,
1951             tx->seg_list, tx->max_desc - 2, &cnt, BUS_DMA_NOWAIT);
1952         if (__predict_false(err != 0))
1953                 goto drop;
1954         bus_dmamap_sync(tx->dmat, tx->info[idx].map, BUS_DMASYNC_PREWRITE);
1955         tx->info[idx].m = m;
1956
1957         /* TSO is different enough, we handle it in another routine */
1958         if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1959                 mxge_encap_tso(ss, m, cnt);
1960                 return;
1961         }
1962
1963         req = tx->req_list;
1964         cksum_offset = 0;
1965         pseudo_hdr_offset = 0;
1966         flags = MXGEFW_FLAGS_NO_TSO;
1967
1968         /* checksum offloading? */
1969         if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) {
1970                 cksum_offset = m->m_pkthdr.csum_lhlen + m->m_pkthdr.csum_iphlen;
1971                 pseudo_hdr_offset = cksum_offset +  m->m_pkthdr.csum_data;
1972                 pseudo_hdr_offset = htobe16(pseudo_hdr_offset);
1973                 req->cksum_offset = cksum_offset;
1974                 flags |= MXGEFW_FLAGS_CKSUM;
1975                 odd_flag = MXGEFW_FLAGS_ALIGN_ODD;
1976         } else {
1977                 odd_flag = 0;
1978         }
1979         if (m->m_pkthdr.len < MXGEFW_SEND_SMALL_SIZE)
1980                 flags |= MXGEFW_FLAGS_SMALL;
1981
1982         /* convert segments into a request list */
1983         cum_len = 0;
1984         seg = tx->seg_list;
1985         req->flags = MXGEFW_FLAGS_FIRST;
1986         for (i = 0; i < cnt; i++) {
1987                 req->addr_low =
1988                         htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr));
1989                 req->addr_high =
1990                         htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
1991                 req->length = htobe16(seg->ds_len);
1992                 req->cksum_offset = cksum_offset;
1993                 if (cksum_offset > seg->ds_len)
1994                         cksum_offset -= seg->ds_len;
1995                 else
1996                         cksum_offset = 0;
1997                 req->pseudo_hdr_offset = pseudo_hdr_offset;
1998                 req->pad = 0; /* complete solid 16-byte block */
1999                 req->rdma_count = 1;
2000                 req->flags |= flags | ((cum_len & 1) * odd_flag);
2001                 cum_len += seg->ds_len;
2002                 seg++;
2003                 req++;
2004                 req->flags = 0;
2005         }
2006         req--;
2007         /* pad runts to 60 bytes */
2008         if (cum_len < 60) {
2009                 req++;
2010                 req->addr_low =
2011                         htobe32(MXGE_LOWPART_TO_U32(sc->zeropad_dma.bus_addr));
2012                 req->addr_high =
2013                         htobe32(MXGE_HIGHPART_TO_U32(sc->zeropad_dma.bus_addr));
2014                 req->length = htobe16(60 - cum_len);
2015                 req->cksum_offset = 0;
2016                 req->pseudo_hdr_offset = pseudo_hdr_offset;
2017                 req->pad = 0; /* complete solid 16-byte block */
2018                 req->rdma_count = 1;
2019                 req->flags |= flags | ((cum_len & 1) * odd_flag);
2020                 cnt++;
2021         }
2022
2023         tx->req_list[0].rdma_count = cnt;
2024 #if 0
2025         /* print what the firmware will see */
2026         for (i = 0; i < cnt; i++) {
2027                 kprintf("%d: addr: 0x%x 0x%x len:%d pso%d,"
2028                     "cso:%d, flags:0x%x, rdma:%d\n",
2029                     i, (int)ntohl(tx->req_list[i].addr_high),
2030                     (int)ntohl(tx->req_list[i].addr_low),
2031                     (int)ntohs(tx->req_list[i].length),
2032                     (int)ntohs(tx->req_list[i].pseudo_hdr_offset),
2033                     tx->req_list[i].cksum_offset, tx->req_list[i].flags,
2034                     tx->req_list[i].rdma_count);
2035         }
2036         kprintf("--------------\n");
2037 #endif
2038         tx->info[((cnt - 1) + tx->req) & tx->mask].flag = 1;
2039         mxge_submit_req(tx, tx->req_list, cnt);
2040 #ifdef IFNET_BUF_RING
2041         if ((ss->sc->num_slices > 1) && tx->queue_active == 0) {
2042                 /* tell the NIC to start polling this slice */
2043                 *tx->send_go = 1;
2044                 tx->queue_active = 1;
2045                 tx->activate++;
2046                 wmb();
2047         }
2048 #endif
2049         return;
2050
2051 drop:
2052         m_freem(m);
2053         ss->oerrors++;
2054 }
2055
2056 static inline void
2057 mxge_start_locked(struct mxge_slice_state *ss)
2058 {
2059         mxge_softc_t *sc;
2060         struct mbuf *m;
2061         struct ifnet *ifp;
2062         mxge_tx_ring_t *tx;
2063
2064         sc = ss->sc;
2065         ifp = sc->ifp;
2066         tx = &ss->tx;
2067         while ((tx->mask - (tx->req - tx->done)) > tx->max_desc) {
2068                 m = ifq_dequeue(&ifp->if_snd);
2069                 if (m == NULL) {
2070                         return;
2071                 }
2072                 /* let BPF see it */
2073                 BPF_MTAP(ifp, m);
2074
2075                 /* give it to the nic */
2076                 mxge_encap(ss, m);
2077         }
2078
2079         /* ran out of transmit slots */
2080         ifq_set_oactive(&ifp->if_snd);
2081 }
2082
2083 static void
2084 mxge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2085 {
2086         mxge_softc_t *sc = ifp->if_softc;
2087         struct mxge_slice_state *ss;
2088
2089         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
2090         ASSERT_SERIALIZED(sc->ifp->if_serializer);
2091         /* only use the first slice for now */
2092         ss = &sc->ss[0];
2093         mxge_start_locked(ss);
2094 }
2095
2096 /*
2097  * copy an array of mcp_kreq_ether_recv_t's to the mcp.  Copy
2098  * at most 32 bytes at a time, so as to avoid involving the software
2099  * pio handler in the nic.   We re-write the first segment's low
2100  * DMA address to mark it valid only after we write the entire chunk
2101  * in a burst
2102  */
2103 static inline void
2104 mxge_submit_8rx(volatile mcp_kreq_ether_recv_t *dst,
2105     mcp_kreq_ether_recv_t *src)
2106 {
2107         uint32_t low;
2108
2109         low = src->addr_low;
2110         src->addr_low = 0xffffffff;
2111         mxge_pio_copy(dst, src, 4 * sizeof (*src));
2112         wmb();
2113         mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
2114         wmb();
2115         src->addr_low = low;
2116         dst->addr_low = low;
2117         wmb();
2118 }
2119
2120 static int
2121 mxge_get_buf_small(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2122 {
2123         bus_dma_segment_t seg;
2124         struct mbuf *m;
2125         mxge_rx_ring_t *rx = &ss->rx_small;
2126         int cnt, err;
2127
2128         m = m_gethdr(MB_DONTWAIT, MT_DATA);
2129         if (m == NULL) {
2130                 rx->alloc_fail++;
2131                 err = ENOBUFS;
2132                 goto done;
2133         }
2134         m->m_len = m->m_pkthdr.len = MHLEN;
2135         err = bus_dmamap_load_mbuf_segment(rx->dmat, map, m,
2136                                       &seg, 1, &cnt, BUS_DMA_NOWAIT);
2137         if (err != 0) {
2138                 kprintf("can't dmamap small (%d)\n", err);
2139                 m_free(m);
2140                 goto done;
2141         }
2142         rx->info[idx].m = m;
2143         rx->shadow[idx].addr_low =
2144                 htobe32(MXGE_LOWPART_TO_U32(seg.ds_addr));
2145         rx->shadow[idx].addr_high =
2146                 htobe32(MXGE_HIGHPART_TO_U32(seg.ds_addr));
2147
2148 done:
2149         if ((idx & 7) == 7)
2150                 mxge_submit_8rx(&rx->lanai[idx - 7], &rx->shadow[idx - 7]);
2151         return err;
2152 }
2153
2154 static int
2155 mxge_get_buf_big(struct mxge_slice_state *ss, bus_dmamap_t map, int idx)
2156 {
2157         bus_dma_segment_t seg[3];
2158         struct mbuf *m;
2159         mxge_rx_ring_t *rx = &ss->rx_big;
2160         int cnt, err, i;
2161
2162         if (rx->cl_size == MCLBYTES)
2163                 m = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR);
2164         else {
2165 #if 0
2166                 m = m_getjcl(MB_DONTWAIT, MT_DATA, M_PKTHDR, rx->cl_size);
2167 #else
2168                 /*
2169                  * XXX: allocate normal sized buffers for big buffers.
2170                  * We should be fine as long as we don't get any jumbo frames
2171                  */
2172                 m = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR);
2173 #endif
2174         }
2175         if (m == NULL) {
2176                 rx->alloc_fail++;
2177                 err = ENOBUFS;
2178                 goto done;
2179         }
2180         m->m_len = m->m_pkthdr.len = rx->mlen;
2181         err = bus_dmamap_load_mbuf_segment(rx->dmat, map, m,
2182                                       seg, 1, &cnt, BUS_DMA_NOWAIT);
2183         if (err != 0) {
2184                 kprintf("can't dmamap big (%d)\n", err);
2185                 m_free(m);
2186                 goto done;
2187         }
2188         rx->info[idx].m = m;
2189         rx->shadow[idx].addr_low =
2190                 htobe32(MXGE_LOWPART_TO_U32(seg->ds_addr));
2191         rx->shadow[idx].addr_high =
2192                 htobe32(MXGE_HIGHPART_TO_U32(seg->ds_addr));
2193
2194 #if MXGE_VIRT_JUMBOS
2195         for (i = 1; i < cnt; i++) {
2196                 rx->shadow[idx + i].addr_low =
2197                         htobe32(MXGE_LOWPART_TO_U32(seg[i].ds_addr));
2198                 rx->shadow[idx + i].addr_high =
2199                         htobe32(MXGE_HIGHPART_TO_U32(seg[i].ds_addr));
2200         }
2201 #endif
2202
2203 done:
2204         for (i = 0; i < rx->nbufs; i++) {
2205                 if ((idx & 7) == 7) {
2206                         mxge_submit_8rx(&rx->lanai[idx - 7],
2207                                         &rx->shadow[idx - 7]);
2208                 }
2209                 idx++;
2210         }
2211         return err;
2212 }
2213
2214 /*
2215  *  Myri10GE hardware checksums are not valid if the sender
2216  *  padded the frame with non-zero padding.  This is because
2217  *  the firmware just does a simple 16-bit 1s complement
2218  *  checksum across the entire frame, excluding the first 14
2219  *  bytes.  It is best to simply to check the checksum and
2220  *  tell the stack about it only if the checksum is good
2221  */
2222 static inline uint16_t
2223 mxge_rx_csum(struct mbuf *m, int csum)
2224 {
2225         struct ether_header *eh;
2226         struct ip *ip;
2227         uint16_t c;
2228
2229         eh = mtod(m, struct ether_header *);
2230
2231         /* only deal with IPv4 TCP & UDP for now */
2232         if (__predict_false(eh->ether_type != htons(ETHERTYPE_IP)))
2233                 return 1;
2234         ip = (struct ip *)(eh + 1);
2235         if (__predict_false(ip->ip_p != IPPROTO_TCP &&
2236                             ip->ip_p != IPPROTO_UDP))
2237                 return 1;
2238 #ifdef INET
2239         c = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2240                       htonl(ntohs(csum) + ntohs(ip->ip_len) +
2241                             - (ip->ip_hl << 2) + ip->ip_p));
2242 #else
2243         c = 1;
2244 #endif
2245         c ^= 0xffff;
2246         return (c);
2247 }
2248
2249 static void
2250 mxge_vlan_tag_remove(struct mbuf *m, uint32_t *csum)
2251 {
2252         struct ether_vlan_header *evl;
2253         uint32_t partial;
2254
2255         evl = mtod(m, struct ether_vlan_header *);
2256
2257         /*
2258          * fix checksum by subtracting EVL_ENCAPLEN bytes
2259          * after what the firmware thought was the end of the ethernet
2260          * header.
2261          */
2262
2263         /* put checksum into host byte order */
2264         *csum = ntohs(*csum);
2265         partial = ntohl(*(uint32_t *)(mtod(m, char *) + ETHER_HDR_LEN));
2266         (*csum) += ~partial;
2267         (*csum) +=  ((*csum) < ~partial);
2268         (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2269         (*csum) = ((*csum) >> 16) + ((*csum) & 0xFFFF);
2270
2271         /* restore checksum to network byte order;
2272            later consumers expect this */
2273         *csum = htons(*csum);
2274
2275         /* save the tag */
2276         m->m_pkthdr.ether_vlantag = ntohs(evl->evl_tag);
2277         m->m_flags |= M_VLANTAG;
2278
2279         /*
2280          * Remove the 802.1q header by copying the Ethernet
2281          * addresses over it and adjusting the beginning of
2282          * the data in the mbuf.  The encapsulated Ethernet
2283          * type field is already in place.
2284          */
2285         bcopy((char *)evl, (char *)evl + EVL_ENCAPLEN,
2286               ETHER_HDR_LEN - ETHER_TYPE_LEN);
2287         m_adj(m, EVL_ENCAPLEN);
2288 }
2289
2290
2291 static inline void
2292 mxge_rx_done_big(struct mxge_slice_state *ss, uint32_t len, uint32_t csum)
2293 {
2294         mxge_softc_t *sc;
2295         struct ifnet *ifp;
2296         struct mbuf *m;
2297         struct ether_header *eh;
2298         mxge_rx_ring_t *rx;
2299         bus_dmamap_t old_map;
2300         int idx;
2301
2302         sc = ss->sc;
2303         ifp = sc->ifp;
2304         rx = &ss->rx_big;
2305         idx = rx->cnt & rx->mask;
2306         rx->cnt += rx->nbufs;
2307         /* save a pointer to the received mbuf */
2308         m = rx->info[idx].m;
2309         /* try to replace the received mbuf */
2310         if (mxge_get_buf_big(ss, rx->extra_map, idx)) {
2311                 /* drop the frame -- the old mbuf is re-cycled */
2312                 IFNET_STAT_INC(ifp, ierrors, 1);
2313                 return;
2314         }
2315
2316         /* unmap the received buffer */
2317         old_map = rx->info[idx].map;
2318         bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2319         bus_dmamap_unload(rx->dmat, old_map);
2320
2321         /* swap the bus_dmamap_t's */
2322         rx->info[idx].map = rx->extra_map;
2323         rx->extra_map = old_map;
2324
2325         /* mcp implicitly skips 1st 2 bytes so that packet is properly
2326          * aligned */
2327         m->m_data += MXGEFW_PAD;
2328
2329         m->m_pkthdr.rcvif = ifp;
2330         m->m_len = m->m_pkthdr.len = len;
2331         ss->ipackets++;
2332         eh = mtod(m, struct ether_header *);
2333         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2334                 mxge_vlan_tag_remove(m, &csum);
2335         }
2336         /* if the checksum is valid, mark it in the mbuf header */
2337         if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2338             0 == mxge_rx_csum(m, csum)) {
2339                 /* Tell the stack that the checksum is good */
2340                 m->m_pkthdr.csum_data = 0xffff;
2341                 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR |
2342                     CSUM_DATA_VALID;
2343         }
2344 #if 0
2345         /* flowid only valid if RSS hashing is enabled */
2346         if (sc->num_slices > 1) {
2347                 m->m_pkthdr.flowid = (ss - sc->ss);
2348                 m->m_flags |= M_FLOWID;
2349         }
2350 #endif
2351         ifp->if_input(ifp, m);
2352 }
2353
2354 static inline void
2355 mxge_rx_done_small(struct mxge_slice_state *ss, uint32_t len, uint32_t csum)
2356 {
2357         mxge_softc_t *sc;
2358         struct ifnet *ifp;
2359         struct ether_header *eh;
2360         struct mbuf *m;
2361         mxge_rx_ring_t *rx;
2362         bus_dmamap_t old_map;
2363         int idx;
2364
2365         sc = ss->sc;
2366         ifp = sc->ifp;
2367         rx = &ss->rx_small;
2368         idx = rx->cnt & rx->mask;
2369         rx->cnt++;
2370         /* save a pointer to the received mbuf */
2371         m = rx->info[idx].m;
2372         /* try to replace the received mbuf */
2373         if (mxge_get_buf_small(ss, rx->extra_map, idx)) {
2374                 /* drop the frame -- the old mbuf is re-cycled */
2375                 IFNET_STAT_INC(ifp, ierrors, 1);
2376                 return;
2377         }
2378
2379         /* unmap the received buffer */
2380         old_map = rx->info[idx].map;
2381         bus_dmamap_sync(rx->dmat, old_map, BUS_DMASYNC_POSTREAD);
2382         bus_dmamap_unload(rx->dmat, old_map);
2383
2384         /* swap the bus_dmamap_t's */
2385         rx->info[idx].map = rx->extra_map;
2386         rx->extra_map = old_map;
2387
2388         /* mcp implicitly skips 1st 2 bytes so that packet is properly
2389          * aligned */
2390         m->m_data += MXGEFW_PAD;
2391
2392         m->m_pkthdr.rcvif = ifp;
2393         m->m_len = m->m_pkthdr.len = len;
2394         ss->ipackets++;
2395         eh = mtod(m, struct ether_header *);
2396         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2397                 mxge_vlan_tag_remove(m, &csum);
2398         }
2399         /* if the checksum is valid, mark it in the mbuf header */
2400         if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2401             0 == mxge_rx_csum(m, csum)) {
2402                 /* Tell the stack that the checksum is good */
2403                 m->m_pkthdr.csum_data = 0xffff;
2404                 m->m_pkthdr.csum_flags = CSUM_PSEUDO_HDR |
2405                     CSUM_DATA_VALID;
2406         }
2407 #if 0
2408         /* flowid only valid if RSS hashing is enabled */
2409         if (sc->num_slices > 1) {
2410                 m->m_pkthdr.flowid = (ss - sc->ss);
2411                 m->m_flags |= M_FLOWID;
2412         }
2413 #endif
2414         ifp->if_input(ifp, m);
2415 }
2416
2417 static inline void
2418 mxge_clean_rx_done(struct mxge_slice_state *ss)
2419 {
2420         mxge_rx_done_t *rx_done = &ss->rx_done;
2421         int limit = 0;
2422         uint16_t length;
2423         uint16_t checksum;
2424
2425         while (rx_done->entry[rx_done->idx].length != 0) {
2426                 length = ntohs(rx_done->entry[rx_done->idx].length);
2427                 rx_done->entry[rx_done->idx].length = 0;
2428                 checksum = rx_done->entry[rx_done->idx].checksum;
2429                 if (length <= (MHLEN - MXGEFW_PAD))
2430                         mxge_rx_done_small(ss, length, checksum);
2431                 else
2432                         mxge_rx_done_big(ss, length, checksum);
2433                 rx_done->cnt++;
2434                 rx_done->idx = rx_done->cnt & rx_done->mask;
2435
2436                 /* limit potential for livelock */
2437                 if (__predict_false(++limit > rx_done->mask / 2))
2438                         break;
2439         }
2440 }
2441
2442 static inline void
2443 mxge_tx_done(struct mxge_slice_state *ss, uint32_t mcp_idx)
2444 {
2445         struct ifnet *ifp;
2446         mxge_tx_ring_t *tx;
2447         struct mbuf *m;
2448         bus_dmamap_t map;
2449         int idx;
2450
2451         tx = &ss->tx;
2452         ifp = ss->sc->ifp;
2453         ASSERT_SERIALIZED(ifp->if_serializer);
2454         while (tx->pkt_done != mcp_idx) {
2455                 idx = tx->done & tx->mask;
2456                 tx->done++;
2457                 m = tx->info[idx].m;
2458                 /* mbuf and DMA map only attached to the first
2459                    segment per-mbuf */
2460                 if (m != NULL) {
2461                         ss->obytes += m->m_pkthdr.len;
2462                         if (m->m_flags & M_MCAST)
2463                                 ss->omcasts++;
2464                         ss->opackets++;
2465                         tx->info[idx].m = NULL;
2466                         map = tx->info[idx].map;
2467                         bus_dmamap_unload(tx->dmat, map);
2468                         m_freem(m);
2469                 }
2470                 if (tx->info[idx].flag) {
2471                         tx->info[idx].flag = 0;
2472                         tx->pkt_done++;
2473                 }
2474         }
2475
2476         /* If we have space, clear OACTIVE to tell the stack that
2477            its OK to send packets */
2478         if (tx->req - tx->done < (tx->mask + 1)/4)
2479                 ifq_clr_oactive(&ifp->if_snd);
2480
2481         if (!ifq_is_empty(&ifp->if_snd))
2482                 if_devstart(ifp);
2483
2484 #ifdef IFNET_BUF_RING
2485         if ((ss->sc->num_slices > 1) && (tx->req == tx->done)) {
2486                 /* let the NIC stop polling this queue, since there
2487                  * are no more transmits pending */
2488                 if (tx->req == tx->done) {
2489                         *tx->send_stop = 1;
2490                         tx->queue_active = 0;
2491                         tx->deactivate++;
2492                         wmb();
2493                 }
2494         }
2495 #endif
2496 }
2497
2498 static struct mxge_media_type mxge_xfp_media_types[] = {
2499         {IFM_10G_CX4,   0x7f,           "10GBASE-CX4 (module)"},
2500         {IFM_10G_SR,    (1 << 7),       "10GBASE-SR"},
2501         {IFM_10G_LR,    (1 << 6),       "10GBASE-LR"},
2502         {0,             (1 << 5),       "10GBASE-ER"},
2503         {IFM_10G_LRM,   (1 << 4),       "10GBASE-LRM"},
2504         {0,             (1 << 3),       "10GBASE-SW"},
2505         {0,             (1 << 2),       "10GBASE-LW"},
2506         {0,             (1 << 1),       "10GBASE-EW"},
2507         {0,             (1 << 0),       "Reserved"}
2508 };
2509
2510 static struct mxge_media_type mxge_sfp_media_types[] = {
2511         {IFM_10G_TWINAX,      0,        "10GBASE-Twinax"},
2512         {0,             (1 << 7),       "Reserved"},
2513         {IFM_10G_LRM,   (1 << 6),       "10GBASE-LRM"},
2514         {IFM_10G_LR,    (1 << 5),       "10GBASE-LR"},
2515         {IFM_10G_SR,    (1 << 4),       "10GBASE-SR"},
2516         {IFM_10G_TWINAX,(1 << 0),       "10GBASE-Twinax"}
2517 };
2518
2519 static void
2520 mxge_media_set(mxge_softc_t *sc, int media_type)
2521 {
2522         ifmedia_add(&sc->media, IFM_ETHER | IFM_FDX | media_type,
2523                     0, NULL);
2524         ifmedia_set(&sc->media, IFM_ETHER | IFM_FDX | media_type);
2525         sc->current_media = media_type;
2526         sc->media.ifm_media = sc->media.ifm_cur->ifm_media;
2527 }
2528
2529 static void
2530 mxge_media_init(mxge_softc_t *sc)
2531 {
2532         char *ptr;
2533         int i;
2534
2535         ifmedia_removeall(&sc->media);
2536         mxge_media_set(sc, IFM_AUTO);
2537
2538         /*
2539          * parse the product code to deterimine the interface type
2540          * (CX4, XFP, Quad Ribbon Fiber) by looking at the character
2541          * after the 3rd dash in the driver's cached copy of the
2542          * EEPROM's product code string.
2543          */
2544         ptr = sc->product_code_string;
2545         if (ptr == NULL) {
2546                 device_printf(sc->dev, "Missing product code\n");
2547                 return;
2548         }
2549
2550         for (i = 0; i < 3; i++, ptr++) {
2551                 ptr = strchr(ptr, '-');
2552                 if (ptr == NULL) {
2553                         device_printf(sc->dev,
2554                                       "only %d dashes in PC?!?\n", i);
2555                         return;
2556                 }
2557         }
2558         if (*ptr == 'C' || *(ptr +1) == 'C') {
2559                 /* -C is CX4 */
2560                 sc->connector = MXGE_CX4;
2561                 mxge_media_set(sc, IFM_10G_CX4);
2562         } else if (*ptr == 'Q') {
2563                 /* -Q is Quad Ribbon Fiber */
2564                 sc->connector = MXGE_QRF;
2565                 device_printf(sc->dev, "Quad Ribbon Fiber Media\n");
2566                 /* FreeBSD has no media type for Quad ribbon fiber */
2567         } else if (*ptr == 'R') {
2568                 /* -R is XFP */
2569                 sc->connector = MXGE_XFP;
2570         } else if (*ptr == 'S' || *(ptr +1) == 'S') {
2571                 /* -S or -2S is SFP+ */
2572                 sc->connector = MXGE_SFP;
2573         } else {
2574                 device_printf(sc->dev, "Unknown media type: %c\n", *ptr);
2575         }
2576 }
2577
2578 /*
2579  * Determine the media type for a NIC.  Some XFPs will identify
2580  * themselves only when their link is up, so this is initiated via a
2581  * link up interrupt.  However, this can potentially take up to
2582  * several milliseconds, so it is run via the watchdog routine, rather
2583  * than in the interrupt handler itself.
2584  */
2585 static void
2586 mxge_media_probe(mxge_softc_t *sc)
2587 {
2588         mxge_cmd_t cmd;
2589         char *cage_type;
2590
2591         struct mxge_media_type *mxge_media_types = NULL;
2592         int i, err, ms, mxge_media_type_entries;
2593         uint32_t byte;
2594
2595         sc->need_media_probe = 0;
2596
2597         if (sc->connector == MXGE_XFP) {
2598                 /* -R is XFP */
2599                 mxge_media_types = mxge_xfp_media_types;
2600                 mxge_media_type_entries =
2601                         sizeof (mxge_xfp_media_types) /
2602                         sizeof (mxge_xfp_media_types[0]);
2603                 byte = MXGE_XFP_COMPLIANCE_BYTE;
2604                 cage_type = "XFP";
2605         } else  if (sc->connector == MXGE_SFP) {
2606                 /* -S or -2S is SFP+ */
2607                 mxge_media_types = mxge_sfp_media_types;
2608                 mxge_media_type_entries =
2609                         sizeof (mxge_sfp_media_types) /
2610                         sizeof (mxge_sfp_media_types[0]);
2611                 cage_type = "SFP+";
2612                 byte = 3;
2613         } else {
2614                 /* nothing to do; media type cannot change */
2615                 return;
2616         }
2617
2618         /*
2619          * At this point we know the NIC has an XFP cage, so now we
2620          * try to determine what is in the cage by using the
2621          * firmware's XFP I2C commands to read the XFP 10GbE compilance
2622          * register.  We read just one byte, which may take over
2623          * a millisecond
2624          */
2625
2626         cmd.data0 = 0;   /* just fetch 1 byte, not all 256 */
2627         cmd.data1 = byte;
2628         err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_READ, &cmd);
2629         if (err == MXGEFW_CMD_ERROR_I2C_FAILURE) {
2630                 device_printf(sc->dev, "failed to read XFP\n");
2631         }
2632         if (err == MXGEFW_CMD_ERROR_I2C_ABSENT) {
2633                 device_printf(sc->dev, "Type R/S with no XFP!?!?\n");
2634         }
2635         if (err != MXGEFW_CMD_OK) {
2636                 return;
2637         }
2638
2639         /* now we wait for the data to be cached */
2640         cmd.data0 = byte;
2641         err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
2642         for (ms = 0; (err == EBUSY) && (ms < 50); ms++) {
2643                 DELAY(1000);
2644                 cmd.data0 = byte;
2645                 err = mxge_send_cmd(sc, MXGEFW_CMD_I2C_BYTE, &cmd);
2646         }
2647         if (err != MXGEFW_CMD_OK) {
2648                 device_printf(sc->dev, "failed to read %s (%d, %dms)\n",
2649                               cage_type, err, ms);
2650                 return;
2651         }
2652
2653         if (cmd.data0 == mxge_media_types[0].bitmask) {
2654                 if (mxge_verbose)
2655                         device_printf(sc->dev, "%s:%s\n", cage_type,
2656                                       mxge_media_types[0].name);
2657                 if (sc->current_media != mxge_media_types[0].flag) {
2658                         mxge_media_init(sc);
2659                         mxge_media_set(sc, mxge_media_types[0].flag);
2660                 }
2661                 return;
2662         }
2663         for (i = 1; i < mxge_media_type_entries; i++) {
2664                 if (cmd.data0 & mxge_media_types[i].bitmask) {
2665                         if (mxge_verbose)
2666                                 device_printf(sc->dev, "%s:%s\n",
2667                                               cage_type,
2668                                               mxge_media_types[i].name);
2669
2670                         if (sc->current_media != mxge_media_types[i].flag) {
2671                                 mxge_media_init(sc);
2672                                 mxge_media_set(sc, mxge_media_types[i].flag);
2673                         }
2674                         return;
2675                 }
2676         }
2677         if (mxge_verbose)
2678                 device_printf(sc->dev, "%s media 0x%x unknown\n",
2679                               cage_type, cmd.data0);
2680
2681         return;
2682 }
2683
2684 static void
2685 mxge_intr(void *arg)
2686 {
2687         struct mxge_slice_state *ss = arg;
2688         mxge_softc_t *sc = ss->sc;
2689         mcp_irq_data_t *stats = ss->fw_stats;
2690         mxge_tx_ring_t *tx = &ss->tx;
2691         mxge_rx_done_t *rx_done = &ss->rx_done;
2692         uint32_t send_done_count;
2693         uint8_t valid;
2694
2695
2696 #ifndef IFNET_BUF_RING
2697         /* an interrupt on a non-zero slice is implicitly valid
2698            since MSI-X irqs are not shared */
2699         if (ss != sc->ss) {
2700                 mxge_clean_rx_done(ss);
2701                 *ss->irq_claim = be32toh(3);
2702                 return;
2703         }
2704 #endif
2705
2706         /* make sure the DMA has finished */
2707         if (!stats->valid) {
2708                 return;
2709         }
2710         valid = stats->valid;
2711
2712         if (sc->irq_type == PCI_INTR_TYPE_LEGACY) {
2713                 /* lower legacy IRQ  */
2714                 *sc->irq_deassert = 0;
2715                 if (!mxge_deassert_wait)
2716                         /* don't wait for conf. that irq is low */
2717                         stats->valid = 0;
2718         } else {
2719                 stats->valid = 0;
2720         }
2721
2722         /* loop while waiting for legacy irq deassertion */
2723         do {
2724                 /* check for transmit completes and receives */
2725                 send_done_count = be32toh(stats->send_done_count);
2726                 while ((send_done_count != tx->pkt_done) ||
2727                        (rx_done->entry[rx_done->idx].length != 0)) {
2728                         if (send_done_count != tx->pkt_done)
2729                                 mxge_tx_done(ss, (int)send_done_count);
2730                         mxge_clean_rx_done(ss);
2731                         send_done_count = be32toh(stats->send_done_count);
2732                 }
2733                 if (sc->irq_type == PCI_INTR_TYPE_LEGACY && mxge_deassert_wait)
2734                         wmb();
2735         } while (*((volatile uint8_t *) &stats->valid));
2736
2737         /* fw link & error stats meaningful only on the first slice */
2738         if (__predict_false((ss == sc->ss) && stats->stats_updated)) {
2739                 if (sc->link_state != stats->link_up) {
2740                         sc->link_state = stats->link_up;
2741                         if (sc->link_state) {
2742                                 sc->ifp->if_link_state = LINK_STATE_UP;
2743                                 if_link_state_change(sc->ifp);
2744                                 if (mxge_verbose)
2745                                         device_printf(sc->dev, "link up\n");
2746                         } else {
2747                                 sc->ifp->if_link_state = LINK_STATE_DOWN;
2748                                 if_link_state_change(sc->ifp);
2749                                 if (mxge_verbose)
2750                                         device_printf(sc->dev, "link down\n");
2751                         }
2752                         sc->need_media_probe = 1;
2753                 }
2754                 if (sc->rdma_tags_available !=
2755                     be32toh(stats->rdma_tags_available)) {
2756                         sc->rdma_tags_available =
2757                                 be32toh(stats->rdma_tags_available);
2758                         device_printf(sc->dev, "RDMA timed out! %d tags "
2759                                       "left\n", sc->rdma_tags_available);
2760                 }
2761
2762                 if (stats->link_down) {
2763                         sc->down_cnt += stats->link_down;
2764                         sc->link_state = 0;
2765                         sc->ifp->if_link_state = LINK_STATE_DOWN;
2766                         if_link_state_change(sc->ifp);
2767                 }
2768         }
2769
2770         /* check to see if we have rx token to pass back */
2771         if (valid & 0x1)
2772             *ss->irq_claim = be32toh(3);
2773         *(ss->irq_claim + 1) = be32toh(3);
2774 }
2775
2776 static void
2777 mxge_init(void *arg)
2778 {
2779         struct mxge_softc *sc = arg;
2780
2781         ASSERT_SERIALIZED(sc->ifp->if_serializer);
2782         if ((sc->ifp->if_flags & IFF_RUNNING) == 0)
2783                 mxge_open(sc);
2784 }
2785
2786 static void
2787 mxge_free_slice_mbufs(struct mxge_slice_state *ss)
2788 {
2789         int i;
2790
2791         for (i = 0; i <= ss->rx_big.mask; i++) {
2792                 if (ss->rx_big.info[i].m == NULL)
2793                         continue;
2794                 bus_dmamap_unload(ss->rx_big.dmat,
2795                                   ss->rx_big.info[i].map);
2796                 m_freem(ss->rx_big.info[i].m);
2797                 ss->rx_big.info[i].m = NULL;
2798         }
2799
2800         for (i = 0; i <= ss->rx_small.mask; i++) {
2801                 if (ss->rx_small.info[i].m == NULL)
2802                         continue;
2803                 bus_dmamap_unload(ss->rx_small.dmat,
2804                                   ss->rx_small.info[i].map);
2805                 m_freem(ss->rx_small.info[i].m);
2806                 ss->rx_small.info[i].m = NULL;
2807         }
2808
2809         /* transmit ring used only on the first slice */
2810         if (ss->tx.info == NULL)
2811                 return;
2812
2813         for (i = 0; i <= ss->tx.mask; i++) {
2814                 ss->tx.info[i].flag = 0;
2815                 if (ss->tx.info[i].m == NULL)
2816                         continue;
2817                 bus_dmamap_unload(ss->tx.dmat,
2818                                   ss->tx.info[i].map);
2819                 m_freem(ss->tx.info[i].m);
2820                 ss->tx.info[i].m = NULL;
2821         }
2822 }
2823
2824 static void
2825 mxge_free_mbufs(mxge_softc_t *sc)
2826 {
2827         int slice;
2828
2829         for (slice = 0; slice < sc->num_slices; slice++)
2830                 mxge_free_slice_mbufs(&sc->ss[slice]);
2831 }
2832
2833 static void
2834 mxge_free_slice_rings(struct mxge_slice_state *ss)
2835 {
2836         int i;
2837
2838
2839         if (ss->rx_done.entry != NULL)
2840                 mxge_dma_free(&ss->rx_done.dma);
2841         ss->rx_done.entry = NULL;
2842
2843         if (ss->tx.req_bytes != NULL)
2844                 kfree(ss->tx.req_bytes, M_DEVBUF);
2845         ss->tx.req_bytes = NULL;
2846
2847         if (ss->tx.seg_list != NULL)
2848                 kfree(ss->tx.seg_list, M_DEVBUF);
2849         ss->tx.seg_list = NULL;
2850
2851         if (ss->rx_small.shadow != NULL)
2852                 kfree(ss->rx_small.shadow, M_DEVBUF);
2853         ss->rx_small.shadow = NULL;
2854
2855         if (ss->rx_big.shadow != NULL)
2856                 kfree(ss->rx_big.shadow, M_DEVBUF);
2857         ss->rx_big.shadow = NULL;
2858
2859         if (ss->tx.info != NULL) {
2860                 if (ss->tx.dmat != NULL) {
2861                         for (i = 0; i <= ss->tx.mask; i++) {
2862                                 bus_dmamap_destroy(ss->tx.dmat,
2863                                                    ss->tx.info[i].map);
2864                         }
2865                         bus_dma_tag_destroy(ss->tx.dmat);
2866                 }
2867                 kfree(ss->tx.info, M_DEVBUF);
2868         }
2869         ss->tx.info = NULL;
2870
2871         if (ss->rx_small.info != NULL) {
2872                 if (ss->rx_small.dmat != NULL) {
2873                         for (i = 0; i <= ss->rx_small.mask; i++) {
2874                                 bus_dmamap_destroy(ss->rx_small.dmat,
2875                                                    ss->rx_small.info[i].map);
2876                         }
2877                         bus_dmamap_destroy(ss->rx_small.dmat,
2878                                            ss->rx_small.extra_map);
2879                         bus_dma_tag_destroy(ss->rx_small.dmat);
2880                 }
2881                 kfree(ss->rx_small.info, M_DEVBUF);
2882         }
2883         ss->rx_small.info = NULL;
2884
2885         if (ss->rx_big.info != NULL) {
2886                 if (ss->rx_big.dmat != NULL) {
2887                         for (i = 0; i <= ss->rx_big.mask; i++) {
2888                                 bus_dmamap_destroy(ss->rx_big.dmat,
2889                                                    ss->rx_big.info[i].map);
2890                         }
2891                         bus_dmamap_destroy(ss->rx_big.dmat,
2892                                            ss->rx_big.extra_map);
2893                         bus_dma_tag_destroy(ss->rx_big.dmat);
2894                 }
2895                 kfree(ss->rx_big.info, M_DEVBUF);
2896         }
2897         ss->rx_big.info = NULL;
2898 }
2899
2900 static void
2901 mxge_free_rings(mxge_softc_t *sc)
2902 {
2903         int slice;
2904
2905         for (slice = 0; slice < sc->num_slices; slice++)
2906                 mxge_free_slice_rings(&sc->ss[slice]);
2907 }
2908
2909 static int
2910 mxge_alloc_slice_rings(struct mxge_slice_state *ss, int rx_ring_entries,
2911     int tx_ring_entries)
2912 {
2913         mxge_softc_t *sc = ss->sc;
2914         size_t bytes;
2915         int err, i;
2916
2917         /* allocate per-slice receive resources */
2918
2919         ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
2920         ss->rx_done.mask = (2 * rx_ring_entries) - 1;
2921
2922         /* allocate the rx shadow rings */
2923         bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
2924         ss->rx_small.shadow = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2925
2926         bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
2927         ss->rx_big.shadow = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2928
2929         /* allocate the rx host info rings */
2930         bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
2931         ss->rx_small.info = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2932
2933         bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
2934         ss->rx_big.info = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
2935
2936         /* allocate the rx busdma resources */
2937         err = bus_dma_tag_create(sc->parent_dmat,       /* parent */
2938                                  1,                     /* alignment */
2939                                  4096,                  /* boundary */
2940                                  BUS_SPACE_MAXADDR,     /* low */
2941                                  BUS_SPACE_MAXADDR,     /* high */
2942                                  NULL, NULL,            /* filter */
2943                                  MHLEN,                 /* maxsize */
2944                                  1,                     /* num segs */
2945                                  MHLEN,                 /* maxsegsize */
2946                                  BUS_DMA_ALLOCNOW,      /* flags */
2947                                  &ss->rx_small.dmat);   /* tag */
2948         if (err != 0) {
2949                 device_printf(sc->dev, "Err %d allocating rx_small dmat\n",
2950                               err);
2951                 return err;
2952         }
2953
2954         err = bus_dma_tag_create(sc->parent_dmat,       /* parent */
2955                                  1,                     /* alignment */
2956 #if MXGE_VIRT_JUMBOS
2957                                  4096,                  /* boundary */
2958 #else
2959                                  0,                     /* boundary */
2960 #endif
2961                                  BUS_SPACE_MAXADDR,     /* low */
2962                                  BUS_SPACE_MAXADDR,     /* high */
2963                                  NULL, NULL,            /* filter */
2964                                  3*4096,                /* maxsize */
2965 #if MXGE_VIRT_JUMBOS
2966                                  3,                     /* num segs */
2967                                  4096,                  /* maxsegsize*/
2968 #else
2969                                  1,                     /* num segs */
2970                                  MJUM9BYTES,            /* maxsegsize*/
2971 #endif
2972                                  BUS_DMA_ALLOCNOW,      /* flags */
2973                                  &ss->rx_big.dmat);     /* tag */
2974         if (err != 0) {
2975                 device_printf(sc->dev, "Err %d allocating rx_big dmat\n",
2976                               err);
2977                 return err;
2978         }
2979         for (i = 0; i <= ss->rx_small.mask; i++) {
2980                 err = bus_dmamap_create(ss->rx_small.dmat, 0,
2981                                         &ss->rx_small.info[i].map);
2982                 if (err != 0) {
2983                         device_printf(sc->dev, "Err %d  rx_small dmamap\n",
2984                                       err);
2985                         return err;
2986                 }
2987         }
2988         err = bus_dmamap_create(ss->rx_small.dmat, 0,
2989                                 &ss->rx_small.extra_map);
2990         if (err != 0) {
2991                 device_printf(sc->dev, "Err %d extra rx_small dmamap\n",
2992                               err);
2993                 return err;
2994         }
2995
2996         for (i = 0; i <= ss->rx_big.mask; i++) {
2997                 err = bus_dmamap_create(ss->rx_big.dmat, 0,
2998                                         &ss->rx_big.info[i].map);
2999                 if (err != 0) {
3000                         device_printf(sc->dev, "Err %d  rx_big dmamap\n",
3001                                       err);
3002                         return err;
3003                 }
3004         }
3005         err = bus_dmamap_create(ss->rx_big.dmat, 0,
3006                                 &ss->rx_big.extra_map);
3007         if (err != 0) {
3008                 device_printf(sc->dev, "Err %d extra rx_big dmamap\n",
3009                               err);
3010                 return err;
3011         }
3012
3013         /* now allocate TX resources */
3014
3015 #ifndef IFNET_BUF_RING
3016         /* only use a single TX ring for now */
3017         if (ss != ss->sc->ss)
3018                 return 0;
3019 #endif
3020
3021         ss->tx.mask = tx_ring_entries - 1;
3022         ss->tx.max_desc = MIN(MXGE_MAX_SEND_DESC, tx_ring_entries / 4);
3023
3024
3025         /* allocate the tx request copy block */
3026         bytes = 8 +
3027                 sizeof (*ss->tx.req_list) * (ss->tx.max_desc + 4);
3028         ss->tx.req_bytes = kmalloc(bytes, M_DEVBUF, M_WAITOK);
3029         /* ensure req_list entries are aligned to 8 bytes */
3030         ss->tx.req_list = (mcp_kreq_ether_send_t *)
3031                 ((unsigned long)(ss->tx.req_bytes + 7) & ~7UL);
3032
3033         /* allocate the tx busdma segment list */
3034         bytes = sizeof (*ss->tx.seg_list) * ss->tx.max_desc;
3035         ss->tx.seg_list = (bus_dma_segment_t *)
3036                 kmalloc(bytes, M_DEVBUF, M_WAITOK);
3037
3038         /* allocate the tx host info ring */
3039         bytes = tx_ring_entries * sizeof (*ss->tx.info);
3040         ss->tx.info = kmalloc(bytes, M_DEVBUF, M_ZERO|M_WAITOK);
3041
3042         /* allocate the tx busdma resources */
3043         err = bus_dma_tag_create(sc->parent_dmat,       /* parent */
3044                                  1,                     /* alignment */
3045                                  sc->tx_boundary,       /* boundary */
3046                                  BUS_SPACE_MAXADDR,     /* low */
3047                                  BUS_SPACE_MAXADDR,     /* high */
3048                                  NULL, NULL,            /* filter */
3049                                  65536 + 256,           /* maxsize */
3050                                  ss->tx.max_desc - 2,   /* num segs */
3051                                  sc->tx_boundary,       /* maxsegsz */
3052                                  BUS_DMA_ALLOCNOW,      /* flags */
3053                                  &ss->tx.dmat);         /* tag */
3054
3055         if (err != 0) {
3056                 device_printf(sc->dev, "Err %d allocating tx dmat\n",
3057                               err);
3058                 return err;
3059         }
3060
3061         /* now use these tags to setup dmamaps for each slot
3062            in the ring */
3063         for (i = 0; i <= ss->tx.mask; i++) {
3064                 err = bus_dmamap_create(ss->tx.dmat, 0,
3065                                         &ss->tx.info[i].map);
3066                 if (err != 0) {
3067                         device_printf(sc->dev, "Err %d  tx dmamap\n",
3068                                       err);
3069                         return err;
3070                 }
3071         }
3072         return 0;
3073
3074 }
3075
3076 static int
3077 mxge_alloc_rings(mxge_softc_t *sc)
3078 {
3079         mxge_cmd_t cmd;
3080         int tx_ring_size;
3081         int tx_ring_entries, rx_ring_entries;
3082         int err, slice;
3083
3084         /* get ring sizes */
3085         err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
3086         tx_ring_size = cmd.data0;
3087         if (err != 0) {
3088                 device_printf(sc->dev, "Cannot determine tx ring sizes\n");
3089                 goto abort;
3090         }
3091
3092         tx_ring_entries = tx_ring_size / sizeof (mcp_kreq_ether_send_t);
3093         rx_ring_entries = sc->rx_ring_size / sizeof (mcp_dma_addr_t);
3094         ifq_set_maxlen(&sc->ifp->if_snd, tx_ring_entries - 1);
3095         ifq_set_ready(&sc->ifp->if_snd);
3096
3097         for (slice = 0; slice < sc->num_slices; slice++) {
3098                 err = mxge_alloc_slice_rings(&sc->ss[slice],
3099                                              rx_ring_entries,
3100                                              tx_ring_entries);
3101                 if (err != 0)
3102                         goto abort;
3103         }
3104         return 0;
3105
3106 abort:
3107         mxge_free_rings(sc);
3108         return err;
3109
3110 }
3111
3112 static void
3113 mxge_choose_params(int mtu, int *big_buf_size, int *cl_size, int *nbufs)
3114 {
3115         int bufsize = mtu + ETHER_HDR_LEN + EVL_ENCAPLEN + MXGEFW_PAD;
3116
3117         if (bufsize < MCLBYTES) {
3118                 /* easy, everything fits in a single buffer */
3119                 *big_buf_size = MCLBYTES;
3120                 *cl_size = MCLBYTES;
3121                 *nbufs = 1;
3122                 return;
3123         }
3124
3125         if (bufsize < MJUMPAGESIZE) {
3126                 /* still easy, everything still fits in a single buffer */
3127                 *big_buf_size = MJUMPAGESIZE;
3128                 *cl_size = MJUMPAGESIZE;
3129                 *nbufs = 1;
3130                 return;
3131         }
3132 #if MXGE_VIRT_JUMBOS
3133         /* now we need to use virtually contiguous buffers */
3134         *cl_size = MJUM9BYTES;
3135         *big_buf_size = 4096;
3136         *nbufs = mtu / 4096 + 1;
3137         /* needs to be a power of two, so round up */
3138         if (*nbufs == 3)
3139                 *nbufs = 4;
3140 #else
3141         *cl_size = MJUM9BYTES;
3142         *big_buf_size = MJUM9BYTES;
3143         *nbufs = 1;
3144 #endif
3145 }
3146
3147 static int
3148 mxge_slice_open(struct mxge_slice_state *ss, int nbufs, int cl_size)
3149 {
3150         mxge_softc_t *sc;
3151         mxge_cmd_t cmd;
3152         bus_dmamap_t map;
3153         int err, i, slice;
3154
3155         sc = ss->sc;
3156         slice = ss - sc->ss;
3157
3158         /* get the lanai pointers to the send and receive rings */
3159
3160         err = 0;
3161 #ifndef IFNET_BUF_RING
3162         /* We currently only send from the first slice */
3163         if (slice == 0) {
3164 #endif
3165                 cmd.data0 = slice;
3166                 err = mxge_send_cmd(sc, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
3167                 ss->tx.lanai =
3168                         (volatile mcp_kreq_ether_send_t *)(sc->sram + cmd.data0);
3169                 ss->tx.send_go = (volatile uint32_t *)
3170                         (sc->sram + MXGEFW_ETH_SEND_GO + 64 * slice);
3171                 ss->tx.send_stop = (volatile uint32_t *)
3172                 (sc->sram + MXGEFW_ETH_SEND_STOP + 64 * slice);
3173 #ifndef IFNET_BUF_RING
3174         }
3175 #endif
3176         cmd.data0 = slice;
3177         err |= mxge_send_cmd(sc,
3178                              MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
3179         ss->rx_small.lanai =
3180                 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
3181         cmd.data0 = slice;
3182         err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
3183         ss->rx_big.lanai =
3184                 (volatile mcp_kreq_ether_recv_t *)(sc->sram + cmd.data0);
3185
3186         if (err != 0) {
3187                 device_printf(sc->dev,
3188                               "failed to get ring sizes or locations\n");
3189                 return EIO;
3190         }
3191
3192         /* stock receive rings */
3193         for (i = 0; i <= ss->rx_small.mask; i++) {
3194                 map = ss->rx_small.info[i].map;
3195                 err = mxge_get_buf_small(ss, map, i);
3196                 if (err) {
3197                         device_printf(sc->dev, "alloced %d/%d smalls\n",
3198                                       i, ss->rx_small.mask + 1);
3199                         return ENOMEM;
3200                 }
3201         }
3202         for (i = 0; i <= ss->rx_big.mask; i++) {
3203                 ss->rx_big.shadow[i].addr_low = 0xffffffff;
3204                 ss->rx_big.shadow[i].addr_high = 0xffffffff;
3205         }
3206         ss->rx_big.nbufs = nbufs;
3207         ss->rx_big.cl_size = cl_size;
3208         ss->rx_big.mlen = ss->sc->ifp->if_mtu + ETHER_HDR_LEN +
3209                 EVL_ENCAPLEN + MXGEFW_PAD;
3210         for (i = 0; i <= ss->rx_big.mask; i += ss->rx_big.nbufs) {
3211                 map = ss->rx_big.info[i].map;
3212                 err = mxge_get_buf_big(ss, map, i);
3213                 if (err) {
3214                         device_printf(sc->dev, "alloced %d/%d bigs\n",
3215                                       i, ss->rx_big.mask + 1);
3216                         return ENOMEM;
3217                 }
3218         }
3219         return 0;
3220 }
3221
3222 static int
3223 mxge_open(mxge_softc_t *sc)
3224 {
3225         mxge_cmd_t cmd;
3226         int err, big_bytes, nbufs, slice, cl_size, i;
3227         bus_addr_t bus;
3228         volatile uint8_t *itable;
3229         struct mxge_slice_state *ss;
3230
3231         ASSERT_SERIALIZED(sc->ifp->if_serializer);
3232         /* Copy the MAC address in case it was overridden */
3233         bcopy(IF_LLADDR(sc->ifp), sc->mac_addr, ETHER_ADDR_LEN);
3234
3235         err = mxge_reset(sc, 1);
3236         if (err != 0) {
3237                 device_printf(sc->dev, "failed to reset\n");
3238                 return EIO;
3239         }
3240
3241         if (sc->num_slices > 1) {
3242                 /* setup the indirection table */
3243                 cmd.data0 = sc->num_slices;
3244                 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
3245                                     &cmd);
3246
3247                 err |= mxge_send_cmd(sc, MXGEFW_CMD_GET_RSS_TABLE_OFFSET,
3248                                      &cmd);
3249                 if (err != 0) {
3250                         device_printf(sc->dev,
3251                                       "failed to setup rss tables\n");
3252                         return err;
3253                 }
3254
3255                 /* just enable an identity mapping */
3256                 itable = sc->sram + cmd.data0;
3257                 for (i = 0; i < sc->num_slices; i++)
3258                         itable[i] = (uint8_t)i;
3259
3260                 cmd.data0 = 1;
3261                 cmd.data1 = mxge_rss_hash_type;
3262                 err = mxge_send_cmd(sc, MXGEFW_CMD_SET_RSS_ENABLE, &cmd);
3263                 if (err != 0) {
3264                         device_printf(sc->dev, "failed to enable slices\n");
3265                         return err;
3266                 }
3267         }
3268
3269         cmd.data0 = MXGEFW_TSO_MODE_NDIS;
3270         err = mxge_send_cmd(sc, MXGEFW_CMD_SET_TSO_MODE, &cmd);
3271         if (err) {
3272                 device_printf(sc->dev, "failed set TSO mode\n");
3273                 sc->ifp->if_capenable &= ~IFCAP_TSO;
3274                 sc->ifp->if_capabilities &= ~IFCAP_TSO;
3275                 sc->ifp->if_hwassist &= ~CSUM_TSO;
3276         }
3277
3278         mxge_choose_params(sc->ifp->if_mtu, &big_bytes, &cl_size, &nbufs);
3279
3280         cmd.data0 = nbufs;
3281         err = mxge_send_cmd(sc, MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS,
3282                             &cmd);
3283         /* error is only meaningful if we're trying to set
3284            MXGEFW_CMD_ALWAYS_USE_N_BIG_BUFFERS > 1 */
3285         if (err && nbufs > 1) {
3286                 device_printf(sc->dev,
3287                               "Failed to set alway-use-n to %d\n",
3288                               nbufs);
3289                 return EIO;
3290         }
3291         /* Give the firmware the mtu and the big and small buffer
3292            sizes.  The firmware wants the big buf size to be a power
3293            of two. Luckily, FreeBSD's clusters are powers of two */
3294         cmd.data0 = sc->ifp->if_mtu + ETHER_HDR_LEN + EVL_ENCAPLEN;
3295         err = mxge_send_cmd(sc, MXGEFW_CMD_SET_MTU, &cmd);
3296         cmd.data0 = MHLEN - MXGEFW_PAD;
3297         err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE,
3298                              &cmd);
3299         cmd.data0 = big_bytes;
3300         err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
3301
3302         if (err != 0) {
3303                 device_printf(sc->dev, "failed to setup params\n");
3304                 goto abort;
3305         }
3306
3307         /* Now give him the pointer to the stats block */
3308         for (slice = 0;
3309 #ifdef IFNET_BUF_RING
3310              slice < sc->num_slices;
3311 #else
3312              slice < 1;
3313 #endif
3314              slice++) {
3315                 ss = &sc->ss[slice];
3316                 cmd.data0 =
3317                         MXGE_LOWPART_TO_U32(ss->fw_stats_dma.bus_addr);
3318                 cmd.data1 =
3319                         MXGE_HIGHPART_TO_U32(ss->fw_stats_dma.bus_addr);
3320                 cmd.data2 = sizeof(struct mcp_irq_data);
3321                 cmd.data2 |= (slice << 16);
3322                 err |= mxge_send_cmd(sc, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
3323         }
3324
3325         if (err != 0) {
3326                 bus = sc->ss->fw_stats_dma.bus_addr;
3327                 bus += offsetof(struct mcp_irq_data, send_done_count);
3328                 cmd.data0 = MXGE_LOWPART_TO_U32(bus);
3329                 cmd.data1 = MXGE_HIGHPART_TO_U32(bus);
3330                 err = mxge_send_cmd(sc,
3331                                     MXGEFW_CMD_SET_STATS_DMA_OBSOLETE,
3332                                     &cmd);
3333                 /* Firmware cannot support multicast without STATS_DMA_V2 */
3334                 sc->fw_multicast_support = 0;
3335         } else {
3336                 sc->fw_multicast_support = 1;
3337         }
3338
3339         if (err != 0) {
3340                 device_printf(sc->dev, "failed to setup params\n");
3341                 goto abort;
3342         }
3343
3344         for (slice = 0; slice < sc->num_slices; slice++) {
3345                 err = mxge_slice_open(&sc->ss[slice], nbufs, cl_size);
3346                 if (err != 0) {
3347                         device_printf(sc->dev, "couldn't open slice %d\n",
3348                                       slice);
3349                         goto abort;
3350                 }
3351         }
3352
3353         /* Finally, start the firmware running */
3354         err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_UP, &cmd);
3355         if (err) {
3356                 device_printf(sc->dev, "Couldn't bring up link\n");
3357                 goto abort;
3358         }
3359         sc->ifp->if_flags |= IFF_RUNNING;
3360         ifq_clr_oactive(&sc->ifp->if_snd);
3361
3362         return 0;
3363
3364
3365 abort:
3366         mxge_free_mbufs(sc);
3367
3368         return err;
3369 }
3370
3371 static int
3372 mxge_close(mxge_softc_t *sc, int down)
3373 {
3374         mxge_cmd_t cmd;
3375         int err, old_down_cnt;
3376
3377         ASSERT_SERIALIZED(sc->ifp->if_serializer);
3378
3379         sc->ifp->if_flags &= ~IFF_RUNNING;
3380         if (!down) {
3381                 old_down_cnt = sc->down_cnt;
3382                 wmb();
3383                 err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
3384                 if (err) {
3385                         device_printf(sc->dev,
3386                                       "Couldn't bring down link\n");
3387                 }
3388                 if (old_down_cnt == sc->down_cnt) {
3389                         /* wait for down irq */
3390                         lwkt_serialize_exit(sc->ifp->if_serializer);
3391                         DELAY(10 * sc->intr_coal_delay);
3392                         lwkt_serialize_enter(sc->ifp->if_serializer);
3393                 }
3394                 wmb();
3395                 if (old_down_cnt == sc->down_cnt) {
3396                         device_printf(sc->dev, "never got down irq\n");
3397                 }
3398         }
3399         mxge_free_mbufs(sc);
3400
3401         return 0;
3402 }
3403
3404 static void
3405 mxge_setup_cfg_space(mxge_softc_t *sc)
3406 {
3407         device_t dev = sc->dev;
3408         int reg;
3409         uint16_t lnk, pectl;
3410
3411         /* find the PCIe link width and set max read request to 4KB*/
3412         if (pci_find_extcap(dev, PCIY_EXPRESS, &reg) == 0) {
3413                 lnk = pci_read_config(dev, reg + 0x12, 2);
3414                 sc->link_width = (lnk >> 4) & 0x3f;
3415
3416                 if (sc->pectl == 0) {
3417                         pectl = pci_read_config(dev, reg + 0x8, 2);
3418                         pectl = (pectl & ~0x7000) | (5 << 12);
3419                         pci_write_config(dev, reg + 0x8, pectl, 2);
3420                         sc->pectl = pectl;
3421                 } else {
3422                         /* restore saved pectl after watchdog reset */
3423                         pci_write_config(dev, reg + 0x8, sc->pectl, 2);
3424                 }
3425         }
3426
3427         /* Enable DMA and Memory space access */
3428         pci_enable_busmaster(dev);
3429 }
3430
3431 static uint32_t
3432 mxge_read_reboot(mxge_softc_t *sc)
3433 {
3434         device_t dev = sc->dev;
3435         uint32_t vs;
3436
3437         /* find the vendor specific offset */
3438         if (pci_find_extcap(dev, PCIY_VENDOR, &vs) != 0) {
3439                 device_printf(sc->dev,
3440                               "could not find vendor specific offset\n");
3441                 return (uint32_t)-1;
3442         }
3443         /* enable read32 mode */
3444         pci_write_config(dev, vs + 0x10, 0x3, 1);
3445         /* tell NIC which register to read */
3446         pci_write_config(dev, vs + 0x18, 0xfffffff0, 4);
3447         return (pci_read_config(dev, vs + 0x14, 4));
3448 }
3449
3450 static void
3451 mxge_watchdog_reset(mxge_softc_t *sc)
3452 {
3453         struct pci_devinfo *dinfo;
3454         int err, running;
3455         uint32_t reboot;
3456         uint16_t cmd;
3457
3458         err = ENXIO;
3459
3460         device_printf(sc->dev, "Watchdog reset!\n");
3461
3462         /*
3463          * check to see if the NIC rebooted.  If it did, then all of
3464          * PCI config space has been reset, and things like the
3465          * busmaster bit will be zero.  If this is the case, then we
3466          * must restore PCI config space before the NIC can be used
3467          * again
3468          */
3469         cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3470         if (cmd == 0xffff) {
3471                 /*
3472                  * maybe the watchdog caught the NIC rebooting; wait
3473                  * up to 100ms for it to finish.  If it does not come
3474                  * back, then give up
3475                  */
3476                 DELAY(1000*100);
3477                 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3478                 if (cmd == 0xffff) {
3479                         device_printf(sc->dev, "NIC disappeared!\n");
3480                 }
3481         }
3482         if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
3483                 /* print the reboot status */
3484                 reboot = mxge_read_reboot(sc);
3485                 device_printf(sc->dev, "NIC rebooted, status = 0x%x\n",
3486                               reboot);
3487                 running = sc->ifp->if_flags & IFF_RUNNING;
3488                 if (running) {
3489
3490                         /*
3491                          * quiesce NIC so that TX routines will not try to
3492                          * xmit after restoration of BAR
3493                          */
3494
3495                         /* Mark the link as down */
3496                         if (sc->link_state) {
3497                                 sc->ifp->if_link_state = LINK_STATE_DOWN;
3498                                 if_link_state_change(sc->ifp);
3499                         }
3500                         mxge_close(sc, 1);
3501                 }
3502                 /* restore PCI configuration space */
3503                 dinfo = device_get_ivars(sc->dev);
3504                 pci_cfg_restore(sc->dev, dinfo);
3505
3506                 /* and redo any changes we made to our config space */
3507                 mxge_setup_cfg_space(sc);
3508
3509                 /* reload f/w */
3510                 err = mxge_load_firmware(sc, 0);
3511                 if (err) {
3512                         device_printf(sc->dev,
3513                                       "Unable to re-load f/w\n");
3514                 }
3515                 if (running) {
3516                         if (!err) {
3517                                 err = mxge_open(sc);
3518                                 if_devstart_sched(sc->ifp);
3519                         }
3520                 }
3521                 sc->watchdog_resets++;
3522         } else {
3523                 device_printf(sc->dev,
3524                               "NIC did not reboot, not resetting\n");
3525                 err = 0;
3526         }
3527         if (err) {
3528                 device_printf(sc->dev, "watchdog reset failed\n");
3529         } else {
3530                 if (sc->dying == 2)
3531                         sc->dying = 0;
3532                 callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
3533         }
3534 }
3535
3536 static void
3537 mxge_warn_stuck(mxge_softc_t *sc, mxge_tx_ring_t *tx, int slice)
3538 {
3539         tx = &sc->ss[slice].tx;
3540         device_printf(sc->dev, "slice %d struck? ring state:\n", slice);
3541         device_printf(sc->dev,
3542                       "tx.req=%d tx.done=%d, tx.queue_active=%d\n",
3543                       tx->req, tx->done, tx->queue_active);
3544         device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n",
3545                               tx->activate, tx->deactivate);
3546         device_printf(sc->dev, "pkt_done=%d fw=%d\n",
3547                       tx->pkt_done,
3548                       be32toh(sc->ss->fw_stats->send_done_count));
3549 }
3550
3551 static int
3552 mxge_watchdog(mxge_softc_t *sc)
3553 {
3554         mxge_tx_ring_t *tx;
3555         uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause);
3556         int i, err = 0;
3557
3558         /* see if we have outstanding transmits, which
3559            have been pending for more than mxge_ticks */
3560         for (i = 0;
3561 #ifdef IFNET_BUF_RING
3562              (i < sc->num_slices) && (err == 0);
3563 #else
3564              (i < 1) && (err == 0);
3565 #endif
3566              i++) {
3567                 tx = &sc->ss[i].tx;
3568                 if (tx->req != tx->done &&
3569                     tx->watchdog_req != tx->watchdog_done &&
3570                     tx->done == tx->watchdog_done) {
3571                         /* check for pause blocking before resetting */
3572                         if (tx->watchdog_rx_pause == rx_pause) {
3573                                 mxge_warn_stuck(sc, tx, i);
3574                                 mxge_watchdog_reset(sc);
3575                                 return (ENXIO);
3576                         }
3577                         else
3578                                 device_printf(sc->dev, "Flow control blocking "
3579                                               "xmits, check link partner\n");
3580                 }
3581
3582                 tx->watchdog_req = tx->req;
3583                 tx->watchdog_done = tx->done;
3584                 tx->watchdog_rx_pause = rx_pause;
3585         }
3586
3587         if (sc->need_media_probe)
3588                 mxge_media_probe(sc);
3589         return (err);
3590 }
3591
3592 static u_long
3593 mxge_update_stats(mxge_softc_t *sc)
3594 {
3595         struct mxge_slice_state *ss;
3596         u_long pkts = 0;
3597         u_long ipackets = 0, old_ipackets;
3598         u_long opackets = 0, old_opackets;
3599 #ifdef IFNET_BUF_RING
3600         u_long obytes = 0;
3601         u_long omcasts = 0;
3602         u_long odrops = 0;
3603 #endif
3604         u_long oerrors = 0;
3605         int slice;
3606
3607         for (slice = 0; slice < sc->num_slices; slice++) {
3608                 ss = &sc->ss[slice];
3609                 ipackets += ss->ipackets;
3610                 opackets += ss->opackets;
3611 #ifdef IFNET_BUF_RING
3612                 obytes += ss->obytes;
3613                 omcasts += ss->omcasts;
3614                 odrops += ss->tx.br->br_drops;
3615 #endif
3616                 oerrors += ss->oerrors;
3617         }
3618         IFNET_STAT_GET(sc->ifp, ipackets, old_ipackets);
3619         IFNET_STAT_GET(sc->ifp, opackets, old_opackets);
3620
3621         pkts = ipackets - old_ipackets;
3622         pkts += opackets - old_opackets;
3623
3624         IFNET_STAT_SET(sc->ifp, ipackets, ipackets);
3625         IFNET_STAT_SET(sc->ifp, opackets, opackets);
3626 #ifdef IFNET_BUF_RING
3627         sc->ifp->if_obytes = obytes;
3628         sc->ifp->if_omcasts = omcasts;
3629         sc->ifp->if_snd.ifq_drops = odrops;
3630 #endif
3631         IFNET_STAT_SET(sc->ifp, oerrors, oerrors);
3632         return pkts;
3633 }
3634
3635 static void
3636 mxge_tick(void *arg)
3637 {
3638         mxge_softc_t *sc = arg;
3639         u_long pkts = 0;
3640         int err = 0;
3641         int running, ticks;
3642         uint16_t cmd;
3643
3644         lwkt_serialize_enter(sc->ifp->if_serializer);
3645
3646         ticks = mxge_ticks;
3647         running = sc->ifp->if_flags & IFF_RUNNING;
3648         if (running) {
3649                 /* aggregate stats from different slices */
3650                 pkts = mxge_update_stats(sc);
3651                 if (!sc->watchdog_countdown) {
3652                         err = mxge_watchdog(sc);
3653                         sc->watchdog_countdown = 4;
3654                 }
3655                 sc->watchdog_countdown--;
3656         }
3657         if (pkts == 0) {
3658                 /* ensure NIC did not suffer h/w fault while idle */
3659                 cmd = pci_read_config(sc->dev, PCIR_COMMAND, 2);
3660                 if ((cmd & PCIM_CMD_BUSMASTEREN) == 0) {
3661                         sc->dying = 2;
3662                         mxge_watchdog_reset(sc);
3663                         err = ENXIO;
3664                 }
3665                 /* look less often if NIC is idle */
3666                 ticks *= 4;
3667         }
3668
3669         if (err == 0)
3670                 callout_reset(&sc->co_hdl, ticks, mxge_tick, sc);
3671
3672         lwkt_serialize_exit(sc->ifp->if_serializer);
3673 }
3674
3675 static int
3676 mxge_media_change(struct ifnet *ifp)
3677 {
3678         return EINVAL;
3679 }
3680
3681 static int
3682 mxge_change_mtu(mxge_softc_t *sc, int mtu)
3683 {
3684         struct ifnet *ifp = sc->ifp;
3685         int real_mtu, old_mtu;
3686         int err = 0;
3687
3688         if (ifp->if_serializer)
3689                 ASSERT_SERIALIZED(ifp->if_serializer);
3690
3691         real_mtu = mtu + ETHER_HDR_LEN + EVL_ENCAPLEN;
3692         if ((real_mtu > sc->max_mtu) || real_mtu < 60)
3693                 return EINVAL;
3694         old_mtu = ifp->if_mtu;
3695         ifp->if_mtu = mtu;
3696         if (ifp->if_flags & IFF_RUNNING) {
3697                 mxge_close(sc, 0);
3698                 err = mxge_open(sc);
3699                 if (err != 0) {
3700                         ifp->if_mtu = old_mtu;
3701                         mxge_close(sc, 0);
3702                         (void) mxge_open(sc);
3703                 }
3704         }
3705         return err;
3706 }
3707
3708 static void
3709 mxge_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
3710 {
3711         mxge_softc_t *sc = ifp->if_softc;
3712
3713
3714         if (sc == NULL)
3715                 return;
3716         ifmr->ifm_status = IFM_AVALID;
3717         ifmr->ifm_active = IFM_ETHER | IFM_FDX;
3718         ifmr->ifm_status |= sc->link_state ? IFM_ACTIVE : 0;
3719         ifmr->ifm_active |= sc->current_media;
3720 }
3721
3722 static int
3723 mxge_ioctl(struct ifnet *ifp, u_long command, caddr_t data,
3724     struct ucred *cr __unused)
3725 {
3726         mxge_softc_t *sc = ifp->if_softc;
3727         struct ifreq *ifr = (struct ifreq *)data;
3728         int err, mask;
3729
3730         err = 0;
3731         ASSERT_SERIALIZED(ifp->if_serializer);
3732         switch (command) {
3733         case SIOCSIFMTU:
3734                 err = mxge_change_mtu(sc, ifr->ifr_mtu);
3735                 break;
3736
3737         case SIOCSIFFLAGS:
3738                 if (sc->dying) {
3739                         return EINVAL;
3740                 }
3741                 if (ifp->if_flags & IFF_UP) {
3742                         if (!(ifp->if_flags & IFF_RUNNING)) {
3743                                 err = mxge_open(sc);
3744                         } else {
3745                                 /* take care of promis can allmulti
3746                                    flag chages */
3747                                 mxge_change_promisc(sc,
3748                                                     ifp->if_flags & IFF_PROMISC);
3749                                 mxge_set_multicast_list(sc);
3750                         }
3751                 } else {
3752                         if (ifp->if_flags & IFF_RUNNING) {
3753                                 mxge_close(sc, 0);
3754                         }
3755                 }
3756                 break;
3757
3758         case SIOCADDMULTI:
3759         case SIOCDELMULTI:
3760                 mxge_set_multicast_list(sc);
3761                 break;
3762
3763         case SIOCSIFCAP:
3764                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3765                 if (mask & IFCAP_TXCSUM) {
3766                         ifp->if_capenable ^= IFCAP_TXCSUM;
3767                         if (ifp->if_capenable & IFCAP_TXCSUM)
3768                                 ifp->if_hwassist |= CSUM_TCP | CSUM_UDP;
3769                         else
3770                                 ifp->if_hwassist &= ~(CSUM_TCP | CSUM_UDP);
3771                 }
3772                 if (mask & IFCAP_TSO) {
3773                         ifp->if_capenable ^= IFCAP_TSO;
3774                         if (ifp->if_capenable & IFCAP_TSO)
3775                                 ifp->if_hwassist |= CSUM_TSO;
3776                         else
3777                                 ifp->if_hwassist &= ~CSUM_TSO;
3778                 }
3779                 if (mask & IFCAP_RXCSUM)
3780                         ifp->if_capenable ^= IFCAP_RXCSUM;
3781                 if (mask & IFCAP_VLAN_HWTAGGING)
3782                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3783                 break;
3784
3785         case SIOCGIFMEDIA:
3786                 mxge_media_probe(sc);
3787                 err = ifmedia_ioctl(ifp, (struct ifreq *)data,
3788                                     &sc->media, command);
3789                 break;
3790
3791         default:
3792                 err = ether_ioctl(ifp, command, data);
3793                 break;
3794         }
3795         return err;
3796 }
3797
3798 static void
3799 mxge_fetch_tunables(mxge_softc_t *sc)
3800 {
3801
3802         TUNABLE_INT_FETCH("hw.mxge.max_slices", &mxge_max_slices);
3803         TUNABLE_INT_FETCH("hw.mxge.flow_control_enabled",
3804                           &mxge_flow_control);
3805         TUNABLE_INT_FETCH("hw.mxge.intr_coal_delay",
3806                           &mxge_intr_coal_delay);
3807         TUNABLE_INT_FETCH("hw.mxge.nvidia_ecrc_enable",
3808                           &mxge_nvidia_ecrc_enable);
3809         TUNABLE_INT_FETCH("hw.mxge.force_firmware",
3810                           &mxge_force_firmware);
3811         TUNABLE_INT_FETCH("hw.mxge.deassert_wait",
3812                           &mxge_deassert_wait);
3813         TUNABLE_INT_FETCH("hw.mxge.verbose",
3814                           &mxge_verbose);
3815         TUNABLE_INT_FETCH("hw.mxge.ticks", &mxge_ticks);
3816         TUNABLE_INT_FETCH("hw.mxge.always_promisc", &mxge_always_promisc);
3817         TUNABLE_INT_FETCH("hw.mxge.throttle", &mxge_throttle);
3818
3819         if (bootverbose)
3820                 mxge_verbose = 1;
3821         if (mxge_intr_coal_delay < 0 || mxge_intr_coal_delay > 10*1000)
3822                 mxge_intr_coal_delay = 30;
3823         if (mxge_ticks == 0)
3824                 mxge_ticks = hz / 2;
3825         sc->pause = mxge_flow_control;
3826
3827         if (mxge_throttle && mxge_throttle > MXGE_MAX_THROTTLE)
3828                 mxge_throttle = MXGE_MAX_THROTTLE;
3829         if (mxge_throttle && mxge_throttle < MXGE_MIN_THROTTLE)
3830                 mxge_throttle = MXGE_MIN_THROTTLE;
3831         sc->throttle = mxge_throttle;
3832 }
3833
3834 static void
3835 mxge_free_slices(mxge_softc_t *sc)
3836 {
3837         struct mxge_slice_state *ss;
3838         int i;
3839
3840
3841         if (sc->ss == NULL)
3842                 return;
3843
3844         for (i = 0; i < sc->num_slices; i++) {
3845                 ss = &sc->ss[i];
3846                 if (ss->fw_stats != NULL) {
3847                         mxge_dma_free(&ss->fw_stats_dma);
3848                         ss->fw_stats = NULL;
3849                 }
3850                 if (ss->rx_done.entry != NULL) {
3851                         mxge_dma_free(&ss->rx_done.dma);
3852                         ss->rx_done.entry = NULL;
3853                 }
3854         }
3855         kfree(sc->ss, M_DEVBUF);
3856         sc->ss = NULL;
3857 }
3858
3859 static int
3860 mxge_alloc_slices(mxge_softc_t *sc)
3861 {
3862         mxge_cmd_t cmd;
3863         struct mxge_slice_state *ss;
3864         size_t bytes;
3865         int err, i, max_intr_slots;
3866
3867         err = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
3868         if (err != 0) {
3869                 device_printf(sc->dev, "Cannot determine rx ring size\n");
3870                 return err;
3871         }
3872         sc->rx_ring_size = cmd.data0;
3873         max_intr_slots = 2 * (sc->rx_ring_size / sizeof (mcp_dma_addr_t));
3874
3875         bytes = sizeof (*sc->ss) * sc->num_slices;
3876         sc->ss = kmalloc(bytes, M_DEVBUF, M_NOWAIT | M_ZERO);
3877         if (sc->ss == NULL)
3878                 return (ENOMEM);
3879         for (i = 0; i < sc->num_slices; i++) {
3880                 ss = &sc->ss[i];
3881
3882                 ss->sc = sc;
3883
3884                 /* allocate per-slice rx interrupt queues */
3885
3886                 bytes = max_intr_slots * sizeof (*ss->rx_done.entry);
3887                 err = mxge_dma_alloc(sc, &ss->rx_done.dma, bytes, 4096);
3888                 if (err != 0)
3889                         goto abort;
3890                 ss->rx_done.entry = ss->rx_done.dma.addr;
3891                 bzero(ss->rx_done.entry, bytes);
3892
3893                 /*
3894                  * allocate the per-slice firmware stats; stats
3895                  * (including tx) are used used only on the first
3896                  * slice for now
3897                  */
3898 #ifndef IFNET_BUF_RING
3899                 if (i > 0)
3900                         continue;
3901 #endif
3902
3903                 bytes = sizeof (*ss->fw_stats);
3904                 err = mxge_dma_alloc(sc, &ss->fw_stats_dma,
3905                                      sizeof (*ss->fw_stats), 64);
3906                 if (err != 0)
3907                         goto abort;
3908                 ss->fw_stats = (mcp_irq_data_t *)ss->fw_stats_dma.addr;
3909         }
3910
3911         return (0);
3912
3913 abort:
3914         mxge_free_slices(sc);
3915         return (ENOMEM);
3916 }
3917
3918 static void
3919 mxge_slice_probe(mxge_softc_t *sc)
3920 {
3921         mxge_cmd_t cmd;
3922         char *old_fw;
3923         int msix_cnt, status, max_intr_slots;
3924
3925         sc->num_slices = 1;
3926         /*
3927          *  don't enable multiple slices if they are not enabled,
3928          *  or if this is not an SMP system
3929          */
3930
3931         if (mxge_max_slices == 0 || mxge_max_slices == 1 || ncpus < 2)
3932                 return;
3933
3934         /* see how many MSI-X interrupts are available */
3935         msix_cnt = pci_msix_count(sc->dev);
3936         if (msix_cnt < 2)
3937                 return;
3938
3939         /* now load the slice aware firmware see what it supports */
3940         old_fw = sc->fw_name;
3941         if (old_fw == mxge_fw_aligned)
3942                 sc->fw_name = mxge_fw_rss_aligned;
3943         else
3944                 sc->fw_name = mxge_fw_rss_unaligned;
3945         status = mxge_load_firmware(sc, 0);
3946         if (status != 0) {
3947                 device_printf(sc->dev, "Falling back to a single slice\n");
3948                 return;
3949         }
3950
3951         /* try to send a reset command to the card to see if it
3952            is alive */
3953         memset(&cmd, 0, sizeof (cmd));
3954         status = mxge_send_cmd(sc, MXGEFW_CMD_RESET, &cmd);
3955         if (status != 0) {
3956                 device_printf(sc->dev, "failed reset\n");
3957                 goto abort_with_fw;
3958         }
3959
3960         /* get rx ring size */
3961         status = mxge_send_cmd(sc, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
3962         if (status != 0) {
3963                 device_printf(sc->dev, "Cannot determine rx ring size\n");
3964                 goto abort_with_fw;
3965         }
3966         max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t));
3967
3968         /* tell it the size of the interrupt queues */
3969         cmd.data0 = max_intr_slots * sizeof (struct mcp_slot);
3970         status = mxge_send_cmd(sc, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
3971         if (status != 0) {
3972                 device_printf(sc->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n");
3973                 goto abort_with_fw;
3974         }
3975
3976         /* ask the maximum number of slices it supports */
3977         status = mxge_send_cmd(sc, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd);
3978         if (status != 0) {
3979                 device_printf(sc->dev,
3980                               "failed MXGEFW_CMD_GET_MAX_RSS_QUEUES\n");
3981                 goto abort_with_fw;
3982         }
3983         sc->num_slices = cmd.data0;
3984         if (sc->num_slices > msix_cnt)
3985                 sc->num_slices = msix_cnt;
3986
3987         if (mxge_max_slices == -1) {
3988                 /* cap to number of CPUs in system */
3989                 if (sc->num_slices > ncpus)
3990                         sc->num_slices = ncpus;
3991         } else {
3992                 if (sc->num_slices > mxge_max_slices)
3993                         sc->num_slices = mxge_max_slices;
3994         }
3995         /* make sure it is a power of two */
3996         while (sc->num_slices & (sc->num_slices - 1))
3997                 sc->num_slices--;
3998
3999         if (mxge_verbose)
4000                 device_printf(sc->dev, "using %d slices\n",
4001                               sc->num_slices);
4002
4003         return;
4004
4005 abort_with_fw:
4006         sc->fw_name = old_fw;
4007         (void) mxge_load_firmware(sc, 0);
4008 }
4009
4010 #if 0
4011 static int
4012 mxge_add_msix_irqs(mxge_softc_t *sc)
4013 {
4014         size_t bytes;
4015         int count, err, i, rid;
4016
4017         rid = PCIR_BAR(2);
4018         sc->msix_table_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
4019                                                     &rid, RF_ACTIVE);
4020
4021         if (sc->msix_table_res == NULL) {
4022                 device_printf(sc->dev, "couldn't alloc MSIX table res\n");
4023                 return ENXIO;
4024         }
4025
4026         count = sc->num_slices;
4027         err = pci_alloc_msix(sc->dev, &count);
4028         if (err != 0) {
4029                 device_printf(sc->dev, "pci_alloc_msix: failed, wanted %d"
4030                               "err = %d \n", sc->num_slices, err);
4031                 goto abort_with_msix_table;
4032         }
4033         if (count < sc->num_slices) {
4034                 device_printf(sc->dev, "pci_alloc_msix: need %d, got %d\n",
4035                               count, sc->num_slices);
4036                 device_printf(sc->dev,
4037                               "Try setting hw.mxge.max_slices to %d\n",
4038                               count);
4039                 err = ENOSPC;
4040                 goto abort_with_msix;
4041         }
4042         bytes = sizeof (*sc->msix_irq_res) * sc->num_slices;
4043         sc->msix_irq_res = kmalloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
4044         if (sc->msix_irq_res == NULL) {
4045                 err = ENOMEM;
4046                 goto abort_with_msix;
4047         }
4048
4049         for (i = 0; i < sc->num_slices; i++) {
4050                 rid = i + 1;
4051                 sc->msix_irq_res[i] = bus_alloc_resource_any(sc->dev,
4052                                                           SYS_RES_IRQ,
4053                                                           &rid, RF_ACTIVE);
4054                 if (sc->msix_irq_res[i] == NULL) {
4055                         device_printf(sc->dev, "couldn't allocate IRQ res"
4056                                       " for message %d\n", i);
4057                         err = ENXIO;
4058                         goto abort_with_res;
4059                 }
4060         }
4061
4062         bytes = sizeof (*sc->msix_ih) * sc->num_slices;
4063         sc->msix_ih =  kmalloc(bytes, M_DEVBUF, M_NOWAIT|M_ZERO);
4064
4065         for (i = 0; i < sc->num_slices; i++) {
4066                 err = bus_setup_intr(sc->dev, sc->msix_irq_res[i],
4067                                      INTR_MPSAFE,
4068                                      mxge_intr, &sc->ss[i], &sc->msix_ih[i],
4069                                      sc->ifp->if_serializer);
4070                 if (err != 0) {
4071                         device_printf(sc->dev, "couldn't setup intr for "
4072                                       "message %d\n", i);
4073                         goto abort_with_intr;
4074                 }
4075         }
4076
4077         if (mxge_verbose) {
4078                 device_printf(sc->dev, "using %d msix IRQs:",
4079                               sc->num_slices);
4080                 for (i = 0; i < sc->num_slices; i++)
4081                         kprintf(" %ld",  rman_get_start(sc->msix_irq_res[i]));
4082                 kprintf("\n");
4083         }
4084         return (0);
4085
4086 abort_with_intr:
4087         for (i = 0; i < sc->num_slices; i++) {
4088                 if (sc->msix_ih[i] != NULL) {
4089                         bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
4090                                           sc->msix_ih[i]);
4091                         sc->msix_ih[i] = NULL;
4092                 }
4093         }
4094         kfree(sc->msix_ih, M_DEVBUF);
4095
4096
4097 abort_with_res:
4098         for (i = 0; i < sc->num_slices; i++) {
4099                 rid = i + 1;
4100                 if (sc->msix_irq_res[i] != NULL)
4101                         bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
4102                                              sc->msix_irq_res[i]);
4103                 sc->msix_irq_res[i] = NULL;
4104         }
4105         kfree(sc->msix_irq_res, M_DEVBUF);
4106
4107
4108 abort_with_msix:
4109         pci_release_msi(sc->dev);
4110
4111 abort_with_msix_table:
4112         bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
4113                              sc->msix_table_res);
4114
4115         return err;
4116 }
4117 #endif
4118
4119 static int
4120 mxge_add_single_irq(mxge_softc_t *sc)
4121 {
4122         u_int irq_flags;
4123         int err;
4124
4125         sc->irq_type = pci_alloc_1intr(sc->dev, 1, &sc->irq_rid, &irq_flags);
4126
4127         sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ,
4128             &sc->irq_rid, irq_flags);
4129         if (sc->irq_res == NULL) {
4130                 device_printf(sc->dev, "could not alloc interrupt\n");
4131                 return ENXIO;
4132         }
4133
4134         err = bus_setup_intr(sc->dev, sc->irq_res,
4135                              INTR_MPSAFE,
4136                              mxge_intr, &sc->ss[0], &sc->ih,
4137                              sc->ifp->if_serializer);
4138         if (err != 0) {
4139                 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
4140                     sc->irq_res);
4141                 if (sc->irq_type == PCI_INTR_TYPE_MSI)
4142                         pci_release_msi(sc->dev);
4143         }
4144         return err;
4145 }
4146
4147 #if 0
4148 static void
4149 mxge_rem_msix_irqs(mxge_softc_t *sc)
4150 {
4151         int i, rid;
4152
4153         for (i = 0; i < sc->num_slices; i++) {
4154                 if (sc->msix_ih[i] != NULL) {
4155                         bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
4156                                           sc->msix_ih[i]);
4157                         sc->msix_ih[i] = NULL;
4158                 }
4159         }
4160         kfree(sc->msix_ih, M_DEVBUF);
4161
4162         for (i = 0; i < sc->num_slices; i++) {
4163                 rid = i + 1;
4164                 if (sc->msix_irq_res[i] != NULL)
4165                         bus_release_resource(sc->dev, SYS_RES_IRQ, rid,
4166                                              sc->msix_irq_res[i]);
4167                 sc->msix_irq_res[i] = NULL;
4168         }
4169         kfree(sc->msix_irq_res, M_DEVBUF);
4170
4171         bus_release_resource(sc->dev, SYS_RES_MEMORY, PCIR_BAR(2),
4172                              sc->msix_table_res);
4173
4174         pci_release_msi(sc->dev);
4175         return;
4176 }
4177 #endif
4178
4179 static void
4180 mxge_rem_single_irq(mxge_softc_t *sc)
4181 {
4182         bus_teardown_intr(sc->dev, sc->irq_res, sc->ih);
4183         bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid, sc->irq_res);
4184         if (sc->irq_type == PCI_INTR_TYPE_MSI)
4185                 pci_release_msi(sc->dev);
4186 }
4187
4188 static void
4189 mxge_rem_irq(mxge_softc_t *sc)
4190 {
4191 #if 0
4192         if (sc->num_slices > 1)
4193                 mxge_rem_msix_irqs(sc);
4194         else
4195 #endif
4196                 mxge_rem_single_irq(sc);
4197 }
4198
4199 static int
4200 mxge_add_irq(mxge_softc_t *sc)
4201 {
4202 #if 0
4203         int err;
4204
4205         if (sc->num_slices > 1)
4206                 err = mxge_add_msix_irqs(sc);
4207         else
4208                 err = mxge_add_single_irq(sc);
4209
4210         if (0 && err == 0 && sc->num_slices > 1) {
4211                 mxge_rem_msix_irqs(sc);
4212                 err = mxge_add_msix_irqs(sc);
4213         }
4214         return err;
4215 #else
4216         return mxge_add_single_irq(sc);
4217 #endif
4218 }
4219
4220 static int
4221 mxge_attach(device_t dev)
4222 {
4223         mxge_softc_t *sc = device_get_softc(dev);
4224         struct ifnet *ifp = &sc->arpcom.ac_if;
4225         int err, rid;
4226
4227         /*
4228          * avoid rewriting half the lines in this file to use
4229          * &sc->arpcom.ac_if instead
4230          */
4231         sc->ifp = ifp;
4232         sc->dev = dev;
4233         mxge_fetch_tunables(sc);
4234
4235         err = bus_dma_tag_create(NULL,                  /* parent */
4236                                  1,                     /* alignment */
4237                                  0,                     /* boundary */
4238                                  BUS_SPACE_MAXADDR,     /* low */
4239                                  BUS_SPACE_MAXADDR,     /* high */
4240                                  NULL, NULL,            /* filter */
4241                                  65536 + 256,           /* maxsize */
4242                                  MXGE_MAX_SEND_DESC,    /* num segs */
4243                                  65536,                 /* maxsegsize */
4244                                  0,                     /* flags */
4245                                  &sc->parent_dmat);     /* tag */
4246
4247         if (err != 0) {
4248                 device_printf(sc->dev, "Err %d allocating parent dmat\n",
4249                               err);
4250                 goto abort_with_nothing;
4251         }
4252
4253         sc->ifp = ifp;
4254         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
4255
4256         callout_init_mp(&sc->co_hdl);
4257
4258         mxge_setup_cfg_space(sc);
4259
4260         /* Map the board into the kernel */
4261         rid = PCIR_BARS;
4262         sc->mem_res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid, 0,
4263                                          ~0, 1, RF_ACTIVE);
4264         if (sc->mem_res == NULL) {
4265                 device_printf(dev, "could not map memory\n");
4266                 err = ENXIO;
4267                 goto abort_with_parent_dmat;
4268         }
4269         sc->sram = rman_get_virtual(sc->mem_res);
4270         sc->sram_size = 2*1024*1024 - (2*(48*1024)+(32*1024)) - 0x100;
4271         if (sc->sram_size > rman_get_size(sc->mem_res)) {
4272                 device_printf(dev, "impossible memory region size %ld\n",
4273                               rman_get_size(sc->mem_res));
4274                 err = ENXIO;
4275                 goto abort_with_mem_res;
4276         }
4277
4278         /* make NULL terminated copy of the EEPROM strings section of
4279            lanai SRAM */
4280         bzero(sc->eeprom_strings, MXGE_EEPROM_STRINGS_SIZE);
4281         bus_space_read_region_1(rman_get_bustag(sc->mem_res),
4282                                 rman_get_bushandle(sc->mem_res),
4283                                 sc->sram_size - MXGE_EEPROM_STRINGS_SIZE,
4284                                 sc->eeprom_strings,
4285                                 MXGE_EEPROM_STRINGS_SIZE - 2);
4286         err = mxge_parse_strings(sc);
4287         if (err != 0)
4288                 goto abort_with_mem_res;
4289
4290         /* Enable write combining for efficient use of PCIe bus */
4291         mxge_enable_wc(sc);
4292
4293         /* Allocate the out of band dma memory */
4294         err = mxge_dma_alloc(sc, &sc->cmd_dma,
4295                              sizeof (mxge_cmd_t), 64);
4296         if (err != 0)
4297                 goto abort_with_mem_res;
4298         sc->cmd = (mcp_cmd_response_t *) sc->cmd_dma.addr;
4299         err = mxge_dma_alloc(sc, &sc->zeropad_dma, 64, 64);
4300         if (err != 0)
4301                 goto abort_with_cmd_dma;
4302
4303         err = mxge_dma_alloc(sc, &sc->dmabench_dma, 4096, 4096);
4304         if (err != 0)
4305                 goto abort_with_zeropad_dma;
4306
4307         /* select & load the firmware */
4308         err = mxge_select_firmware(sc);
4309         if (err != 0)
4310                 goto abort_with_dmabench;
4311         sc->intr_coal_delay = mxge_intr_coal_delay;
4312
4313         mxge_slice_probe(sc);
4314         err = mxge_alloc_slices(sc);
4315         if (err != 0)
4316                 goto abort_with_dmabench;
4317
4318         err = mxge_reset(sc, 0);
4319         if (err != 0)
4320                 goto abort_with_slices;
4321
4322         err = mxge_alloc_rings(sc);
4323         if (err != 0) {
4324                 device_printf(sc->dev, "failed to allocate rings\n");
4325                 goto abort_with_slices;
4326         }
4327
4328         ifp->if_baudrate = IF_Gbps(10UL);
4329         ifp->if_capabilities = IFCAP_RXCSUM | IFCAP_TXCSUM | IFCAP_TSO;
4330         ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_TSO;
4331
4332         ifp->if_capabilities |= IFCAP_VLAN_MTU;
4333 #if 0
4334         /* Well, its software, sigh */
4335         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
4336 #endif
4337         ifp->if_capenable = ifp->if_capabilities;
4338
4339         ifp->if_softc = sc;
4340         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
4341         ifp->if_init = mxge_init;
4342         ifp->if_ioctl = mxge_ioctl;
4343         ifp->if_start = mxge_start;
4344         /* XXX watchdog */
4345
4346         /* Initialise the ifmedia structure */
4347         ifmedia_init(&sc->media, 0, mxge_media_change,
4348                      mxge_media_status);
4349         mxge_media_init(sc);
4350         mxge_media_probe(sc);
4351
4352         ether_ifattach(ifp, sc->mac_addr, NULL);
4353
4354         sc->max_mtu = ETHERMTU + EVL_ENCAPLEN;
4355         sc->dying = 0;
4356
4357         /* must come after ether_ifattach() */
4358         err = mxge_add_irq(sc);
4359         if (err != 0) {
4360                 device_printf(sc->dev, "failed to add irq\n");
4361                 goto abort_with_rings;
4362         }
4363         ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->irq_res));
4364
4365         mxge_add_sysctls(sc);
4366
4367         callout_reset(&sc->co_hdl, mxge_ticks, mxge_tick, sc);
4368         return 0;
4369
4370 abort_with_rings:
4371         mxge_free_rings(sc);
4372 abort_with_slices:
4373         mxge_free_slices(sc);
4374 abort_with_dmabench:
4375         mxge_dma_free(&sc->dmabench_dma);
4376 abort_with_zeropad_dma:
4377         mxge_dma_free(&sc->zeropad_dma);
4378 abort_with_cmd_dma:
4379         mxge_dma_free(&sc->cmd_dma);
4380 abort_with_mem_res:
4381         bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
4382         pci_disable_busmaster(dev);
4383 abort_with_parent_dmat:
4384         bus_dma_tag_destroy(sc->parent_dmat);
4385 abort_with_nothing:
4386         return err;
4387 }
4388
4389 static int
4390 mxge_detach(device_t dev)
4391 {
4392         mxge_softc_t *sc = device_get_softc(dev);
4393
4394         lwkt_serialize_enter(sc->ifp->if_serializer);
4395         sc->dying = 1;
4396         if (sc->ifp->if_flags & IFF_RUNNING)
4397                 mxge_close(sc, 1);
4398         /*
4399          * XXX: race: the callout callback could be spinning on
4400          * the serializer and run anyway
4401          */
4402         callout_stop(&sc->co_hdl);
4403         lwkt_serialize_exit(sc->ifp->if_serializer);
4404
4405         callout_terminate(&sc->co_hdl);
4406
4407         ether_ifdetach(sc->ifp);
4408         ifmedia_removeall(&sc->media);
4409         mxge_dummy_rdma(sc, 0);
4410         mxge_rem_sysctls(sc);
4411         mxge_rem_irq(sc);
4412         mxge_free_rings(sc);
4413         mxge_free_slices(sc);
4414         mxge_dma_free(&sc->dmabench_dma);
4415         mxge_dma_free(&sc->zeropad_dma);
4416         mxge_dma_free(&sc->cmd_dma);
4417         bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BARS, sc->mem_res);
4418         pci_disable_busmaster(dev);
4419         bus_dma_tag_destroy(sc->parent_dmat);
4420         return 0;
4421 }
4422
4423 static int
4424 mxge_shutdown(device_t dev)
4425 {
4426         return 0;
4427 }