1 /******************************************************************************
3 Copyright (c) 2001-2012, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
36 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 static void ixgbe_start(struct ifnet *);
106 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111 static void ixgbe_qflush(struct ifnet *);
113 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void ixgbe_init(void *);
115 static void ixgbe_init_locked(struct adapter *);
116 static void ixgbe_stop(void *);
117 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int ixgbe_media_change(struct ifnet *);
119 static void ixgbe_identify_hardware(struct adapter *);
120 static int ixgbe_allocate_pci_resources(struct adapter *);
121 static int ixgbe_allocate_msix(struct adapter *);
122 static int ixgbe_allocate_legacy(struct adapter *);
123 static int ixgbe_allocate_queues(struct adapter *);
124 static int ixgbe_setup_msix(struct adapter *);
125 static void ixgbe_free_pci_resources(struct adapter *);
126 static void ixgbe_local_timer(void *);
127 static int ixgbe_setup_interface(device_t, struct adapter *);
128 static void ixgbe_config_link(struct adapter *);
130 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
131 static int ixgbe_setup_transmit_structures(struct adapter *);
132 static void ixgbe_setup_transmit_ring(struct tx_ring *);
133 static void ixgbe_initialize_transmit_units(struct adapter *);
134 static void ixgbe_free_transmit_structures(struct adapter *);
135 static void ixgbe_free_transmit_buffers(struct tx_ring *);
137 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
138 static int ixgbe_setup_receive_structures(struct adapter *);
139 static int ixgbe_setup_receive_ring(struct rx_ring *);
140 static void ixgbe_initialize_receive_units(struct adapter *);
141 static void ixgbe_free_receive_structures(struct adapter *);
142 static void ixgbe_free_receive_buffers(struct rx_ring *);
144 static void ixgbe_setup_hw_rsc(struct rx_ring *);
147 static void ixgbe_enable_intr(struct adapter *);
148 static void ixgbe_disable_intr(struct adapter *);
149 static void ixgbe_update_stats_counters(struct adapter *);
150 static void ixgbe_txeof(struct tx_ring *);
151 static void ixgbe_rxeof(struct ix_queue *, int);
152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void ixgbe_set_promisc(struct adapter *);
154 static void ixgbe_set_multi(struct adapter *);
155 static void ixgbe_update_link_status(struct adapter *);
156 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
162 struct ixgbe_dma_alloc *, int);
163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
165 const char *, int *, int);
166 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
167 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
168 static int ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
169 static void ixgbe_add_sysctl(struct adapter *);
170 static void ixgbe_set_eitr(struct adapter *, int, int);
171 static int ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
172 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void ixgbe_configure_ivars(struct adapter *);
174 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
176 static void ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
180 static void ixgbe_add_hw_stats(struct adapter *adapter);
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
186 /* Support for pluggable optic modules */
187 static bool ixgbe_sfp_probe(struct adapter *);
188 static void ixgbe_setup_optics(struct adapter *);
190 /* Legacy (single vector interrupt handler */
191 static void ixgbe_legacy_irq(void *);
193 /* The MSI/X Interrupt handlers */
194 static void ixgbe_msix_que(void *);
195 static void ixgbe_msix_link(void *);
197 /* Deferred interrupt tasklets */
198 static void ixgbe_handle_msf(void *, int);
199 static void ixgbe_handle_mod(void *, int);
202 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
203 static void ixgbe_reinit_fdir(void *, int);
206 /*********************************************************************
207 * FreeBSD Device Interface Entry Points
208 *********************************************************************/
210 static device_method_t ixgbe_methods[] = {
211 /* Device interface */
212 DEVMETHOD(device_probe, ixgbe_probe),
213 DEVMETHOD(device_attach, ixgbe_attach),
214 DEVMETHOD(device_detach, ixgbe_detach),
215 DEVMETHOD(device_shutdown, ixgbe_shutdown),
219 static driver_t ixgbe_driver = {
220 "ix", ixgbe_methods, sizeof(struct adapter),
223 devclass_t ixgbe_devclass;
224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230 ** TUNEABLE PARAMETERS:
233 /* How many packets rxeof tries to clean at a time */
234 static int ixgbe_rx_process_limit = 128;
235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
238 ** Smart speed setting, default to on
239 ** this only works as a compile option
240 ** right now as its during attach, set
241 ** this to 'ixgbe_smart_speed_off' to
244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
246 static int ixgbe_msi_enable = 1;
247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
250 * MSIX should be the default for best performance,
251 * but this allows it to be forced off for testing.
253 static int ixgbe_enable_msix = 1;
254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
257 * Header split: this causes the hardware to DMA
258 * the header into a separate mbuf from the payload,
259 * it can be a performance win in some workloads, but
260 * in others it actually hurts, its off by default.
262 static int ixgbe_header_split = FALSE;
263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
266 * Number of Queues, can be set to 0,
267 * it then autoconfigures based on the
268 * number of cpus with a max of 8. This
269 * can be overriden manually here.
271 static int ixgbe_num_queues = 0;
272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
275 ** Number of TX descriptors per ring,
276 ** setting higher than RX as this seems
277 ** the better performing choice.
279 static int ixgbe_txd = PERFORM_TXD;
280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
282 /* Number of RX descriptors per ring */
283 static int ixgbe_rxd = PERFORM_RXD;
284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
286 /* Keep running tab on them for sanity check */
287 static int ixgbe_total_ports;
291 ** For Flow Director: this is the
292 ** number of TX packets we sample
293 ** for the filter pool, this means
294 ** every 20th packet will be probed.
296 ** This feature can be disabled by
297 ** setting this to 0.
299 static int atr_sample_rate = 20;
301 ** Flow Director actually 'steals'
302 ** part of the packet buffer as its
303 ** filter pool, this variable controls
305 ** 0 = 64K, 1 = 128K, 2 = 256K
307 static int fdir_pballoc = 1;
312 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
313 * be a reference on how to implement netmap support in a driver.
314 * Additional comments are in ixgbe_netmap.h .
316 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
317 * that extend the standard driver.
319 #include <dev/netmap/ixgbe_netmap.h>
320 #endif /* DEV_NETMAP */
322 /*********************************************************************
323 * Device identification routine
325 * ixgbe_probe determines if the driver should be loaded on
326 * adapter based on PCI vendor/device id of the adapter.
328 * return BUS_PROBE_DEFAULT on success, positive on failure
329 *********************************************************************/
332 ixgbe_probe(device_t dev)
334 ixgbe_vendor_info_t *ent;
336 u16 pci_vendor_id = 0;
337 u16 pci_device_id = 0;
338 u16 pci_subvendor_id = 0;
339 u16 pci_subdevice_id = 0;
340 char adapter_name[256];
342 INIT_DEBUGOUT("ixgbe_probe: begin");
344 pci_vendor_id = pci_get_vendor(dev);
345 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
348 pci_device_id = pci_get_device(dev);
349 pci_subvendor_id = pci_get_subvendor(dev);
350 pci_subdevice_id = pci_get_subdevice(dev);
352 ent = ixgbe_vendor_info_array;
353 while (ent->vendor_id != 0) {
354 if ((pci_vendor_id == ent->vendor_id) &&
355 (pci_device_id == ent->device_id) &&
357 ((pci_subvendor_id == ent->subvendor_id) ||
358 (ent->subvendor_id == 0)) &&
360 ((pci_subdevice_id == ent->subdevice_id) ||
361 (ent->subdevice_id == 0))) {
362 ksprintf(adapter_name, "%s, Version - %s",
363 ixgbe_strings[ent->index],
364 ixgbe_driver_version);
365 device_set_desc_copy(dev, adapter_name);
367 return (BUS_PROBE_DEFAULT);
374 /*********************************************************************
375 * Device initialization routine
377 * The attach entry point is called when the driver is being loaded.
378 * This routine identifies the type of hardware, allocates all resources
379 * and initializes the hardware.
381 * return 0 on success, positive on failure
382 *********************************************************************/
385 ixgbe_attach(device_t dev)
387 struct adapter *adapter;
393 INIT_DEBUGOUT("ixgbe_attach: begin");
395 if (resource_disabled("ixgbe", device_get_unit(dev))) {
396 device_printf(dev, "Disabled by device hint\n");
400 /* Allocate, clear, and link in our adapter structure */
401 adapter = device_get_softc(dev);
402 adapter->dev = adapter->osdep.dev = dev;
406 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
408 /* Set up the timer callout */
409 callout_init_mp(&adapter->timer);
411 /* Determine hardware revision */
412 ixgbe_identify_hardware(adapter);
414 /* Enable bus mastering */
415 pci_enable_busmaster(dev);
417 /* Do base PCI setup - map BAR0 */
418 if (ixgbe_allocate_pci_resources(adapter)) {
419 device_printf(dev, "Allocation of PCI resources failed\n");
424 /* Do descriptor calc and sanity checks */
425 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
426 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
427 device_printf(dev, "TXD config issue, using default!\n");
428 adapter->num_tx_desc = DEFAULT_TXD;
430 adapter->num_tx_desc = ixgbe_txd;
433 ** With many RX rings it is easy to exceed the
434 ** system mbuf allocation. Tuning nmbclusters
435 ** can alleviate this.
437 if (nmbclusters > 0 ) {
439 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
440 if (s > nmbclusters) {
441 device_printf(dev, "RX Descriptors exceed "
442 "system mbuf max, using default instead!\n");
443 ixgbe_rxd = DEFAULT_RXD;
447 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
448 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
449 device_printf(dev, "RXD config issue, using default!\n");
450 adapter->num_rx_desc = DEFAULT_RXD;
452 adapter->num_rx_desc = ixgbe_rxd;
454 /* Allocate our TX/RX Queues */
455 if (ixgbe_allocate_queues(adapter)) {
460 /* Allocate multicast array memory. */
461 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
462 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
463 if (adapter->mta == NULL) {
464 device_printf(dev, "Can not allocate multicast setup array\n");
469 /* Initialize the shared code */
470 error = ixgbe_init_shared_code(hw);
471 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
473 ** No optics in this port, set up
474 ** so the timer routine will probe
475 ** for later insertion.
477 adapter->sfp_probe = TRUE;
479 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
480 device_printf(dev,"Unsupported SFP+ module detected!\n");
484 device_printf(dev,"Unable to initialize the shared code\n");
489 /* Make sure we have a good EEPROM before we read from it */
490 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
491 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
496 error = ixgbe_init_hw(hw);
498 case IXGBE_ERR_EEPROM_VERSION:
499 device_printf(dev, "This device is a pre-production adapter/"
500 "LOM. Please be aware there may be issues associated "
501 "with your hardware.\n If you are experiencing problems "
502 "please contact your Intel or hardware representative "
503 "who provided you with this hardware.\n");
505 case IXGBE_ERR_SFP_NOT_SUPPORTED:
506 device_printf(dev,"Unsupported SFP+ Module\n");
508 device_printf(dev,"Hardware Initialization Failure\n");
510 case IXGBE_ERR_SFP_NOT_PRESENT:
511 device_printf(dev,"No SFP+ Module found\n");
517 /* Detect and set physical type */
518 ixgbe_setup_optics(adapter);
520 if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
521 adapter->intr_type = PCI_INTR_TYPE_MSIX;
522 error = ixgbe_allocate_msix(adapter);
524 error = ixgbe_allocate_legacy(adapter);
529 /* Setup OS specific network interface */
530 if (ixgbe_setup_interface(dev, adapter) != 0)
533 /* Add sysctl tree */
534 ixgbe_add_sysctl(adapter);
536 /* Initialize statistics */
537 ixgbe_update_stats_counters(adapter);
539 /* Register for VLAN events */
540 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
541 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
542 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
543 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
545 /* Print PCIE bus type/speed/width info */
546 ixgbe_get_bus_info(hw);
547 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
548 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
549 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
550 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
551 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
552 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
555 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
556 (hw->bus.speed == ixgbe_bus_speed_2500)) {
557 device_printf(dev, "PCI-Express bandwidth available"
558 " for this card\n is not sufficient for"
559 " optimal performance.\n");
560 device_printf(dev, "For optimal performance a x8 "
561 "PCIE, or x4 PCIE 2 slot is required.\n");
564 /* let hardware know driver is loaded */
565 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
566 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
567 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
569 ixgbe_add_hw_stats(adapter);
572 ixgbe_netmap_attach(adapter);
573 #endif /* DEV_NETMAP */
574 INIT_DEBUGOUT("ixgbe_attach: end");
577 ixgbe_free_transmit_structures(adapter);
578 ixgbe_free_receive_structures(adapter);
580 if (adapter->ifp != NULL)
581 if_free(adapter->ifp);
582 ixgbe_free_pci_resources(adapter);
583 kfree(adapter->mta, M_DEVBUF);
588 /*********************************************************************
589 * Device removal routine
591 * The detach entry point is called when the driver is being removed.
592 * This routine stops the adapter and deallocates all the resources
593 * that were allocated for driver operation.
595 * return 0 on success, positive on failure
596 *********************************************************************/
599 ixgbe_detach(device_t dev)
601 struct adapter *adapter = device_get_softc(dev);
604 INIT_DEBUGOUT("ixgbe_detach: begin");
606 /* Make sure VLANS are not using driver */
607 if (adapter->ifp->if_vlantrunks != NULL) {
608 device_printf(dev,"Vlan in use, detach first\n");
612 IXGBE_CORE_LOCK(adapter);
614 IXGBE_CORE_UNLOCK(adapter);
616 /* Drain the Link queue */
618 taskqueue_drain(adapter->tq, &adapter->mod_task);
619 taskqueue_drain(adapter->tq, &adapter->msf_task);
621 taskqueue_drain(adapter->tq, &adapter->fdir_task);
623 taskqueue_free(adapter->tq);
626 /* let hardware know driver is unloading */
627 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
628 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
629 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
631 /* Unregister VLAN events */
632 if (adapter->vlan_attach != NULL)
633 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
634 if (adapter->vlan_detach != NULL)
635 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
637 ether_ifdetach(adapter->ifp);
638 callout_stop(&adapter->timer);
640 netmap_detach(adapter->ifp);
641 #endif /* DEV_NETMAP */
642 ixgbe_free_pci_resources(adapter);
643 bus_generic_detach(dev);
644 if_free(adapter->ifp);
646 ixgbe_free_transmit_structures(adapter);
647 ixgbe_free_receive_structures(adapter);
648 kfree(adapter->mta, M_DEVBUF);
649 sysctl_ctx_free(&adapter->sysctl_ctx);
651 IXGBE_CORE_LOCK_DESTROY(adapter);
655 /*********************************************************************
657 * Shutdown entry point
659 **********************************************************************/
662 ixgbe_shutdown(device_t dev)
664 struct adapter *adapter = device_get_softc(dev);
665 IXGBE_CORE_LOCK(adapter);
667 IXGBE_CORE_UNLOCK(adapter);
672 /*********************************************************************
673 * Transmit entry point
675 * ixgbe_start is called by the stack to initiate a transmit.
676 * The driver will remain in this routine as long as there are
677 * packets to transmit and transmit resources are available.
678 * In case resources are not available stack is notified and
679 * the packet is requeued.
680 **********************************************************************/
683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
686 struct adapter *adapter = txr->adapter;
688 IXGBE_TX_LOCK_ASSERT(txr);
690 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
692 if (!adapter->link_active)
695 while (!ifq_is_empty(&ifp->if_snd)) {
696 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
697 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
701 m_head = ifq_dequeue(&ifp->if_snd, NULL);
705 if (ixgbe_xmit(txr, &m_head)) {
706 #if 0 /* XXX: prepend to an ALTQ queue ? */
708 IF_PREPEND(&ifp->if_snd, m_head);
710 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
711 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
714 /* Send a copy of the frame to the BPF listener */
715 ETHER_BPF_MTAP(ifp, m_head);
717 /* Set watchdog on */
718 txr->watchdog_time = ticks;
719 txr->queue_status = IXGBE_QUEUE_WORKING;
726 * Legacy TX start - called by the stack, this
727 * always uses the first tx ring, and should
728 * not be used with multiqueue tx enabled.
731 ixgbe_start(struct ifnet *ifp)
733 struct adapter *adapter = ifp->if_softc;
734 struct tx_ring *txr = adapter->tx_rings;
736 if (ifp->if_flags & IFF_RUNNING) {
738 ixgbe_start_locked(txr, ifp);
739 IXGBE_TX_UNLOCK(txr);
744 #if 0 /* __FreeBSD_version >= 800000 */
746 ** Multiqueue Transmit driver
750 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
752 struct adapter *adapter = ifp->if_softc;
753 struct ix_queue *que;
757 /* Which queue to use */
758 if ((m->m_flags & M_FLOWID) != 0)
759 i = m->m_pkthdr.flowid % adapter->num_queues;
761 i = curcpu % adapter->num_queues;
763 txr = &adapter->tx_rings[i];
764 que = &adapter->queues[i];
766 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
767 IXGBE_TX_TRYLOCK(txr)) {
768 err = ixgbe_mq_start_locked(ifp, txr, m);
769 IXGBE_TX_UNLOCK(txr);
771 err = drbr_enqueue(ifp, txr->br, m);
772 taskqueue_enqueue(que->tq, &que->que_task);
779 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
781 struct adapter *adapter = txr->adapter;
783 int enqueued, err = 0;
785 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
786 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
787 adapter->link_active == 0) {
789 err = drbr_enqueue(ifp, txr->br, m);
795 next = drbr_dequeue(ifp, txr->br);
796 } else if (drbr_needs_enqueue(ifp, txr->br)) {
797 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
799 next = drbr_dequeue(ifp, txr->br);
803 /* Process the queue */
804 while (next != NULL) {
805 if ((err = ixgbe_xmit(txr, &next)) != 0) {
807 err = drbr_enqueue(ifp, txr->br, next);
811 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
812 /* Send a copy of the frame to the BPF listener */
813 ETHER_BPF_MTAP(ifp, next);
814 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
816 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
818 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
819 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
822 next = drbr_dequeue(ifp, txr->br);
826 /* Set watchdog on */
827 txr->queue_status |= IXGBE_QUEUE_WORKING;
828 txr->watchdog_time = ticks;
831 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
838 ** Flush all ring buffers
841 ixgbe_qflush(struct ifnet *ifp)
843 struct adapter *adapter = ifp->if_softc;
844 struct tx_ring *txr = adapter->tx_rings;
847 for (int i = 0; i < adapter->num_queues; i++, txr++) {
849 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
851 IXGBE_TX_UNLOCK(txr);
855 #endif /* __FreeBSD_version >= 800000 */
857 /*********************************************************************
860 * ixgbe_ioctl is called when the user wants to configure the
863 * return 0 on success, positive on failure
864 **********************************************************************/
867 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
869 struct adapter *adapter = ifp->if_softc;
870 struct ifreq *ifr = (struct ifreq *) data;
871 #if defined(INET) || defined(INET6)
872 struct ifaddr *ifa = (struct ifaddr *)data;
873 bool avoid_reset = FALSE;
881 if (ifa->ifa_addr->sa_family == AF_INET)
885 if (ifa->ifa_addr->sa_family == AF_INET6)
888 #if defined(INET) || defined(INET6)
890 ** Calling init results in link renegotiation,
891 ** so we avoid doing it when possible.
894 ifp->if_flags |= IFF_UP;
895 if (!(ifp->if_flags & IFF_RUNNING))
897 if (!(ifp->if_flags & IFF_NOARP))
898 arp_ifinit(ifp, ifa);
900 error = ether_ioctl(ifp, command, data);
904 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
905 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
908 IXGBE_CORE_LOCK(adapter);
909 ifp->if_mtu = ifr->ifr_mtu;
910 adapter->max_frame_size =
911 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
912 ixgbe_init_locked(adapter);
913 IXGBE_CORE_UNLOCK(adapter);
917 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
918 IXGBE_CORE_LOCK(adapter);
919 if (ifp->if_flags & IFF_UP) {
920 if ((ifp->if_flags & IFF_RUNNING)) {
921 if ((ifp->if_flags ^ adapter->if_flags) &
922 (IFF_PROMISC | IFF_ALLMULTI)) {
923 ixgbe_set_promisc(adapter);
926 ixgbe_init_locked(adapter);
928 if (ifp->if_flags & IFF_RUNNING)
930 adapter->if_flags = ifp->if_flags;
931 IXGBE_CORE_UNLOCK(adapter);
935 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
936 if (ifp->if_flags & IFF_RUNNING) {
937 IXGBE_CORE_LOCK(adapter);
938 ixgbe_disable_intr(adapter);
939 ixgbe_set_multi(adapter);
940 ixgbe_enable_intr(adapter);
941 IXGBE_CORE_UNLOCK(adapter);
946 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
947 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
951 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
952 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
953 if (mask & IFCAP_HWCSUM)
954 ifp->if_capenable ^= IFCAP_HWCSUM;
955 if (mask & IFCAP_TSO4)
956 ifp->if_capenable ^= IFCAP_TSO4;
957 if (mask & IFCAP_TSO6)
958 ifp->if_capenable ^= IFCAP_TSO6;
960 if (mask & IFCAP_LRO)
961 ifp->if_capenable ^= IFCAP_LRO;
963 if (mask & IFCAP_VLAN_HWTAGGING)
964 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
965 if (mask & IFCAP_VLAN_HWFILTER)
966 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
968 if (mask & IFCAP_VLAN_HWTSO)
969 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
971 if (ifp->if_flags & IFF_RUNNING) {
972 IXGBE_CORE_LOCK(adapter);
973 ixgbe_init_locked(adapter);
974 IXGBE_CORE_UNLOCK(adapter);
977 VLAN_CAPABILITIES(ifp);
983 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
984 error = ether_ioctl(ifp, command, data);
991 /*********************************************************************
994 * This routine is used in two ways. It is used by the stack as
995 * init entry point in network interface structure. It is also used
996 * by the driver as a hw/sw initialization routine to get to a
999 * return 0 on success, positive on failure
1000 **********************************************************************/
1001 #define IXGBE_MHADD_MFS_SHIFT 16
1004 ixgbe_init_locked(struct adapter *adapter)
1006 struct ifnet *ifp = adapter->ifp;
1007 device_t dev = adapter->dev;
1008 struct ixgbe_hw *hw = &adapter->hw;
1009 u32 k, txdctl, mhadd, gpie;
1012 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1013 INIT_DEBUGOUT("ixgbe_init: begin");
1014 hw->adapter_stopped = FALSE;
1015 ixgbe_stop_adapter(hw);
1016 callout_stop(&adapter->timer);
1018 /* reprogram the RAR[0] in case user changed it. */
1019 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1021 /* Get the latest mac address, User can use a LAA */
1022 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1023 IXGBE_ETH_LENGTH_OF_ADDRESS);
1024 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1025 hw->addr_ctrl.rar_used_count = 1;
1027 /* Set the various hardware offload abilities */
1028 ifp->if_hwassist = 0;
1029 if (ifp->if_capenable & IFCAP_TSO)
1030 ifp->if_hwassist |= CSUM_TSO;
1031 if (ifp->if_capenable & IFCAP_TXCSUM) {
1032 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1034 if (hw->mac.type != ixgbe_mac_82598EB)
1035 ifp->if_hwassist |= CSUM_SCTP;
1039 /* Prepare transmit descriptors and buffers */
1040 if (ixgbe_setup_transmit_structures(adapter)) {
1041 device_printf(dev,"Could not setup transmit structures\n");
1042 ixgbe_stop(adapter);
1047 ixgbe_initialize_transmit_units(adapter);
1049 /* Setup Multicast table */
1050 ixgbe_set_multi(adapter);
1053 ** Determine the correct mbuf pool
1054 ** for doing jumbo/headersplit
1056 if (adapter->max_frame_size <= 2048)
1057 adapter->rx_mbuf_sz = MCLBYTES;
1058 else if (adapter->max_frame_size <= 4096)
1059 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1060 else if (adapter->max_frame_size <= 9216)
1061 adapter->rx_mbuf_sz = MJUM9BYTES;
1063 adapter->rx_mbuf_sz = MJUM16BYTES;
1065 /* Prepare receive descriptors and buffers */
1066 if (ixgbe_setup_receive_structures(adapter)) {
1067 device_printf(dev,"Could not setup receive structures\n");
1068 ixgbe_stop(adapter);
1072 /* Configure RX settings */
1073 ixgbe_initialize_receive_units(adapter);
1075 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1077 /* Enable Fan Failure Interrupt */
1078 gpie |= IXGBE_SDP1_GPIEN;
1080 /* Add for Module detection */
1081 if (hw->mac.type == ixgbe_mac_82599EB)
1082 gpie |= IXGBE_SDP2_GPIEN;
1084 /* Thermal Failure Detection */
1085 if (hw->mac.type == ixgbe_mac_X540)
1086 gpie |= IXGBE_SDP0_GPIEN;
1088 if (adapter->msix > 1) {
1089 /* Enable Enhanced MSIX mode */
1090 gpie |= IXGBE_GPIE_MSIX_MODE;
1091 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1094 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1097 if (ifp->if_mtu > ETHERMTU) {
1098 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1099 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1100 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1101 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1104 /* Now enable all the queues */
1106 for (int i = 0; i < adapter->num_queues; i++) {
1107 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1108 txdctl |= IXGBE_TXDCTL_ENABLE;
1109 /* Set WTHRESH to 8, burst writeback */
1110 txdctl |= (8 << 16);
1112 * When the internal queue falls below PTHRESH (32),
1113 * start prefetching as long as there are at least
1114 * HTHRESH (1) buffers ready. The values are taken
1115 * from the Intel linux driver 3.8.21.
1116 * Prefetching enables tx line rate even with 1 queue.
1118 txdctl |= (32 << 0) | (1 << 8);
1119 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1122 for (int i = 0; i < adapter->num_queues; i++) {
1123 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1124 if (hw->mac.type == ixgbe_mac_82598EB) {
1130 rxdctl &= ~0x3FFFFF;
1133 rxdctl |= IXGBE_RXDCTL_ENABLE;
1134 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1135 for (k = 0; k < 10; k++) {
1136 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1137 IXGBE_RXDCTL_ENABLE)
1145 * In netmap mode, we must preserve the buffers made
1146 * available to userspace before the if_init()
1147 * (this is true by default on the TX side, because
1148 * init makes all buffers available to userspace).
1150 * netmap_reset() and the device specific routines
1151 * (e.g. ixgbe_setup_receive_rings()) map these
1152 * buffers at the end of the NIC ring, so here we
1153 * must set the RDT (tail) register to make sure
1154 * they are not overwritten.
1156 * In this driver the NIC ring starts at RDH = 0,
1157 * RDT points to the last slot available for reception (?),
1158 * so RDT = num_rx_desc - 1 means the whole ring is available.
1160 if (ifp->if_capenable & IFCAP_NETMAP) {
1161 struct netmap_adapter *na = NA(adapter->ifp);
1162 struct netmap_kring *kring = &na->rx_rings[i];
1163 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1165 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1167 #endif /* DEV_NETMAP */
1168 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1171 /* Set up VLAN support and filter */
1172 ixgbe_setup_vlan_hw_support(adapter);
1174 /* Enable Receive engine */
1175 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1176 if (hw->mac.type == ixgbe_mac_82598EB)
1177 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1178 rxctrl |= IXGBE_RXCTRL_RXEN;
1179 ixgbe_enable_rx_dma(hw, rxctrl);
1181 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1183 /* Set up MSI/X routing */
1184 if (ixgbe_enable_msix) {
1185 ixgbe_configure_ivars(adapter);
1186 /* Set up auto-mask */
1187 if (hw->mac.type == ixgbe_mac_82598EB)
1188 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1190 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1191 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1193 } else { /* Simple settings for Legacy/MSI */
1194 ixgbe_set_ivar(adapter, 0, 0, 0);
1195 ixgbe_set_ivar(adapter, 0, 0, 1);
1196 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1200 /* Init Flow director */
1201 if (hw->mac.type != ixgbe_mac_82598EB) {
1202 u32 hdrm = 32 << fdir_pballoc;
1204 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1205 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1210 ** Check on any SFP devices that
1211 ** need to be kick-started
1213 if (hw->phy.type == ixgbe_phy_none) {
1214 int err = hw->phy.ops.identify(hw);
1215 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1217 "Unsupported SFP+ module type was detected.\n");
1222 /* Set moderation on the Link interrupt */
1223 ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1225 /* Config/Enable Link */
1226 ixgbe_config_link(adapter);
1228 /* Hardware Packet Buffer & Flow Control setup */
1230 u32 rxpb, frame, size, tmp;
1232 frame = adapter->max_frame_size;
1234 /* Calculate High Water */
1235 if (hw->mac.type == ixgbe_mac_X540)
1236 tmp = IXGBE_DV_X540(frame, frame);
1238 tmp = IXGBE_DV(frame, frame);
1239 size = IXGBE_BT2KB(tmp);
1240 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1241 hw->fc.high_water[0] = rxpb - size;
1243 /* Now calculate Low Water */
1244 if (hw->mac.type == ixgbe_mac_X540)
1245 tmp = IXGBE_LOW_DV_X540(frame);
1247 tmp = IXGBE_LOW_DV(frame);
1248 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1250 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1251 hw->fc.pause_time = IXGBE_FC_PAUSE;
1252 hw->fc.send_xon = TRUE;
1254 /* Initialize the FC settings */
1257 /* And now turn on interrupts */
1258 ixgbe_enable_intr(adapter);
1260 /* Now inform the stack we're ready */
1261 ifp->if_flags |= IFF_RUNNING;
1262 ifp->if_flags &= ~IFF_OACTIVE;
1268 ixgbe_init(void *arg)
1270 struct adapter *adapter = arg;
1272 IXGBE_CORE_LOCK(adapter);
1273 ixgbe_init_locked(adapter);
1274 IXGBE_CORE_UNLOCK(adapter);
1281 ** MSIX Interrupt Handlers and Tasklets
1286 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1288 struct ixgbe_hw *hw = &adapter->hw;
1289 u64 queue = (u64)(1 << vector);
1292 if (hw->mac.type == ixgbe_mac_82598EB) {
1293 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1294 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1296 mask = (queue & 0xFFFFFFFF);
1298 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1299 mask = (queue >> 32);
1301 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1306 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1308 struct ixgbe_hw *hw = &adapter->hw;
1309 u64 queue = (u64)(1 << vector);
1312 if (hw->mac.type == ixgbe_mac_82598EB) {
1313 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1314 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1316 mask = (queue & 0xFFFFFFFF);
1318 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1319 mask = (queue >> 32);
1321 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1326 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1330 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1331 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1332 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1334 mask = (queues & 0xFFFFFFFF);
1335 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1336 mask = (queues >> 32);
1337 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1341 /*********************************************************************
1343 * Legacy Interrupt Service routine
1345 **********************************************************************/
1348 ixgbe_legacy_irq(void *arg)
1350 struct ix_queue *que = arg;
1351 struct adapter *adapter = que->adapter;
1352 struct ixgbe_hw *hw = &adapter->hw;
1353 struct tx_ring *txr = adapter->tx_rings;
1357 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1360 if (reg_eicr == 0) {
1361 ixgbe_enable_intr(adapter);
1365 ixgbe_rxeof(que, adapter->rx_process_limit);
1369 if (!ifq_is_empty(&adapter->ifp->if_snd))
1370 ixgbe_start_locked(txr, adapter->ifp);
1371 IXGBE_TX_UNLOCK(txr);
1373 /* Check for fan failure */
1374 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1375 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1376 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1377 "REPLACE IMMEDIATELY!!\n");
1378 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1381 /* Link status change */
1382 if (reg_eicr & IXGBE_EICR_LSC) {
1383 ixgbe_check_link(&adapter->hw,
1384 &adapter->link_speed, &adapter->link_up, 0);
1385 ixgbe_update_link_status(adapter);
1388 ixgbe_enable_intr(adapter);
1392 /*********************************************************************
1394 * MSIX Queue Interrupt Service routine
1396 **********************************************************************/
1398 ixgbe_msix_que(void *arg)
1400 struct ix_queue *que = arg;
1401 struct adapter *adapter = que->adapter;
1402 struct tx_ring *txr = que->txr;
1404 ixgbe_disable_queue(adapter, que->msix);
1407 ixgbe_rxeof(que, adapter->rx_process_limit);
1411 if (!ifq_is_empty(&adapter->ifp->if_snd))
1412 ixgbe_start_locked(txr, adapter->ifp);
1413 IXGBE_TX_UNLOCK(txr);
1415 /* Reenable this interrupt */
1416 ixgbe_enable_queue(adapter, que->msix);
1421 ixgbe_msix_link(void *arg)
1423 struct adapter *adapter = arg;
1424 struct ixgbe_hw *hw = &adapter->hw;
1427 ++adapter->link_irq;
1429 /* First get the cause */
1430 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1431 /* Clear interrupt with write */
1432 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1434 /* Link status change */
1435 if (reg_eicr & IXGBE_EICR_LSC) {
1436 ixgbe_check_link(&adapter->hw,
1437 &adapter->link_speed, &adapter->link_up, 0);
1438 ixgbe_update_link_status(adapter);
1441 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1443 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1444 /* This is probably overkill :) */
1445 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1447 /* Disable the interrupt */
1448 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1449 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1452 if (reg_eicr & IXGBE_EICR_ECC) {
1453 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1454 "Please Reboot!!\n");
1455 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1458 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1459 /* Clear the interrupt */
1460 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1461 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1462 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1463 /* Clear the interrupt */
1464 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1465 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1469 /* Check for fan failure */
1470 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1471 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1472 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1473 "REPLACE IMMEDIATELY!!\n");
1474 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1477 /* Check for over temp condition */
1478 if ((hw->mac.type == ixgbe_mac_X540) &&
1479 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1480 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1481 "PHY IS SHUT DOWN!!\n");
1482 device_printf(adapter->dev, "System shutdown required\n");
1483 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1486 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1490 /*********************************************************************
1492 * Media Ioctl callback
1494 * This routine is called whenever the user queries the status of
1495 * the interface using ifconfig.
1497 **********************************************************************/
1499 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1501 struct adapter *adapter = ifp->if_softc;
1503 INIT_DEBUGOUT("ixgbe_media_status: begin");
1504 IXGBE_CORE_LOCK(adapter);
1505 ixgbe_update_link_status(adapter);
1507 ifmr->ifm_status = IFM_AVALID;
1508 ifmr->ifm_active = IFM_ETHER;
1510 if (!adapter->link_active) {
1511 IXGBE_CORE_UNLOCK(adapter);
1515 ifmr->ifm_status |= IFM_ACTIVE;
1517 switch (adapter->link_speed) {
1518 case IXGBE_LINK_SPEED_100_FULL:
1519 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1521 case IXGBE_LINK_SPEED_1GB_FULL:
1522 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1524 case IXGBE_LINK_SPEED_10GB_FULL:
1525 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1529 IXGBE_CORE_UNLOCK(adapter);
1534 /*********************************************************************
1536 * Media Ioctl callback
1538 * This routine is called when the user changes speed/duplex using
1539 * media/mediopt option with ifconfig.
1541 **********************************************************************/
1543 ixgbe_media_change(struct ifnet * ifp)
1545 struct adapter *adapter = ifp->if_softc;
1546 struct ifmedia *ifm = &adapter->media;
1548 INIT_DEBUGOUT("ixgbe_media_change: begin");
1550 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1553 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1555 adapter->hw.phy.autoneg_advertised =
1556 IXGBE_LINK_SPEED_100_FULL |
1557 IXGBE_LINK_SPEED_1GB_FULL |
1558 IXGBE_LINK_SPEED_10GB_FULL;
1561 device_printf(adapter->dev, "Only auto media type\n");
1568 /*********************************************************************
1570 * This routine maps the mbufs to tx descriptors, allowing the
1571 * TX engine to transmit the packets.
1572 * - return 0 on success, positive on failure
1574 **********************************************************************/
1577 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1579 struct adapter *adapter = txr->adapter;
1580 u32 olinfo_status = 0, cmd_type_len;
1582 int i, j, error, nsegs, maxsegs;
1583 int first, last = 0;
1584 struct mbuf *m_head;
1585 bus_dma_segment_t segs[adapter->num_segs];
1587 struct ixgbe_tx_buf *txbuf;
1588 union ixgbe_adv_tx_desc *txd = NULL;
1592 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1593 error = ixgbe_tso_pullup(txr, m_headp);
1599 /* Basic descriptor defines */
1600 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1601 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1603 if (m_head->m_flags & M_VLANTAG)
1604 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1607 * Important to capture the first descriptor
1608 * used because it will contain the index of
1609 * the one we tell the hardware to report back
1611 first = txr->next_avail_desc;
1612 txbuf = &txr->tx_buffers[first];
1616 * Map the packet for DMA.
1618 maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1619 if (maxsegs > adapter->num_segs)
1620 maxsegs = adapter->num_segs;
1622 error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1623 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1625 if (error == ENOBUFS)
1626 adapter->mbuf_defrag_failed++;
1628 adapter->no_tx_dma_setup++;
1635 /* Make certain there are enough descriptors */
1636 if (nsegs > txr->tx_avail - 2) {
1637 txr->no_desc_avail++;
1644 ** Set up the appropriate offload context
1645 ** this becomes the first descriptor of
1648 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1649 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1650 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1651 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1652 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1656 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1657 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1659 #ifdef IXGBE_IEEE1588
1660 /* This is changing soon to an mtag detection */
1661 if (we detect this mbuf has a TSTAMP mtag)
1662 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1666 /* Do the flow director magic */
1667 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1669 if (txr->atr_count >= atr_sample_rate) {
1670 ixgbe_atr(txr, m_head);
1675 /* Record payload length */
1677 olinfo_status |= m_head->m_pkthdr.len <<
1678 IXGBE_ADVTXD_PAYLEN_SHIFT;
1680 i = txr->next_avail_desc;
1681 for (j = 0; j < nsegs; j++) {
1685 txbuf = &txr->tx_buffers[i];
1686 txd = &txr->tx_base[i];
1687 seglen = segs[j].ds_len;
1688 segaddr = htole64(segs[j].ds_addr);
1690 txd->read.buffer_addr = segaddr;
1691 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1692 cmd_type_len |seglen);
1693 txd->read.olinfo_status = htole32(olinfo_status);
1694 last = i; /* descriptor that will get completion IRQ */
1696 if (++i == adapter->num_tx_desc)
1699 txbuf->m_head = NULL;
1700 txbuf->eop_index = -1;
1703 txd->read.cmd_type_len |=
1704 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1705 txr->tx_avail -= nsegs;
1706 txr->next_avail_desc = i;
1708 txbuf->m_head = m_head;
1709 /* Swap the dma map between the first and last descriptor */
1710 txr->tx_buffers[first].map = txbuf->map;
1712 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1714 /* Set the index of the descriptor that will be marked done */
1715 txbuf = &txr->tx_buffers[first];
1716 txbuf->eop_index = last;
1718 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1719 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1721 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1722 * hardware that this frame is available to transmit.
1724 ++txr->total_packets;
1725 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1730 bus_dmamap_unload(txr->txtag, txbuf->map);
1736 ixgbe_set_promisc(struct adapter *adapter)
1739 struct ifnet *ifp = adapter->ifp;
1741 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1742 reg_rctl &= (~IXGBE_FCTRL_UPE);
1743 reg_rctl &= (~IXGBE_FCTRL_MPE);
1744 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1746 if (ifp->if_flags & IFF_PROMISC) {
1747 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1748 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1749 } else if (ifp->if_flags & IFF_ALLMULTI) {
1750 reg_rctl |= IXGBE_FCTRL_MPE;
1751 reg_rctl &= ~IXGBE_FCTRL_UPE;
1752 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1758 /*********************************************************************
1761 * This routine is called whenever multicast address list is updated.
1763 **********************************************************************/
1764 #define IXGBE_RAR_ENTRIES 16
1767 ixgbe_set_multi(struct adapter *adapter)
1772 struct ifmultiaddr *ifma;
1774 struct ifnet *ifp = adapter->ifp;
1776 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1779 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1780 MAX_NUM_MULTICAST_ADDRESSES);
1782 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1783 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1784 if (ifp->if_flags & IFF_PROMISC)
1785 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1786 else if (ifp->if_flags & IFF_ALLMULTI) {
1787 fctrl |= IXGBE_FCTRL_MPE;
1788 fctrl &= ~IXGBE_FCTRL_UPE;
1790 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1792 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1794 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1795 if (ifma->ifma_addr->sa_family != AF_LINK)
1797 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1798 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1799 IXGBE_ETH_LENGTH_OF_ADDRESS);
1804 ixgbe_update_mc_addr_list(&adapter->hw,
1805 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1811 * This is an iterator function now needed by the multicast
1812 * shared code. It simply feeds the shared code routine the
1813 * addresses in the array of ixgbe_set_multi() one by one.
1816 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1818 u8 *addr = *update_ptr;
1822 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1823 *update_ptr = newptr;
1828 /*********************************************************************
1831 * This routine checks for link status,updates statistics,
1832 * and runs the watchdog check.
1834 **********************************************************************/
1837 ixgbe_local_timer(void *arg)
1839 struct adapter *adapter = arg;
1840 device_t dev = adapter->dev;
1841 struct ifnet *ifp = adapter->ifp;
1842 struct ix_queue *que = adapter->queues;
1843 struct tx_ring *txr = adapter->tx_rings;
1844 int hung, busy, paused;
1846 IXGBE_CORE_LOCK(adapter);
1847 hung = busy = paused = 0;
1849 /* Check for pluggable optics */
1850 if (adapter->sfp_probe)
1851 if (!ixgbe_sfp_probe(adapter))
1852 goto out; /* Nothing to do */
1854 ixgbe_update_link_status(adapter);
1855 ixgbe_update_stats_counters(adapter);
1858 * If the interface has been paused
1859 * then don't do the watchdog check
1861 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1865 ** Check the TX queues status
1866 ** - central locked handling of OACTIVE
1867 ** - watchdog only if all queues show hung
1869 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1870 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1873 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1876 /* Only truely watchdog if all queues show hung */
1877 if (hung == adapter->num_queues)
1879 /* Only turn off the stack flow when ALL are depleted */
1880 if (busy == adapter->num_queues)
1881 ifp->if_flags |= IFF_OACTIVE;
1882 else if ((ifp->if_flags & IFF_OACTIVE) &&
1883 (busy < adapter->num_queues))
1884 ifp->if_flags &= ~IFF_OACTIVE;
1887 ixgbe_rearm_queues(adapter, adapter->que_mask);
1888 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1889 IXGBE_CORE_UNLOCK(adapter);
1893 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1894 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1895 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1896 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1897 device_printf(dev,"TX(%d) desc avail = %d,"
1898 "Next TX to Clean = %d\n",
1899 txr->me, txr->tx_avail, txr->next_to_clean);
1900 adapter->ifp->if_flags &= ~IFF_RUNNING;
1901 adapter->watchdog_events++;
1902 ixgbe_init_locked(adapter);
1904 IXGBE_CORE_UNLOCK(adapter);
1908 ** Note: this routine updates the OS on the link state
1909 ** the real check of the hardware only happens with
1910 ** a link interrupt.
1913 ixgbe_update_link_status(struct adapter *adapter)
1915 struct ifnet *ifp = adapter->ifp;
1916 struct tx_ring *txr = adapter->tx_rings;
1917 device_t dev = adapter->dev;
1920 if (adapter->link_up){
1921 if (adapter->link_active == FALSE) {
1923 device_printf(dev,"Link is up %d Gbps %s \n",
1924 ((adapter->link_speed == 128)? 10:1),
1926 adapter->link_active = TRUE;
1927 /* Update any Flow Control changes */
1928 ixgbe_fc_enable(&adapter->hw);
1929 ifp->if_link_state = LINK_STATE_UP;
1930 if_link_state_change(ifp);
1932 } else { /* Link down */
1933 if (adapter->link_active == TRUE) {
1935 device_printf(dev,"Link is Down\n");
1936 ifp->if_link_state = LINK_STATE_DOWN;
1937 if_link_state_change(ifp);
1938 adapter->link_active = FALSE;
1939 for (int i = 0; i < adapter->num_queues;
1941 txr->queue_status = IXGBE_QUEUE_IDLE;
1949 /*********************************************************************
1951 * This routine disables all traffic on the adapter by issuing a
1952 * global reset on the MAC and deallocates TX/RX buffers.
1954 **********************************************************************/
1957 ixgbe_stop(void *arg)
1960 struct adapter *adapter = arg;
1961 struct ixgbe_hw *hw = &adapter->hw;
1964 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1966 INIT_DEBUGOUT("ixgbe_stop: begin\n");
1967 ixgbe_disable_intr(adapter);
1968 callout_stop(&adapter->timer);
1970 /* Let the stack know...*/
1971 ifp->if_flags &= ~IFF_RUNNING;
1972 ifp->if_flags |= IFF_OACTIVE;
1975 hw->adapter_stopped = FALSE;
1976 ixgbe_stop_adapter(hw);
1977 /* Turn off the laser */
1978 if (hw->phy.multispeed_fiber)
1979 ixgbe_disable_tx_laser(hw);
1981 /* reprogram the RAR[0] in case user changed it. */
1982 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1988 /*********************************************************************
1990 * Determine hardware revision.
1992 **********************************************************************/
1994 ixgbe_identify_hardware(struct adapter *adapter)
1996 device_t dev = adapter->dev;
1997 struct ixgbe_hw *hw = &adapter->hw;
1999 /* Save off the information about this board */
2000 hw->vendor_id = pci_get_vendor(dev);
2001 hw->device_id = pci_get_device(dev);
2002 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2003 hw->subsystem_vendor_id =
2004 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2005 hw->subsystem_device_id =
2006 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2008 /* We need this here to set the num_segs below */
2009 ixgbe_set_mac_type(hw);
2011 /* Pick up the 82599 and VF settings */
2012 if (hw->mac.type != ixgbe_mac_82598EB) {
2013 hw->phy.smart_speed = ixgbe_smart_speed;
2014 adapter->num_segs = IXGBE_82599_SCATTER;
2016 adapter->num_segs = IXGBE_82598_SCATTER;
2021 /*********************************************************************
2023 * Determine optic type
2025 **********************************************************************/
2027 ixgbe_setup_optics(struct adapter *adapter)
2029 struct ixgbe_hw *hw = &adapter->hw;
2032 layer = ixgbe_get_supported_physical_layer(hw);
2034 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2035 adapter->optics = IFM_10G_T;
2039 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2040 adapter->optics = IFM_1000_T;
2044 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2045 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2046 adapter->optics = IFM_10G_LR;
2050 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2051 adapter->optics = IFM_10G_SR;
2055 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2056 adapter->optics = IFM_10G_TWINAX;
2060 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2061 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2062 adapter->optics = IFM_10G_CX4;
2066 /* If we get here just set the default */
2067 adapter->optics = IFM_ETHER | IFM_AUTO;
2071 /*********************************************************************
2073 * Setup the Legacy or MSI Interrupt handler
2075 **********************************************************************/
2077 ixgbe_allocate_legacy(struct adapter *adapter)
2079 device_t dev = adapter->dev;
2080 struct ix_queue *que = adapter->queues;
2082 unsigned int intr_flags;
2085 if (adapter->msix == 1)
2088 /* Try allocating a MSI interrupt first */
2089 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2092 /* We allocate a single interrupt resource */
2093 adapter->res = bus_alloc_resource_any(dev,
2094 SYS_RES_IRQ, &rid, intr_flags);
2095 if (adapter->res == NULL) {
2096 device_printf(dev, "Unable to allocate bus resource: "
2101 /* Tasklets for Link, SFP and Multispeed Fiber */
2102 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2103 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2105 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2107 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2108 taskqueue_thread_enqueue, &adapter->tq);
2109 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2110 device_get_nameunit(adapter->dev));
2112 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2113 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2114 device_printf(dev, "Failed to register fast interrupt "
2115 "handler: %d\n", error);
2116 taskqueue_free(adapter->tq);
2120 /* For simplicity in the handlers */
2121 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2127 /*********************************************************************
2129 * Setup MSIX Interrupt resources and handlers
2131 **********************************************************************/
2133 ixgbe_allocate_msix(struct adapter *adapter)
2135 device_t dev = adapter->dev;
2136 struct ix_queue *que = adapter->queues;
2137 int error, rid, vector = 0;
2140 error = pci_setup_msix(dev);
2142 device_printf(dev, "MSI-X setup failed\n");
2146 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2150 ** Bind the msix vector, and thus the
2151 ** ring to the corresponding cpu.
2153 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2155 device_printf(dev, "pci_alloc_msix_vector failed\n");
2159 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2160 RF_SHAREABLE | RF_ACTIVE);
2161 if (que->res == NULL) {
2162 device_printf(dev,"Unable to allocate"
2163 " bus resource: que interrupt [%d]\n", vector);
2166 /* Set the handler function */
2167 ksnprintf(desc, sizeof(desc), "%s que %d",
2168 device_get_nameunit(dev), i);
2169 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2170 ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2173 device_printf(dev, "Failed to register QUE handler");
2177 adapter->que_mask |= (u64)(1 << que->msix);
2180 /* and Link, bind vector to cpu #0 */
2182 error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2184 device_printf(dev, "pci_alloc_msix_vector failed\n");
2187 adapter->res = bus_alloc_resource_any(dev,
2188 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2189 if (!adapter->res) {
2190 device_printf(dev,"Unable to allocate"
2191 " bus resource: Link interrupt [%d]\n", rid);
2194 /* Set the link handler function */
2195 error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2196 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2199 adapter->res = NULL;
2200 device_printf(dev, "Failed to register LINK handler");
2203 pci_enable_msix(dev);
2205 adapter->linkvec = vector;
2206 /* Tasklets for Link, SFP and Multispeed Fiber */
2207 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2208 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2210 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2212 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2213 taskqueue_thread_enqueue, &adapter->tq);
2214 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2215 device_get_nameunit(adapter->dev));
2221 * Setup Either MSI/X or MSI
2224 ixgbe_setup_msix(struct adapter *adapter)
2226 device_t dev = adapter->dev;
2227 int rid, want, queues, msgs;
2229 /* Override by tuneable */
2230 if (ixgbe_enable_msix == 0)
2233 /* First try MSI/X */
2234 rid = PCIR_BAR(MSIX_82598_BAR);
2235 adapter->msix_mem = bus_alloc_resource_any(dev,
2236 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2237 if (!adapter->msix_mem) {
2238 rid += 4; /* 82599 maps in higher BAR */
2239 adapter->msix_mem = bus_alloc_resource_any(dev,
2240 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2242 if (!adapter->msix_mem) {
2243 /* May not be enabled */
2244 device_printf(adapter->dev,
2245 "Unable to map MSIX table \n");
2249 msgs = pci_msix_count(dev);
2250 if (msgs == 0) { /* system has msix disabled */
2251 bus_release_resource(dev, SYS_RES_MEMORY,
2252 rid, adapter->msix_mem);
2253 adapter->msix_mem = NULL;
2257 /* Figure out a reasonable auto config value */
2258 queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2260 if (ixgbe_num_queues != 0)
2261 queues = ixgbe_num_queues;
2262 /* Set max queues to 8 when autoconfiguring */
2263 else if ((ixgbe_num_queues == 0) && (queues > 8))
2267 ** Want one vector (RX/TX pair) per queue
2268 ** plus an additional for Link.
2274 device_printf(adapter->dev,
2275 "MSIX Configuration Problem, "
2276 "%d vectors but %d queues wanted!\n",
2278 return (0); /* Will go to Legacy setup */
2281 device_printf(adapter->dev,
2282 "Using MSIX interrupts with %d vectors\n", msgs);
2283 adapter->num_queues = queues;
2287 msgs = pci_msi_count(dev);
2293 ixgbe_allocate_pci_resources(struct adapter *adapter)
2296 device_t dev = adapter->dev;
2299 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2302 if (!(adapter->pci_mem)) {
2303 device_printf(dev,"Unable to allocate bus resource: memory\n");
2307 adapter->osdep.mem_bus_space_tag =
2308 rman_get_bustag(adapter->pci_mem);
2309 adapter->osdep.mem_bus_space_handle =
2310 rman_get_bushandle(adapter->pci_mem);
2311 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2313 /* Legacy defaults */
2314 adapter->num_queues = 1;
2315 adapter->hw.back = &adapter->osdep;
2318 ** Now setup MSI or MSI/X, should
2319 ** return us the number of supported
2320 ** vectors. (Will be 1 for MSI)
2322 adapter->msix = ixgbe_setup_msix(adapter);
2327 ixgbe_free_pci_resources(struct adapter * adapter)
2329 struct ix_queue *que = adapter->queues;
2330 device_t dev = adapter->dev;
2333 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2334 memrid = PCIR_BAR(MSIX_82598_BAR);
2336 memrid = PCIR_BAR(MSIX_82599_BAR);
2339 ** There is a slight possibility of a failure mode
2340 ** in attach that will result in entering this function
2341 ** before interrupt resources have been initialized, and
2342 ** in that case we do not want to execute the loops below
2343 ** We can detect this reliably by the state of the adapter
2346 if (adapter->res == NULL)
2350 ** Release all msix queue resources:
2352 for (int i = 0; i < adapter->num_queues; i++, que++) {
2353 rid = que->msix + 1;
2354 if (que->tag != NULL) {
2355 bus_teardown_intr(dev, que->res, que->tag);
2358 if (que->res != NULL)
2359 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2363 /* Clean the Legacy or Link interrupt last */
2364 if (adapter->linkvec) /* we are doing MSIX */
2365 rid = adapter->linkvec + 1;
2367 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2369 if (adapter->tag != NULL) {
2370 bus_teardown_intr(dev, adapter->res, adapter->tag);
2371 adapter->tag = NULL;
2373 if (adapter->res != NULL)
2374 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2375 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2376 pci_release_msi(adapter->dev);
2380 pci_release_msi(dev);
2382 if (adapter->msix_mem != NULL)
2383 bus_release_resource(dev, SYS_RES_MEMORY,
2384 memrid, adapter->msix_mem);
2386 if (adapter->pci_mem != NULL)
2387 bus_release_resource(dev, SYS_RES_MEMORY,
2388 PCIR_BAR(0), adapter->pci_mem);
2393 /*********************************************************************
2395 * Setup networking device structure and register an interface.
2397 **********************************************************************/
2399 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2401 struct ixgbe_hw *hw = &adapter->hw;
2404 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2406 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2408 device_printf(dev, "can not allocate ifnet structure\n");
2411 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2412 ifp->if_baudrate = 1000000000;
2413 ifp->if_init = ixgbe_init;
2414 ifp->if_softc = adapter;
2415 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2416 ifp->if_ioctl = ixgbe_ioctl;
2417 ifp->if_start = ixgbe_start;
2418 #if 0 /* __FreeBSD_version >= 800000 */
2419 ifp->if_transmit = ixgbe_mq_start;
2420 ifp->if_qflush = ixgbe_qflush;
2422 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2424 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2426 adapter->max_frame_size =
2427 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2430 * Tell the upper layer(s) we support long frames.
2432 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2434 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2435 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2436 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2441 ifp->if_capenable = ifp->if_capabilities;
2443 /* Don't enable LRO by default */
2445 ifp->if_capabilities |= IFCAP_LRO;
2449 ** Don't turn this on by default, if vlans are
2450 ** created on another pseudo device (eg. lagg)
2451 ** then vlan events are not passed thru, breaking
2452 ** operation, but with HW FILTER off it works. If
2453 ** using vlans directly on the ixgbe driver you can
2454 ** enable this and get full hardware tag filtering.
2456 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2459 * Specify the media types supported by this adapter and register
2460 * callbacks to update media and link information
2462 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2463 ixgbe_media_status);
2464 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2465 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2466 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2467 ifmedia_add(&adapter->media,
2468 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2469 ifmedia_add(&adapter->media,
2470 IFM_ETHER | IFM_1000_T, 0, NULL);
2472 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2473 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2479 ixgbe_config_link(struct adapter *adapter)
2481 struct ixgbe_hw *hw = &adapter->hw;
2482 u32 autoneg, err = 0;
2483 bool sfp, negotiate;
2485 sfp = ixgbe_is_sfp(hw);
2488 if (hw->phy.multispeed_fiber) {
2489 hw->mac.ops.setup_sfp(hw);
2490 ixgbe_enable_tx_laser(hw);
2491 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2493 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2495 if (hw->mac.ops.check_link)
2496 err = ixgbe_check_link(hw, &autoneg,
2497 &adapter->link_up, FALSE);
2500 autoneg = hw->phy.autoneg_advertised;
2501 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2502 err = hw->mac.ops.get_link_capabilities(hw,
2503 &autoneg, &negotiate);
2506 if (hw->mac.ops.setup_link)
2507 err = hw->mac.ops.setup_link(hw, autoneg,
2508 negotiate, adapter->link_up);
2514 /********************************************************************
2515 * Manage DMA'able memory.
2516 *******************************************************************/
2518 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2522 *(bus_addr_t *) arg = segs->ds_addr;
2527 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2528 struct ixgbe_dma_alloc *dma, int mapflags)
2530 device_t dev = adapter->dev;
2533 r = bus_dma_tag_create(NULL, /* parent */
2534 DBA_ALIGN, 0, /* alignment, bounds */
2535 BUS_SPACE_MAXADDR, /* lowaddr */
2536 BUS_SPACE_MAXADDR, /* highaddr */
2537 NULL, NULL, /* filter, filterarg */
2540 size, /* maxsegsize */
2541 BUS_DMA_ALLOCNOW, /* flags */
2544 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2548 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2549 BUS_DMA_NOWAIT, &dma->dma_map);
2551 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2555 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2559 mapflags | BUS_DMA_NOWAIT);
2561 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2565 dma->dma_size = size;
2568 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2570 bus_dma_tag_destroy(dma->dma_tag);
2572 dma->dma_map = NULL;
2573 dma->dma_tag = NULL;
2578 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2580 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2581 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2582 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2583 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2584 bus_dma_tag_destroy(dma->dma_tag);
2588 /*********************************************************************
2590 * Allocate memory for the transmit and receive rings, and then
2591 * the descriptors associated with each, called only once at attach.
2593 **********************************************************************/
2595 ixgbe_allocate_queues(struct adapter *adapter)
2597 device_t dev = adapter->dev;
2598 struct ix_queue *que;
2599 struct tx_ring *txr;
2600 struct rx_ring *rxr;
2601 int rsize, tsize, error = IXGBE_SUCCESS;
2602 int txconf = 0, rxconf = 0;
2604 /* First allocate the top level queue structs */
2605 if (!(adapter->queues =
2606 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2607 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2608 device_printf(dev, "Unable to allocate queue memory\n");
2613 /* First allocate the TX ring struct memory */
2614 if (!(adapter->tx_rings =
2615 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2616 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2617 device_printf(dev, "Unable to allocate TX ring memory\n");
2622 /* Next allocate the RX */
2623 if (!(adapter->rx_rings =
2624 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2625 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2626 device_printf(dev, "Unable to allocate RX ring memory\n");
2631 /* For the ring itself */
2632 tsize = roundup2(adapter->num_tx_desc *
2633 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2636 * Now set up the TX queues, txconf is needed to handle the
2637 * possibility that things fail midcourse and we need to
2638 * undo memory gracefully
2640 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2641 /* Set up some basics */
2642 txr = &adapter->tx_rings[i];
2643 txr->adapter = adapter;
2646 /* Initialize the TX side lock */
2647 IXGBE_TX_LOCK_INIT(txr);
2649 if (ixgbe_dma_malloc(adapter, tsize,
2650 &txr->txdma, BUS_DMA_NOWAIT)) {
2652 "Unable to allocate TX Descriptor memory\n");
2656 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2657 bzero((void *)txr->tx_base, tsize);
2659 /* Now allocate transmit buffers for the ring */
2660 if (ixgbe_allocate_transmit_buffers(txr)) {
2662 "Critical Failure setting up transmit buffers\n");
2666 #if 0 /* __FreeBSD_version >= 800000 */
2667 /* Allocate a buf ring */
2668 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2669 M_WAITOK, &txr->tx_mtx);
2670 if (txr->br == NULL) {
2672 "Critical Failure setting up buf ring\n");
2680 * Next the RX queues...
2682 rsize = roundup2(adapter->num_rx_desc *
2683 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2684 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2685 rxr = &adapter->rx_rings[i];
2686 /* Set up some basics */
2687 rxr->adapter = adapter;
2690 /* Initialize the RX side lock */
2691 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2692 device_get_nameunit(dev), rxr->me);
2693 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2695 if (ixgbe_dma_malloc(adapter, rsize,
2696 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2698 "Unable to allocate RxDescriptor memory\n");
2702 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2703 bzero((void *)rxr->rx_base, rsize);
2705 /* Allocate receive buffers for the ring*/
2706 if (ixgbe_allocate_receive_buffers(rxr)) {
2708 "Critical Failure setting up receive buffers\n");
2715 ** Finally set up the queue holding structs
2717 for (int i = 0; i < adapter->num_queues; i++) {
2718 que = &adapter->queues[i];
2719 que->adapter = adapter;
2720 que->txr = &adapter->tx_rings[i];
2721 que->rxr = &adapter->rx_rings[i];
2727 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2728 ixgbe_dma_free(adapter, &rxr->rxdma);
2730 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2731 ixgbe_dma_free(adapter, &txr->txdma);
2732 kfree(adapter->rx_rings, M_DEVBUF);
2734 kfree(adapter->tx_rings, M_DEVBUF);
2736 kfree(adapter->queues, M_DEVBUF);
2741 /*********************************************************************
2743 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2744 * the information needed to transmit a packet on the wire. This is
2745 * called only once at attach, setup is done every reset.
2747 **********************************************************************/
2749 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2751 struct adapter *adapter = txr->adapter;
2752 device_t dev = adapter->dev;
2753 struct ixgbe_tx_buf *txbuf;
2757 * Setup DMA descriptor areas.
2759 if ((error = bus_dma_tag_create(
2761 1, 0, /* alignment, bounds */
2762 BUS_SPACE_MAXADDR, /* lowaddr */
2763 BUS_SPACE_MAXADDR, /* highaddr */
2764 NULL, NULL, /* filter, filterarg */
2765 IXGBE_TSO_SIZE, /* maxsize */
2766 adapter->num_segs, /* nsegments */
2767 PAGE_SIZE, /* maxsegsize */
2770 device_printf(dev,"Unable to allocate TX DMA tag\n");
2774 if (!(txr->tx_buffers =
2775 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2776 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2777 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2782 /* Create the descriptor buffer dma maps */
2783 txbuf = txr->tx_buffers;
2784 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2785 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2787 device_printf(dev, "Unable to create TX DMA map\n");
2794 /* We free all, it handles case where we are in the middle */
2795 ixgbe_free_transmit_structures(adapter);
2799 /*********************************************************************
2801 * Initialize a transmit ring.
2803 **********************************************************************/
2805 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2807 struct adapter *adapter = txr->adapter;
2808 struct ixgbe_tx_buf *txbuf;
2811 struct netmap_adapter *na = NA(adapter->ifp);
2812 struct netmap_slot *slot;
2813 #endif /* DEV_NETMAP */
2815 /* Clear the old ring contents */
2819 * (under lock): if in netmap mode, do some consistency
2820 * checks and set slot to entry 0 of the netmap ring.
2822 slot = netmap_reset(na, NR_TX, txr->me, 0);
2823 #endif /* DEV_NETMAP */
2824 bzero((void *)txr->tx_base,
2825 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2827 txr->next_avail_desc = 0;
2828 txr->next_to_clean = 0;
2830 /* Free any existing tx buffers. */
2831 txbuf = txr->tx_buffers;
2832 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2833 if (txbuf->m_head != NULL) {
2834 bus_dmamap_sync(txr->txtag, txbuf->map,
2835 BUS_DMASYNC_POSTWRITE);
2836 bus_dmamap_unload(txr->txtag, txbuf->map);
2837 m_freem(txbuf->m_head);
2838 txbuf->m_head = NULL;
2842 * In netmap mode, set the map for the packet buffer.
2843 * NOTE: Some drivers (not this one) also need to set
2844 * the physical buffer address in the NIC ring.
2845 * Slots in the netmap ring (indexed by "si") are
2846 * kring->nkr_hwofs positions "ahead" wrt the
2847 * corresponding slot in the NIC ring. In some drivers
2848 * (not here) nkr_hwofs can be negative. Function
2849 * netmap_idx_n2k() handles wraparounds properly.
2852 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2853 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2855 #endif /* DEV_NETMAP */
2856 /* Clear the EOP index */
2857 txbuf->eop_index = -1;
2861 /* Set the rate at which we sample packets */
2862 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2863 txr->atr_sample = atr_sample_rate;
2866 /* Set number of descriptors available */
2867 txr->tx_avail = adapter->num_tx_desc;
2869 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2870 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2871 IXGBE_TX_UNLOCK(txr);
2874 /*********************************************************************
2876 * Initialize all transmit rings.
2878 **********************************************************************/
2880 ixgbe_setup_transmit_structures(struct adapter *adapter)
2882 struct tx_ring *txr = adapter->tx_rings;
2884 for (int i = 0; i < adapter->num_queues; i++, txr++)
2885 ixgbe_setup_transmit_ring(txr);
2890 /*********************************************************************
2892 * Enable transmit unit.
2894 **********************************************************************/
2896 ixgbe_initialize_transmit_units(struct adapter *adapter)
2898 struct tx_ring *txr = adapter->tx_rings;
2899 struct ixgbe_hw *hw = &adapter->hw;
2901 /* Setup the Base and Length of the Tx Descriptor Ring */
2903 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2904 u64 tdba = txr->txdma.dma_paddr;
2907 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2908 (tdba & 0x00000000ffffffffULL));
2909 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2910 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2911 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2913 /* Setup the HW Tx Head and Tail descriptor pointers */
2914 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2915 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2917 /* Setup Transmit Descriptor Cmd Settings */
2918 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2919 txr->queue_status = IXGBE_QUEUE_IDLE;
2921 /* Disable Head Writeback */
2922 switch (hw->mac.type) {
2923 case ixgbe_mac_82598EB:
2924 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2926 case ixgbe_mac_82599EB:
2927 case ixgbe_mac_X540:
2929 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2932 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2933 switch (hw->mac.type) {
2934 case ixgbe_mac_82598EB:
2935 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2937 case ixgbe_mac_82599EB:
2938 case ixgbe_mac_X540:
2940 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2946 if (hw->mac.type != ixgbe_mac_82598EB) {
2947 u32 dmatxctl, rttdcs;
2948 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2949 dmatxctl |= IXGBE_DMATXCTL_TE;
2950 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2951 /* Disable arbiter to set MTQC */
2952 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2953 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2954 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2955 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2956 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2957 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2963 /*********************************************************************
2965 * Free all transmit rings.
2967 **********************************************************************/
2969 ixgbe_free_transmit_structures(struct adapter *adapter)
2971 struct tx_ring *txr = adapter->tx_rings;
2973 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2975 ixgbe_free_transmit_buffers(txr);
2976 ixgbe_dma_free(adapter, &txr->txdma);
2977 IXGBE_TX_UNLOCK(txr);
2978 IXGBE_TX_LOCK_DESTROY(txr);
2980 kfree(adapter->tx_rings, M_DEVBUF);
2983 /*********************************************************************
2985 * Free transmit ring related data structures.
2987 **********************************************************************/
2989 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2991 struct adapter *adapter = txr->adapter;
2992 struct ixgbe_tx_buf *tx_buffer;
2995 INIT_DEBUGOUT("free_transmit_ring: begin");
2997 if (txr->tx_buffers == NULL)
3000 tx_buffer = txr->tx_buffers;
3001 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3002 if (tx_buffer->m_head != NULL) {
3003 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3004 BUS_DMASYNC_POSTWRITE);
3005 bus_dmamap_unload(txr->txtag,
3007 m_freem(tx_buffer->m_head);
3008 tx_buffer->m_head = NULL;
3009 if (tx_buffer->map != NULL) {
3010 bus_dmamap_destroy(txr->txtag,
3012 tx_buffer->map = NULL;
3014 } else if (tx_buffer->map != NULL) {
3015 bus_dmamap_unload(txr->txtag,
3017 bus_dmamap_destroy(txr->txtag,
3019 tx_buffer->map = NULL;
3022 #if 0 /* __FreeBSD_version >= 800000 */
3023 if (txr->br != NULL)
3024 buf_ring_free(txr->br, M_DEVBUF);
3026 if (txr->tx_buffers != NULL) {
3027 kfree(txr->tx_buffers, M_DEVBUF);
3028 txr->tx_buffers = NULL;
3030 if (txr->txtag != NULL) {
3031 bus_dma_tag_destroy(txr->txtag);
3037 /*********************************************************************
3039 * Advanced Context Descriptor setup for VLAN or CSUM
3041 **********************************************************************/
3044 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3046 struct adapter *adapter = txr->adapter;
3047 struct ixgbe_adv_tx_context_desc *TXD;
3048 struct ixgbe_tx_buf *tx_buffer;
3049 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3050 struct ether_vlan_header *eh;
3052 struct ip6_hdr *ip6;
3053 int ehdrlen, ip_hlen = 0;
3056 bool offload = TRUE;
3057 int ctxd = txr->next_avail_desc;
3061 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3064 tx_buffer = &txr->tx_buffers[ctxd];
3065 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3068 ** In advanced descriptors the vlan tag must
3069 ** be placed into the descriptor itself.
3071 if (mp->m_flags & M_VLANTAG) {
3072 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3073 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3074 } else if (offload == FALSE)
3078 * Determine where frame payload starts.
3079 * Jump over vlan headers if already present,
3080 * helpful for QinQ too.
3082 eh = mtod(mp, struct ether_vlan_header *);
3083 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3084 etype = ntohs(eh->evl_proto);
3085 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3087 etype = ntohs(eh->evl_encap_proto);
3088 ehdrlen = ETHER_HDR_LEN;
3091 /* Set the ether header length */
3092 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3096 ip = (struct ip *)(mp->m_data + ehdrlen);
3097 ip_hlen = ip->ip_hl << 2;
3099 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3101 case ETHERTYPE_IPV6:
3102 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3103 ip_hlen = sizeof(struct ip6_hdr);
3104 /* XXX-BZ this will go badly in case of ext hdrs. */
3105 ipproto = ip6->ip6_nxt;
3106 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3113 vlan_macip_lens |= ip_hlen;
3114 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3118 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3119 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3123 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3124 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3129 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3130 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3138 /* Now copy bits into descriptor */
3139 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3140 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3141 TXD->seqnum_seed = htole32(0);
3142 TXD->mss_l4len_idx = htole32(0);
3144 tx_buffer->m_head = NULL;
3145 tx_buffer->eop_index = -1;
3147 /* We've consumed the first desc, adjust counters */
3148 if (++ctxd == adapter->num_tx_desc)
3150 txr->next_avail_desc = ctxd;
3156 /**********************************************************************
3158 * Setup work for hardware segmentation offload (TSO) on
3159 * adapters using advanced tx descriptors
3161 **********************************************************************/
3163 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3166 struct adapter *adapter = txr->adapter;
3167 struct ixgbe_adv_tx_context_desc *TXD;
3168 struct ixgbe_tx_buf *tx_buffer;
3169 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3170 u16 vtag = 0, eh_type;
3171 u32 mss_l4len_idx = 0, len;
3172 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3173 struct ether_vlan_header *eh;
3174 #if 0 /* IPv6 TSO */
3176 struct ip6_hdr *ip6;
3186 * Determine where frame payload starts.
3187 * Jump over vlan headers if already present
3189 eh = mtod(mp, struct ether_vlan_header *);
3190 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3191 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3192 eh_type = eh->evl_proto;
3194 ehdrlen = ETHER_HDR_LEN;
3195 eh_type = eh->evl_encap_proto;
3198 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3199 len = ehdrlen + sizeof(struct tcphdr);
3200 switch (ntohs(eh_type)) {
3201 #if 0 /* IPv6 TSO */
3203 case ETHERTYPE_IPV6:
3204 if (mp->m_len < len + sizeof(struct ip6_hdr))
3206 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3207 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3208 if (ip6->ip6_nxt != IPPROTO_TCP)
3210 ip_hlen = sizeof(struct ip6_hdr);
3211 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3212 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3213 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3219 if (mp->m_len < len + sizeof(struct ip))
3221 ip = (struct ip *)(mp->m_data + ehdrlen);
3222 if (ip->ip_p != IPPROTO_TCP)
3225 ip_hlen = ip->ip_hl << 2;
3226 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3227 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3228 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3229 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3230 /* Tell transmit desc to also do IPv4 checksum. */
3231 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3235 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3236 __func__, ntohs(eh_type));
3240 ctxd = txr->next_avail_desc;
3241 tx_buffer = &txr->tx_buffers[ctxd];
3242 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3244 tcp_hlen = th->th_off << 2;
3246 /* This is used in the transmit desc in encap */
3247 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3249 /* VLAN MACLEN IPLEN */
3250 if (mp->m_flags & M_VLANTAG) {
3251 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3252 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3255 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3256 vlan_macip_lens |= ip_hlen;
3257 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3259 /* ADV DTYPE TUCMD */
3260 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3261 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3262 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3265 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3266 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3267 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3269 TXD->seqnum_seed = htole32(0);
3270 tx_buffer->m_head = NULL;
3271 tx_buffer->eop_index = -1;
3273 if (++ctxd == adapter->num_tx_desc)
3277 txr->next_avail_desc = ctxd;
3283 ** This routine parses packet headers so that Flow
3284 ** Director can make a hashed filter table entry
3285 ** allowing traffic flows to be identified and kept
3286 ** on the same cpu. This would be a performance
3287 ** hit, but we only do it at IXGBE_FDIR_RATE of
3291 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3293 struct adapter *adapter = txr->adapter;
3294 struct ix_queue *que;
3298 struct ether_vlan_header *eh;
3299 union ixgbe_atr_hash_dword input = {.dword = 0};
3300 union ixgbe_atr_hash_dword common = {.dword = 0};
3301 int ehdrlen, ip_hlen;
3304 eh = mtod(mp, struct ether_vlan_header *);
3305 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3306 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3307 etype = eh->evl_proto;
3309 ehdrlen = ETHER_HDR_LEN;
3310 etype = eh->evl_encap_proto;
3313 /* Only handling IPv4 */
3314 if (etype != htons(ETHERTYPE_IP))
3317 ip = (struct ip *)(mp->m_data + ehdrlen);
3318 ip_hlen = ip->ip_hl << 2;
3320 /* check if we're UDP or TCP */
3323 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3324 /* src and dst are inverted */
3325 common.port.dst ^= th->th_sport;
3326 common.port.src ^= th->th_dport;
3327 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3330 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3331 /* src and dst are inverted */
3332 common.port.dst ^= uh->uh_sport;
3333 common.port.src ^= uh->uh_dport;
3334 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3340 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3341 if (mp->m_pkthdr.ether_vtag)
3342 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3344 common.flex_bytes ^= etype;
3345 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3347 que = &adapter->queues[txr->me];
3349 ** This assumes the Rx queue and Tx
3350 ** queue are bound to the same CPU
3352 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3353 input, common, que->msix);
3355 #endif /* IXGBE_FDIR */
3357 /**********************************************************************
3359 * Examine each tx_buffer in the used queue. If the hardware is done
3360 * processing the packet then free associated resources. The
3361 * tx_buffer is put back on the free queue.
3363 **********************************************************************/
3365 ixgbe_txeof(struct tx_ring *txr)
3367 struct adapter *adapter = txr->adapter;
3368 struct ifnet *ifp = adapter->ifp;
3369 u32 first, last, done, processed;
3370 struct ixgbe_tx_buf *tx_buffer;
3371 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3373 IXGBE_TX_LOCK_ASSERT(txr);
3376 if (ifp->if_capenable & IFCAP_NETMAP) {
3377 struct netmap_adapter *na = NA(ifp);
3378 struct netmap_kring *kring = &na->tx_rings[txr->me];
3380 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3382 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3383 BUS_DMASYNC_POSTREAD);
3385 * In netmap mode, all the work is done in the context
3386 * of the client thread. Interrupt handlers only wake up
3387 * clients, which may be sleeping on individual rings
3388 * or on a global resource for all rings.
3389 * To implement tx interrupt mitigation, we wake up the client
3390 * thread roughly every half ring, even if the NIC interrupts
3391 * more frequently. This is implemented as follows:
3392 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3393 * the slot that should wake up the thread (nkr_num_slots
3394 * means the user thread should not be woken up);
3395 * - the driver ignores tx interrupts unless netmap_mitigate=0
3396 * or the slot has the DD bit set.
3398 * When the driver has separate locks, we need to
3399 * release and re-acquire txlock to avoid deadlocks.
3400 * XXX see if we can find a better way.
3402 if (!netmap_mitigate ||
3403 (kring->nr_kflags < kring->nkr_num_slots &&
3404 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3405 kring->nr_kflags = kring->nkr_num_slots;
3406 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3407 IXGBE_TX_UNLOCK(txr);
3408 IXGBE_CORE_LOCK(adapter);
3409 selwakeuppri(&na->tx_si, PI_NET);
3410 IXGBE_CORE_UNLOCK(adapter);
3415 #endif /* DEV_NETMAP */
3417 if (txr->tx_avail == adapter->num_tx_desc) {
3418 txr->queue_status = IXGBE_QUEUE_IDLE;
3423 first = txr->next_to_clean;
3424 tx_buffer = &txr->tx_buffers[first];
3425 /* For cleanup we just use legacy struct */
3426 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3427 last = tx_buffer->eop_index;
3430 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3433 ** Get the index of the first descriptor
3434 ** BEYOND the EOP and call that 'done'.
3435 ** I do this so the comparison in the
3436 ** inner while loop below can be simple
3438 if (++last == adapter->num_tx_desc) last = 0;
3441 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3442 BUS_DMASYNC_POSTREAD);
3444 ** Only the EOP descriptor of a packet now has the DD
3445 ** bit set, this is what we look for...
3447 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3448 /* We clean the range of the packet */
3449 while (first != done) {
3450 tx_desc->upper.data = 0;
3451 tx_desc->lower.data = 0;
3452 tx_desc->buffer_addr = 0;
3456 if (tx_buffer->m_head) {
3458 tx_buffer->m_head->m_pkthdr.len;
3459 bus_dmamap_sync(txr->txtag,
3461 BUS_DMASYNC_POSTWRITE);
3462 bus_dmamap_unload(txr->txtag,
3464 m_freem(tx_buffer->m_head);
3465 tx_buffer->m_head = NULL;
3466 tx_buffer->map = NULL;
3468 tx_buffer->eop_index = -1;
3469 txr->watchdog_time = ticks;
3471 if (++first == adapter->num_tx_desc)
3474 tx_buffer = &txr->tx_buffers[first];
3476 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3480 /* See if there is more work now */
3481 last = tx_buffer->eop_index;
3484 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3485 /* Get next done point */
3486 if (++last == adapter->num_tx_desc) last = 0;
3491 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3492 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3494 txr->next_to_clean = first;
3497 ** Watchdog calculation, we know there's
3498 ** work outstanding or the first return
3499 ** would have been taken, so none processed
3500 ** for too long indicates a hang.
3502 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3503 txr->queue_status = IXGBE_QUEUE_HUNG;
3505 /* With a minimum free clear the depleted state bit. */
3506 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3507 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3509 if (txr->tx_avail == adapter->num_tx_desc) {
3510 txr->queue_status = IXGBE_QUEUE_IDLE;
3514 /*********************************************************************
3516 * Refresh mbuf buffers for RX descriptor rings
3517 * - now keeps its own state so discards due to resource
3518 * exhaustion are unnecessary, if an mbuf cannot be obtained
3519 * it just returns, keeping its placeholder, thus it can simply
3520 * be recalled to try again.
3522 **********************************************************************/
3524 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3526 struct adapter *adapter = rxr->adapter;
3527 bus_dma_segment_t hseg[1];
3528 bus_dma_segment_t pseg[1];
3529 struct ixgbe_rx_buf *rxbuf;
3530 struct mbuf *mh, *mp;
3531 int i, j, nsegs, error;
3532 bool refreshed = FALSE;
3534 i = j = rxr->next_to_refresh;
3535 /* Control the loop with one beyond */
3536 if (++j == adapter->num_rx_desc)
3539 while (j != limit) {
3540 rxbuf = &rxr->rx_buffers[i];
3541 if (rxr->hdr_split == FALSE)
3544 if (rxbuf->m_head == NULL) {
3545 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3551 mh->m_pkthdr.len = mh->m_len = MHLEN;
3553 mh->m_flags |= M_PKTHDR;
3554 /* Get the memory mapping */
3555 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3556 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3558 kprintf("Refresh mbufs: hdr dmamap load"
3559 " failure - %d\n", error);
3561 rxbuf->m_head = NULL;
3565 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3566 BUS_DMASYNC_PREREAD);
3567 rxr->rx_base[i].read.hdr_addr =
3568 htole64(hseg[0].ds_addr);
3571 if (rxbuf->m_pack == NULL) {
3572 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3573 M_PKTHDR, adapter->rx_mbuf_sz);
3579 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3580 /* Get the memory mapping */
3581 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3582 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3584 kprintf("Refresh mbufs: payload dmamap load"
3585 " failure - %d\n", error);
3587 rxbuf->m_pack = NULL;
3591 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3592 BUS_DMASYNC_PREREAD);
3593 rxr->rx_base[i].read.pkt_addr =
3594 htole64(pseg[0].ds_addr);
3597 /* Next is precalculated */
3599 rxr->next_to_refresh = i;
3600 if (++j == adapter->num_rx_desc)
3604 if (refreshed) /* Update hardware tail index */
3605 IXGBE_WRITE_REG(&adapter->hw,
3606 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3610 /*********************************************************************
3612 * Allocate memory for rx_buffer structures. Since we use one
3613 * rx_buffer per received packet, the maximum number of rx_buffer's
3614 * that we'll need is equal to the number of receive descriptors
3615 * that we've allocated.
3617 **********************************************************************/
3619 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3621 struct adapter *adapter = rxr->adapter;
3622 device_t dev = adapter->dev;
3623 struct ixgbe_rx_buf *rxbuf;
3624 int i, bsize, error;
3626 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3627 if (!(rxr->rx_buffers =
3628 (struct ixgbe_rx_buf *) kmalloc(bsize,
3629 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3630 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3635 if ((error = bus_dma_tag_create(NULL, /* parent */
3636 1, 0, /* alignment, bounds */
3637 BUS_SPACE_MAXADDR, /* lowaddr */
3638 BUS_SPACE_MAXADDR, /* highaddr */
3639 NULL, NULL, /* filter, filterarg */
3640 MSIZE, /* maxsize */
3642 MSIZE, /* maxsegsize */
3645 device_printf(dev, "Unable to create RX DMA tag\n");
3649 if ((error = bus_dma_tag_create(NULL, /* parent */
3650 1, 0, /* alignment, bounds */
3651 BUS_SPACE_MAXADDR, /* lowaddr */
3652 BUS_SPACE_MAXADDR, /* highaddr */
3653 NULL, NULL, /* filter, filterarg */
3654 MJUM16BYTES, /* maxsize */
3656 MJUM16BYTES, /* maxsegsize */
3659 device_printf(dev, "Unable to create RX DMA tag\n");
3663 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3664 rxbuf = &rxr->rx_buffers[i];
3665 error = bus_dmamap_create(rxr->htag,
3666 BUS_DMA_NOWAIT, &rxbuf->hmap);
3668 device_printf(dev, "Unable to create RX head map\n");
3671 error = bus_dmamap_create(rxr->ptag,
3672 BUS_DMA_NOWAIT, &rxbuf->pmap);
3674 device_printf(dev, "Unable to create RX pkt map\n");
3682 /* Frees all, but can handle partial completion */
3683 ixgbe_free_receive_structures(adapter);
3688 ** Used to detect a descriptor that has
3689 ** been merged by Hardware RSC.
3692 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3694 return (le32toh(rx->wb.lower.lo_dword.data) &
3695 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3698 /*********************************************************************
3700 * Initialize Hardware RSC (LRO) feature on 82599
3701 * for an RX ring, this is toggled by the LRO capability
3702 * even though it is transparent to the stack.
3704 **********************************************************************/
3707 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3709 struct adapter *adapter = rxr->adapter;
3710 struct ixgbe_hw *hw = &adapter->hw;
3711 u32 rscctrl, rdrxctl;
3713 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3714 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3715 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3716 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3717 #endif /* DEV_NETMAP */
3718 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3719 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3720 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3722 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3723 rscctrl |= IXGBE_RSCCTL_RSCEN;
3725 ** Limit the total number of descriptors that
3726 ** can be combined, so it does not exceed 64K
3728 if (adapter->rx_mbuf_sz == MCLBYTES)
3729 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3730 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3731 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3732 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3733 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3734 else /* Using 16K cluster */
3735 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3737 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3739 /* Enable TCP header recognition */
3740 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3741 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3742 IXGBE_PSRTYPE_TCPHDR));
3744 /* Disable RSC for ACK packets */
3745 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3746 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3753 ixgbe_free_receive_ring(struct rx_ring *rxr)
3755 struct adapter *adapter;
3756 struct ixgbe_rx_buf *rxbuf;
3759 adapter = rxr->adapter;
3760 for (i = 0; i < adapter->num_rx_desc; i++) {
3761 rxbuf = &rxr->rx_buffers[i];
3762 if (rxbuf->m_head != NULL) {
3763 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3764 BUS_DMASYNC_POSTREAD);
3765 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3766 rxbuf->m_head->m_flags |= M_PKTHDR;
3767 m_freem(rxbuf->m_head);
3769 if (rxbuf->m_pack != NULL) {
3770 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3771 BUS_DMASYNC_POSTREAD);
3772 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3773 rxbuf->m_pack->m_flags |= M_PKTHDR;
3774 m_freem(rxbuf->m_pack);
3776 rxbuf->m_head = NULL;
3777 rxbuf->m_pack = NULL;
3782 /*********************************************************************
3784 * Initialize a receive ring and its buffers.
3786 **********************************************************************/
3788 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3790 struct adapter *adapter;
3793 struct ixgbe_rx_buf *rxbuf;
3794 bus_dma_segment_t pseg[1], hseg[1];
3796 struct lro_ctrl *lro = &rxr->lro;
3798 int rsize, nsegs, error = 0;
3800 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3801 struct netmap_slot *slot;
3802 #endif /* DEV_NETMAP */
3804 adapter = rxr->adapter;
3808 /* Clear the ring contents */
3811 /* same as in ixgbe_setup_transmit_ring() */
3812 slot = netmap_reset(na, NR_RX, rxr->me, 0);
3813 #endif /* DEV_NETMAP */
3814 rsize = roundup2(adapter->num_rx_desc *
3815 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3816 bzero((void *)rxr->rx_base, rsize);
3818 /* Free current RX buffer structs and their mbufs */
3819 ixgbe_free_receive_ring(rxr);
3821 /* Configure header split? */
3822 if (ixgbe_header_split)
3823 rxr->hdr_split = TRUE;
3825 /* Now replenish the mbufs */
3826 for (int j = 0; j != adapter->num_rx_desc; ++j) {
3827 struct mbuf *mh, *mp;
3829 rxbuf = &rxr->rx_buffers[j];
3832 * In netmap mode, fill the map and set the buffer
3833 * address in the NIC ring, considering the offset
3834 * between the netmap and NIC rings (see comment in
3835 * ixgbe_setup_transmit_ring() ). No need to allocate
3836 * an mbuf, so end the block with a continue;
3839 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3843 addr = PNMB(slot + sj, &paddr);
3844 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3845 /* Update descriptor */
3846 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3849 #endif /* DEV_NETMAP */
3851 ** Don't allocate mbufs if not
3852 ** doing header split, its wasteful
3854 if (rxr->hdr_split == FALSE)
3857 /* First the header */
3858 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
3859 if (rxbuf->m_head == NULL) {
3863 m_adj(rxbuf->m_head, ETHER_ALIGN);
3865 mh->m_len = mh->m_pkthdr.len = MHLEN;
3866 mh->m_flags |= M_PKTHDR;
3867 /* Get the memory mapping */
3868 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3869 rxbuf->hmap, rxbuf->m_head, hseg, 1,
3870 &nsegs, BUS_DMA_NOWAIT);
3872 if (error != 0) /* Nothing elegant to do here */
3874 bus_dmamap_sync(rxr->htag,
3875 rxbuf->hmap, BUS_DMASYNC_PREREAD);
3876 /* Update descriptor */
3877 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3880 /* Now the payload cluster */
3881 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
3882 M_PKTHDR, adapter->rx_mbuf_sz);
3883 if (rxbuf->m_pack == NULL) {
3888 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3889 /* Get the memory mapping */
3890 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3891 rxbuf->pmap, mp, hseg, 1,
3892 &nsegs, BUS_DMA_NOWAIT);
3895 bus_dmamap_sync(rxr->ptag,
3896 rxbuf->pmap, BUS_DMASYNC_PREREAD);
3897 /* Update descriptor */
3898 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3902 /* Setup our descriptor indices */
3903 rxr->next_to_check = 0;
3904 rxr->next_to_refresh = 0;
3905 rxr->lro_enabled = FALSE;
3906 rxr->rx_split_packets = 0;
3908 rxr->discard = FALSE;
3909 rxr->vtag_strip = FALSE;
3911 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3912 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3915 ** Now set up the LRO interface:
3916 ** 82598 uses software LRO, the
3917 ** 82599 and X540 use a hardware assist.
3920 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3921 (ifp->if_capenable & IFCAP_RXCSUM) &&
3922 (ifp->if_capenable & IFCAP_LRO))
3923 ixgbe_setup_hw_rsc(rxr);
3924 else if (ifp->if_capenable & IFCAP_LRO) {
3925 int err = tcp_lro_init(lro);
3927 device_printf(dev, "LRO Initialization failed!\n");
3930 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3931 rxr->lro_enabled = TRUE;
3932 lro->ifp = adapter->ifp;
3936 IXGBE_RX_UNLOCK(rxr);
3940 ixgbe_free_receive_ring(rxr);
3941 IXGBE_RX_UNLOCK(rxr);
3945 /*********************************************************************
3947 * Initialize all receive rings.
3949 **********************************************************************/
3951 ixgbe_setup_receive_structures(struct adapter *adapter)
3953 struct rx_ring *rxr = adapter->rx_rings;
3956 for (j = 0; j < adapter->num_queues; j++, rxr++)
3957 if (ixgbe_setup_receive_ring(rxr))
3963 * Free RX buffers allocated so far, we will only handle
3964 * the rings that completed, the failing case will have
3965 * cleaned up for itself. 'j' failed, so its the terminus.
3967 for (int i = 0; i < j; ++i) {
3968 rxr = &adapter->rx_rings[i];
3969 ixgbe_free_receive_ring(rxr);
3975 /*********************************************************************
3977 * Setup receive registers and features.
3979 **********************************************************************/
3980 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3982 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3985 ixgbe_initialize_receive_units(struct adapter *adapter)
3987 struct rx_ring *rxr = adapter->rx_rings;
3988 struct ixgbe_hw *hw = &adapter->hw;
3989 struct ifnet *ifp = adapter->ifp;
3990 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
3991 u32 reta, mrqc = 0, hlreg, random[10];
3995 * Make sure receives are disabled while
3996 * setting up the descriptor ring
3998 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3999 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4000 rxctrl & ~IXGBE_RXCTRL_RXEN);
4002 /* Enable broadcasts */
4003 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4004 fctrl |= IXGBE_FCTRL_BAM;
4005 fctrl |= IXGBE_FCTRL_DPF;
4006 fctrl |= IXGBE_FCTRL_PMCF;
4007 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4009 /* Set for Jumbo Frames? */
4010 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4011 if (ifp->if_mtu > ETHERMTU)
4012 hlreg |= IXGBE_HLREG0_JUMBOEN;
4014 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4016 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4017 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4018 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4020 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4021 #endif /* DEV_NETMAP */
4022 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4024 bufsz = (adapter->rx_mbuf_sz +
4025 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4027 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4028 u64 rdba = rxr->rxdma.dma_paddr;
4030 /* Setup the Base and Length of the Rx Descriptor Ring */
4031 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4032 (rdba & 0x00000000ffffffffULL));
4033 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4034 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4035 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4037 /* Set up the SRRCTL register */
4038 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4039 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4040 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4042 if (rxr->hdr_split) {
4043 /* Use a standard mbuf for the header */
4044 srrctl |= ((IXGBE_RX_HDR <<
4045 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4046 & IXGBE_SRRCTL_BSIZEHDR_MASK);
4047 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4049 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4050 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4052 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4053 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4054 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4057 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4058 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4059 IXGBE_PSRTYPE_UDPHDR |
4060 IXGBE_PSRTYPE_IPV4HDR |
4061 IXGBE_PSRTYPE_IPV6HDR;
4062 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4065 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4068 if (adapter->num_queues > 1) {
4072 /* set up random bits */
4073 karc4rand(&random, sizeof(random));
4075 /* Set up the redirection table */
4076 for (i = 0, j = 0; i < 128; i++, j++) {
4077 if (j == adapter->num_queues) j = 0;
4078 reta = (reta << 8) | (j * 0x11);
4080 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4083 /* Now fill our hash function seeds */
4084 for (int i = 0; i < 10; i++)
4085 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4087 /* Perform hash on these packet types */
4088 mrqc = IXGBE_MRQC_RSSEN
4089 | IXGBE_MRQC_RSS_FIELD_IPV4
4090 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4091 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4092 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4093 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4094 | IXGBE_MRQC_RSS_FIELD_IPV6
4095 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4096 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4097 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4098 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4100 /* RSS and RX IPP Checksum are mutually exclusive */
4101 rxcsum |= IXGBE_RXCSUM_PCSD;
4104 if (ifp->if_capenable & IFCAP_RXCSUM)
4105 rxcsum |= IXGBE_RXCSUM_PCSD;
4107 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4108 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4110 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4115 /*********************************************************************
4117 * Free all receive rings.
4119 **********************************************************************/
4121 ixgbe_free_receive_structures(struct adapter *adapter)
4123 struct rx_ring *rxr = adapter->rx_rings;
4125 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4127 struct lro_ctrl *lro = &rxr->lro;
4129 ixgbe_free_receive_buffers(rxr);
4130 /* Free LRO memory */
4134 /* Free the ring memory as well */
4135 ixgbe_dma_free(adapter, &rxr->rxdma);
4138 kfree(adapter->rx_rings, M_DEVBUF);
4142 /*********************************************************************
4144 * Free receive ring data structures
4146 **********************************************************************/
4148 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4150 struct adapter *adapter = rxr->adapter;
4151 struct ixgbe_rx_buf *rxbuf;
4153 INIT_DEBUGOUT("free_receive_structures: begin");
4155 /* Cleanup any existing buffers */
4156 if (rxr->rx_buffers != NULL) {
4157 for (int i = 0; i < adapter->num_rx_desc; i++) {
4158 rxbuf = &rxr->rx_buffers[i];
4159 if (rxbuf->m_head != NULL) {
4160 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4161 BUS_DMASYNC_POSTREAD);
4162 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4163 rxbuf->m_head->m_flags |= M_PKTHDR;
4164 m_freem(rxbuf->m_head);
4166 if (rxbuf->m_pack != NULL) {
4167 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4168 BUS_DMASYNC_POSTREAD);
4169 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4170 rxbuf->m_pack->m_flags |= M_PKTHDR;
4171 m_freem(rxbuf->m_pack);
4173 rxbuf->m_head = NULL;
4174 rxbuf->m_pack = NULL;
4175 if (rxbuf->hmap != NULL) {
4176 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4179 if (rxbuf->pmap != NULL) {
4180 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4184 if (rxr->rx_buffers != NULL) {
4185 kfree(rxr->rx_buffers, M_DEVBUF);
4186 rxr->rx_buffers = NULL;
4190 if (rxr->htag != NULL) {
4191 bus_dma_tag_destroy(rxr->htag);
4194 if (rxr->ptag != NULL) {
4195 bus_dma_tag_destroy(rxr->ptag);
4202 static __inline void
4203 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4207 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4208 * should be computed by hardware. Also it should not have VLAN tag in
4209 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4212 if (rxr->lro_enabled &&
4213 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4214 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4215 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4216 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4217 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4218 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4219 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4220 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4222 * Send to the stack if:
4223 ** - LRO not enabled, or
4224 ** - no LRO resources, or
4225 ** - lro enqueue fails
4227 if (rxr->lro.lro_cnt != 0)
4228 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4232 IXGBE_RX_UNLOCK(rxr);
4233 (*ifp->if_input)(ifp, m);
4237 static __inline void
4238 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4240 struct ixgbe_rx_buf *rbuf;
4242 rbuf = &rxr->rx_buffers[i];
4244 if (rbuf->fmp != NULL) {/* Partial chain ? */
4245 rbuf->fmp->m_flags |= M_PKTHDR;
4251 ** With advanced descriptors the writeback
4252 ** clobbers the buffer addrs, so its easier
4253 ** to just free the existing mbufs and take
4254 ** the normal refresh path to get new buffers
4258 m_free(rbuf->m_head);
4259 rbuf->m_head = NULL;
4263 m_free(rbuf->m_pack);
4264 rbuf->m_pack = NULL;
4271 ixgbe_add_sysctl(struct adapter *adapter)
4273 sysctl_ctx_init(&adapter->sysctl_ctx);
4274 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4275 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4276 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4277 if (adapter->sysctl_tree == NULL) {
4278 device_printf(adapter->dev, "can't add sysctl node\n");
4281 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4282 SYSCTL_CHILDREN(adapter->sysctl_tree),
4283 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4284 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4286 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4287 SYSCTL_CHILDREN(adapter->sysctl_tree),
4288 OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4289 adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4292 ** Allow a kind of speed control by forcing the autoneg
4293 ** advertised speed list to only a certain value, this
4294 ** supports 1G on 82599 devices, and 100Mb on x540.
4296 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4297 SYSCTL_CHILDREN(adapter->sysctl_tree),
4298 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4299 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4301 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4302 SYSCTL_CHILDREN(adapter->sysctl_tree),
4303 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4304 0, ixgbe_set_thermal_test, "I", "Thermal Test");
4306 /* Sysctl for limiting the amount of work done in the taskqueue */
4307 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4308 "max number of rx packets to process", &adapter->rx_process_limit,
4309 ixgbe_rx_process_limit);
4312 /*********************************************************************
4314 * This routine executes in interrupt context. It replenishes
4315 * the mbufs in the descriptor and sends data which has been
4316 * dma'ed into host memory to upper layer.
4318 * We loop at most count times if count is > 0, or until done if
4321 * Return TRUE for more work, FALSE for all clean.
4322 *********************************************************************/
4324 ixgbe_rxeof(struct ix_queue *que, int count)
4326 struct adapter *adapter = que->adapter;
4327 struct rx_ring *rxr = que->rxr;
4328 struct ifnet *ifp = adapter->ifp;
4330 struct lro_ctrl *lro = &rxr->lro;
4331 struct lro_entry *queued;
4333 int i, nextp, processed = 0;
4335 union ixgbe_adv_rx_desc *cur;
4336 struct ixgbe_rx_buf *rbuf, *nbuf;
4341 if (ifp->if_capenable & IFCAP_NETMAP) {
4343 * Same as the txeof routine: only wakeup clients on intr.
4344 * NKR_PENDINTR in nr_kflags is used to implement interrupt
4345 * mitigation (ixgbe_rxsync() will not look for new packets
4346 * unless NKR_PENDINTR is set).
4348 struct netmap_adapter *na = NA(ifp);
4350 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4351 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4352 IXGBE_RX_UNLOCK(rxr);
4353 IXGBE_CORE_LOCK(adapter);
4354 selwakeuppri(&na->rx_si, PI_NET);
4355 IXGBE_CORE_UNLOCK(adapter);
4358 #endif /* DEV_NETMAP */
4359 for (i = rxr->next_to_check; count != 0;) {
4360 struct mbuf *sendmp, *mh, *mp;
4362 u16 hlen, plen, hdr;
4366 /* Sync the ring. */
4367 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4368 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4370 cur = &rxr->rx_base[i];
4371 staterr = le32toh(cur->wb.upper.status_error);
4373 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4375 if ((ifp->if_flags & IFF_RUNNING) == 0)
4382 cur->wb.upper.status_error = 0;
4383 rbuf = &rxr->rx_buffers[i];
4387 plen = le16toh(cur->wb.upper.length);
4388 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4389 IXGBE_RXDADV_PKTTYPE_MASK;
4390 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4391 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4393 /* Process vlan info */
4394 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4395 vtag = le16toh(cur->wb.upper.vlan);
4397 /* Make sure bad packets are discarded */
4398 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4401 rxr->rx_discarded++;
4403 rxr->discard = FALSE;
4405 rxr->discard = TRUE;
4406 ixgbe_rx_discard(rxr, i);
4411 ** On 82599 which supports a hardware
4412 ** LRO (called HW RSC), packets need
4413 ** not be fragmented across sequential
4414 ** descriptors, rather the next descriptor
4415 ** is indicated in bits of the descriptor.
4416 ** This also means that we might proceses
4417 ** more than one packet at a time, something
4418 ** that has never been true before, it
4419 ** required eliminating global chain pointers
4420 ** in favor of what we are doing here. -jfv
4424 ** Figure out the next descriptor
4427 if (rxr->hw_rsc == TRUE) {
4428 rsc = ixgbe_rsc_count(cur);
4429 rxr->rsc_num += (rsc - 1);
4431 if (rsc) { /* Get hardware index */
4433 IXGBE_RXDADV_NEXTP_MASK) >>
4434 IXGBE_RXDADV_NEXTP_SHIFT);
4435 } else { /* Just sequential */
4437 if (nextp == adapter->num_rx_desc)
4440 nbuf = &rxr->rx_buffers[nextp];
4444 ** The header mbuf is ONLY used when header
4445 ** split is enabled, otherwise we get normal
4446 ** behavior, ie, both header and payload
4447 ** are DMA'd into the payload buffer.
4449 ** Rather than using the fmp/lmp global pointers
4450 ** we now keep the head of a packet chain in the
4451 ** buffer struct and pass this along from one
4452 ** descriptor to the next, until we get EOP.
4454 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4455 /* This must be an initial descriptor */
4456 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4457 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4458 if (hlen > IXGBE_RX_HDR)
4459 hlen = IXGBE_RX_HDR;
4461 mh->m_flags |= M_PKTHDR;
4463 mh->m_pkthdr.len = mh->m_len;
4464 /* Null buf pointer so it is refreshed */
4465 rbuf->m_head = NULL;
4467 ** Check the payload length, this
4468 ** could be zero if its a small
4474 mp->m_flags &= ~M_PKTHDR;
4476 mh->m_pkthdr.len += mp->m_len;
4477 /* Null buf pointer so it is refreshed */
4478 rbuf->m_pack = NULL;
4479 rxr->rx_split_packets++;
4482 ** Now create the forward
4483 ** chain so when complete
4487 /* stash the chain head */
4489 /* Make forward chain */
4491 mp->m_next = nbuf->m_pack;
4493 mh->m_next = nbuf->m_pack;
4495 /* Singlet, prepare to send */
4497 /* If hardware handled vtag */
4499 sendmp->m_pkthdr.ether_vlantag = vtag;
4500 sendmp->m_flags |= M_VLANTAG;
4505 ** Either no header split, or a
4506 ** secondary piece of a fragmented
4511 ** See if there is a stored head
4512 ** that determines what we are
4515 rbuf->m_pack = rbuf->fmp = NULL;
4517 if (sendmp != NULL) { /* secondary frag */
4518 mp->m_flags &= ~M_PKTHDR;
4519 sendmp->m_pkthdr.len += mp->m_len;
4521 /* first desc of a non-ps chain */
4523 sendmp->m_flags |= M_PKTHDR;
4524 sendmp->m_pkthdr.len = mp->m_len;
4525 if (staterr & IXGBE_RXD_STAT_VP) {
4526 sendmp->m_pkthdr.ether_vlantag = vtag;
4527 sendmp->m_flags |= M_VLANTAG;
4530 /* Pass the head pointer on */
4534 mp->m_next = nbuf->m_pack;
4538 /* Sending this frame? */
4540 sendmp->m_pkthdr.rcvif = ifp;
4543 /* capture data for AIM */
4544 rxr->bytes += sendmp->m_pkthdr.len;
4545 rxr->rx_bytes += sendmp->m_pkthdr.len;
4546 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4547 ixgbe_rx_checksum(staterr, sendmp, ptype);
4548 #if 0 /* __FreeBSD_version >= 800000 */
4549 sendmp->m_pkthdr.flowid = que->msix;
4550 sendmp->m_flags |= M_FLOWID;
4554 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4555 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4557 /* Advance our pointers to the next descriptor. */
4558 if (++i == adapter->num_rx_desc)
4561 /* Now send to the stack or do LRO */
4562 if (sendmp != NULL) {
4563 rxr->next_to_check = i;
4564 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4565 i = rxr->next_to_check;
4568 /* Every 8 descriptors we go to refresh mbufs */
4569 if (processed == 8) {
4570 ixgbe_refresh_mbufs(rxr, i);
4575 /* Refresh any remaining buf structs */
4576 if (ixgbe_rx_unrefreshed(rxr))
4577 ixgbe_refresh_mbufs(rxr, i);
4579 rxr->next_to_check = i;
4582 * Flush any outstanding LRO work
4585 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4586 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4587 tcp_lro_flush(lro, queued);
4591 IXGBE_RX_UNLOCK(rxr);
4594 ** We still have cleaning to do?
4595 ** Schedule another interrupt if so.
4597 if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4598 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4603 /*********************************************************************
4605 * Verify that the hardware indicated that the checksum is valid.
4606 * Inform the stack about the status of checksum so that stack
4607 * doesn't spend time verifying the checksum.
4609 *********************************************************************/
4611 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4613 u16 status = (u16) staterr;
4614 u8 errors = (u8) (staterr >> 24);
4617 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4618 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4621 if (status & IXGBE_RXD_STAT_IPCS) {
4622 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4623 /* IP Checksum Good */
4624 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4625 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4628 mp->m_pkthdr.csum_flags = 0;
4630 if (status & IXGBE_RXD_STAT_L4CS) {
4631 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4634 type = CSUM_SCTP_VALID;
4636 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4637 mp->m_pkthdr.csum_flags |= type;
4639 mp->m_pkthdr.csum_data = htons(0xffff);
4647 ** This routine is run via an vlan config EVENT,
4648 ** it enables us to use the HW Filter table since
4649 ** we can get the vlan id. This just creates the
4650 ** entry in the soft version of the VFTA, init will
4651 ** repopulate the real table.
4654 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4656 struct adapter *adapter = ifp->if_softc;
4659 if (ifp->if_softc != arg) /* Not our event */
4662 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4665 IXGBE_CORE_LOCK(adapter);
4666 index = (vtag >> 5) & 0x7F;
4668 adapter->shadow_vfta[index] |= (1 << bit);
4669 ++adapter->num_vlans;
4670 ixgbe_init_locked(adapter);
4671 IXGBE_CORE_UNLOCK(adapter);
4675 ** This routine is run via an vlan
4676 ** unconfig EVENT, remove our entry
4677 ** in the soft vfta.
4680 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4682 struct adapter *adapter = ifp->if_softc;
4685 if (ifp->if_softc != arg)
4688 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4691 IXGBE_CORE_LOCK(adapter);
4692 index = (vtag >> 5) & 0x7F;
4694 adapter->shadow_vfta[index] &= ~(1 << bit);
4695 --adapter->num_vlans;
4696 /* Re-init to load the changes */
4697 ixgbe_init_locked(adapter);
4698 IXGBE_CORE_UNLOCK(adapter);
4702 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4704 struct ifnet *ifp = adapter->ifp;
4705 struct ixgbe_hw *hw = &adapter->hw;
4706 struct rx_ring *rxr;
4710 ** We get here thru init_locked, meaning
4711 ** a soft reset, this has already cleared
4712 ** the VFTA and other state, so if there
4713 ** have been no vlan's registered do nothing.
4715 if (adapter->num_vlans == 0)
4719 ** A soft reset zero's out the VFTA, so
4720 ** we need to repopulate it now.
4722 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4723 if (adapter->shadow_vfta[i] != 0)
4724 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4725 adapter->shadow_vfta[i]);
4727 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4728 /* Enable the Filter Table if enabled */
4729 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4730 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4731 ctrl |= IXGBE_VLNCTRL_VFE;
4733 if (hw->mac.type == ixgbe_mac_82598EB)
4734 ctrl |= IXGBE_VLNCTRL_VME;
4735 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4737 /* Setup the queues for vlans */
4738 for (int i = 0; i < adapter->num_queues; i++) {
4739 rxr = &adapter->rx_rings[i];
4740 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4741 if (hw->mac.type != ixgbe_mac_82598EB) {
4742 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4743 ctrl |= IXGBE_RXDCTL_VME;
4744 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4746 rxr->vtag_strip = TRUE;
4751 ixgbe_enable_intr(struct adapter *adapter)
4753 struct ixgbe_hw *hw = &adapter->hw;
4754 struct ix_queue *que = adapter->queues;
4755 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4758 /* Enable Fan Failure detection */
4759 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4760 mask |= IXGBE_EIMS_GPI_SDP1;
4762 mask |= IXGBE_EIMS_ECC;
4763 mask |= IXGBE_EIMS_GPI_SDP0;
4764 mask |= IXGBE_EIMS_GPI_SDP1;
4765 mask |= IXGBE_EIMS_GPI_SDP2;
4767 mask |= IXGBE_EIMS_FLOW_DIR;
4771 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4773 /* With RSS we use auto clear */
4774 if (adapter->msix_mem) {
4775 mask = IXGBE_EIMS_ENABLE_MASK;
4776 /* Don't autoclear Link */
4777 mask &= ~IXGBE_EIMS_OTHER;
4778 mask &= ~IXGBE_EIMS_LSC;
4779 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4783 ** Now enable all queues, this is done separately to
4784 ** allow for handling the extended (beyond 32) MSIX
4785 ** vectors that can be used by 82599
4787 for (int i = 0; i < adapter->num_queues; i++, que++)
4788 ixgbe_enable_queue(adapter, que->msix);
4790 IXGBE_WRITE_FLUSH(hw);
4796 ixgbe_disable_intr(struct adapter *adapter)
4798 if (adapter->msix_mem)
4799 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4800 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4801 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4803 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4804 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4805 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4807 IXGBE_WRITE_FLUSH(&adapter->hw);
4812 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4816 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4823 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4825 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4832 ** Setup the correct IVAR register for a particular MSIX interrupt
4833 ** (yes this is all very magic and confusing :)
4834 ** - entry is the register array entry
4835 ** - vector is the MSIX vector for this queue
4836 ** - type is RX/TX/MISC
4839 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4841 struct ixgbe_hw *hw = &adapter->hw;
4844 vector |= IXGBE_IVAR_ALLOC_VAL;
4846 switch (hw->mac.type) {
4848 case ixgbe_mac_82598EB:
4850 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4852 entry += (type * 64);
4853 index = (entry >> 2) & 0x1F;
4854 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4855 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4856 ivar |= (vector << (8 * (entry & 0x3)));
4857 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4860 case ixgbe_mac_82599EB:
4861 case ixgbe_mac_X540:
4862 if (type == -1) { /* MISC IVAR */
4863 index = (entry & 1) * 8;
4864 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4865 ivar &= ~(0xFF << index);
4866 ivar |= (vector << index);
4867 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4868 } else { /* RX/TX IVARS */
4869 index = (16 * (entry & 1)) + (8 * type);
4870 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4871 ivar &= ~(0xFF << index);
4872 ivar |= (vector << index);
4873 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4882 ixgbe_configure_ivars(struct adapter *adapter)
4884 struct ix_queue *que = adapter->queues;
4886 for (int i = 0; i < adapter->num_queues; i++, que++) {
4887 /* First the RX queue entry */
4888 ixgbe_set_ivar(adapter, i, que->msix, 0);
4889 /* ... and the TX */
4890 ixgbe_set_ivar(adapter, i, que->msix, 1);
4891 /* Set an Initial EITR value */
4892 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4895 /* For the Link interrupt */
4896 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4900 ** ixgbe_sfp_probe - called in the local timer to
4901 ** determine if a port had optics inserted.
4903 static bool ixgbe_sfp_probe(struct adapter *adapter)
4905 struct ixgbe_hw *hw = &adapter->hw;
4906 device_t dev = adapter->dev;
4907 bool result = FALSE;
4909 if ((hw->phy.type == ixgbe_phy_nl) &&
4910 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4911 s32 ret = hw->phy.ops.identify_sfp(hw);
4914 ret = hw->phy.ops.reset(hw);
4915 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4916 device_printf(dev,"Unsupported SFP+ module detected!");
4917 kprintf(" Reload driver with supported module.\n");
4918 adapter->sfp_probe = FALSE;
4921 device_printf(dev,"SFP+ module detected!\n");
4922 /* We now have supported optics */
4923 adapter->sfp_probe = FALSE;
4924 /* Set the optics type so system reports correctly */
4925 ixgbe_setup_optics(adapter);
4933 ** Tasklet for handling SFP module interrupts
4936 ixgbe_handle_mod(void *context, int pending)
4938 struct adapter *adapter = context;
4939 struct ixgbe_hw *hw = &adapter->hw;
4940 device_t dev = adapter->dev;
4943 err = hw->phy.ops.identify_sfp(hw);
4944 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4946 "Unsupported SFP+ module type was detected.\n");
4949 err = hw->mac.ops.setup_sfp(hw);
4950 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4952 "Setup failure - unsupported SFP+ module type.\n");
4955 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4961 ** Tasklet for handling MSF (multispeed fiber) interrupts
4964 ixgbe_handle_msf(void *context, int pending)
4966 struct adapter *adapter = context;
4967 struct ixgbe_hw *hw = &adapter->hw;
4971 autoneg = hw->phy.autoneg_advertised;
4972 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4973 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4974 if (hw->mac.ops.setup_link)
4975 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4981 ** Tasklet for reinitializing the Flow Director filter table
4984 ixgbe_reinit_fdir(void *context, int pending)
4986 struct adapter *adapter = context;
4987 struct ifnet *ifp = adapter->ifp;
4989 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4991 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4992 adapter->fdir_reinit = 0;
4993 /* re-enable flow director interrupts */
4994 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4995 /* Restart the interface */
4996 ifp->if_drv_flags |= IFF_DRV_RUNNING;
5001 /**********************************************************************
5003 * Update the board statistics counters.
5005 **********************************************************************/
5007 ixgbe_update_stats_counters(struct adapter *adapter)
5009 struct ifnet *ifp = adapter->ifp;
5010 struct ixgbe_hw *hw = &adapter->hw;
5011 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5012 u64 total_missed_rx = 0;
5014 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5015 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5016 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5017 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5019 for (int i = 0; i < 8; i++) {
5021 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5022 /* missed_rx tallies misses for the gprc workaround */
5024 /* global total per queue */
5025 adapter->stats.mpc[i] += mp;
5026 /* Running comprehensive total for stats display */
5027 total_missed_rx += adapter->stats.mpc[i];
5028 if (hw->mac.type == ixgbe_mac_82598EB)
5029 adapter->stats.rnbc[i] +=
5030 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5031 adapter->stats.pxontxc[i] +=
5032 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5033 adapter->stats.pxonrxc[i] +=
5034 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5035 adapter->stats.pxofftxc[i] +=
5036 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5037 adapter->stats.pxoffrxc[i] +=
5038 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5039 adapter->stats.pxon2offc[i] +=
5040 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5042 for (int i = 0; i < 16; i++) {
5043 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5044 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5045 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5046 adapter->stats.qbrc[i] +=
5047 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5048 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5049 adapter->stats.qbtc[i] +=
5050 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5051 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5053 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5054 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5055 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5057 /* Hardware workaround, gprc counts missed packets */
5058 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5059 adapter->stats.gprc -= missed_rx;
5061 if (hw->mac.type != ixgbe_mac_82598EB) {
5062 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5063 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5064 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5065 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5066 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5067 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5068 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5069 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5071 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5072 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5073 /* 82598 only has a counter in the high register */
5074 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5075 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5076 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5080 * Workaround: mprc hardware is incorrectly counting
5081 * broadcasts, so for now we subtract those.
5083 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5084 adapter->stats.bprc += bprc;
5085 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5086 if (hw->mac.type == ixgbe_mac_82598EB)
5087 adapter->stats.mprc -= bprc;
5089 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5090 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5091 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5092 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5093 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5094 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5096 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5097 adapter->stats.lxontxc += lxon;
5098 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5099 adapter->stats.lxofftxc += lxoff;
5100 total = lxon + lxoff;
5102 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5103 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5104 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5105 adapter->stats.gptc -= total;
5106 adapter->stats.mptc -= total;
5107 adapter->stats.ptc64 -= total;
5108 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5110 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5111 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5112 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5113 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5114 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5115 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5116 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5117 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5118 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5119 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5120 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5121 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5122 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5123 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5124 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5125 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5126 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5127 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5128 /* Only read FCOE on 82599 */
5129 if (hw->mac.type != ixgbe_mac_82598EB) {
5130 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5131 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5132 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5133 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5134 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5137 /* Fill out the OS statistics structure */
5138 ifp->if_ipackets = adapter->stats.gprc;
5139 ifp->if_opackets = adapter->stats.gptc;
5140 ifp->if_ibytes = adapter->stats.gorc;
5141 ifp->if_obytes = adapter->stats.gotc;
5142 ifp->if_imcasts = adapter->stats.mprc;
5143 ifp->if_collisions = 0;
5146 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5147 adapter->stats.rlec;
5150 /** ixgbe_sysctl_tdh_handler - Handler function
5151 * Retrieves the TDH value from the hardware
5154 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5158 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5161 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5162 error = sysctl_handle_int(oidp, &val, 0, req);
5163 if (error || !req->newptr)
5168 /** ixgbe_sysctl_tdt_handler - Handler function
5169 * Retrieves the TDT value from the hardware
5172 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5176 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5179 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5180 error = sysctl_handle_int(oidp, &val, 0, req);
5181 if (error || !req->newptr)
5186 /** ixgbe_sysctl_rdh_handler - Handler function
5187 * Retrieves the RDH value from the hardware
5190 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5194 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5197 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5198 error = sysctl_handle_int(oidp, &val, 0, req);
5199 if (error || !req->newptr)
5204 /** ixgbe_sysctl_rdt_handler - Handler function
5205 * Retrieves the RDT value from the hardware
5208 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5212 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5215 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5216 error = sysctl_handle_int(oidp, &val, 0, req);
5217 if (error || !req->newptr)
5223 * Add sysctl variables, one per statistic, to the system.
5226 ixgbe_add_hw_stats(struct adapter *adapter)
5228 struct tx_ring *txr = adapter->tx_rings;
5229 struct rx_ring *rxr = adapter->rx_rings;
5231 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5232 struct sysctl_oid *tree = adapter->sysctl_tree;
5233 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5234 struct ixgbe_hw_stats *stats = &adapter->stats;
5236 struct sysctl_oid *stat_node, *queue_node;
5237 struct sysctl_oid_list *stat_list, *queue_list;
5239 #define QUEUE_NAME_LEN 32
5240 char namebuf[QUEUE_NAME_LEN];
5242 /* Driver Statistics */
5243 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5244 CTLFLAG_RD, &adapter->dropped_pkts,
5245 "Driver dropped packets");
5246 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5247 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5248 "m_defrag() failed");
5249 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5250 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5251 "Driver tx dma failure in xmit");
5252 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5253 CTLFLAG_RD, &adapter->watchdog_events,
5254 "Watchdog timeouts");
5255 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5256 CTLFLAG_RD, &adapter->tso_tx,
5258 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5259 CTLFLAG_RD, &adapter->link_irq,
5260 "Link MSIX IRQ Handled");
5262 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5263 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5264 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5265 CTLFLAG_RD, NULL, "Queue Name");
5266 queue_list = SYSCTL_CHILDREN(queue_node);
5267 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5268 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5269 "irqs on this queue");
5270 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5271 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5272 ixgbe_sysctl_tdh_handler, "IU",
5273 "Transmit Descriptor Head");
5274 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5275 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5276 ixgbe_sysctl_tdt_handler, "IU",
5277 "Transmit Descriptor Tail");
5278 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5279 CTLFLAG_RD, &txr->no_desc_avail, 0,
5280 "Queue No Descriptor Available");
5281 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5282 CTLFLAG_RD, &txr->total_packets, 0,
5283 "Queue Packets Transmitted");
5286 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5287 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5288 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5289 CTLFLAG_RD, NULL, "Queue Name");
5290 queue_list = SYSCTL_CHILDREN(queue_node);
5293 struct lro_ctrl *lro = &rxr->lro;
5296 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5297 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5298 CTLFLAG_RD, NULL, "Queue Name");
5299 queue_list = SYSCTL_CHILDREN(queue_node);
5301 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5302 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5303 ixgbe_sysctl_rdh_handler, "IU",
5304 "Receive Descriptor Head");
5305 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5306 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5307 ixgbe_sysctl_rdt_handler, "IU",
5308 "Receive Descriptor Tail");
5309 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5310 CTLFLAG_RD, &rxr->rx_packets, 0,
5311 "Queue Packets Received");
5312 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5313 CTLFLAG_RD, &rxr->rx_bytes, 0,
5314 "Queue Bytes Received");
5316 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5317 CTLFLAG_RD, &lro->lro_queued, 0,
5319 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5320 CTLFLAG_RD, &lro->lro_flushed, 0,
5325 /* MAC stats get the own sub node */
5327 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5328 CTLFLAG_RD, NULL, "MAC Statistics");
5329 stat_list = SYSCTL_CHILDREN(stat_node);
5331 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5332 CTLFLAG_RD, &stats->crcerrs, 0,
5334 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5335 CTLFLAG_RD, &stats->illerrc, 0,
5336 "Illegal Byte Errors");
5337 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5338 CTLFLAG_RD, &stats->errbc, 0,
5340 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5341 CTLFLAG_RD, &stats->mspdc, 0,
5342 "MAC Short Packets Discarded");
5343 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5344 CTLFLAG_RD, &stats->mlfc, 0,
5345 "MAC Local Faults");
5346 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5347 CTLFLAG_RD, &stats->mrfc, 0,
5348 "MAC Remote Faults");
5349 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5350 CTLFLAG_RD, &stats->rlec, 0,
5351 "Receive Length Errors");
5352 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5353 CTLFLAG_RD, &stats->lxontxc, 0,
5354 "Link XON Transmitted");
5355 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5356 CTLFLAG_RD, &stats->lxonrxc, 0,
5357 "Link XON Received");
5358 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5359 CTLFLAG_RD, &stats->lxofftxc, 0,
5360 "Link XOFF Transmitted");
5361 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5362 CTLFLAG_RD, &stats->lxoffrxc, 0,
5363 "Link XOFF Received");
5365 /* Packet Reception Stats */
5366 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5367 CTLFLAG_RD, &stats->tor, 0,
5368 "Total Octets Received");
5369 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5370 CTLFLAG_RD, &stats->gorc, 0,
5371 "Good Octets Received");
5372 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5373 CTLFLAG_RD, &stats->tpr, 0,
5374 "Total Packets Received");
5375 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5376 CTLFLAG_RD, &stats->gprc, 0,
5377 "Good Packets Received");
5378 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5379 CTLFLAG_RD, &stats->mprc, 0,
5380 "Multicast Packets Received");
5381 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5382 CTLFLAG_RD, &stats->bprc, 0,
5383 "Broadcast Packets Received");
5384 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5385 CTLFLAG_RD, &stats->prc64, 0,
5386 "64 byte frames received ");
5387 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5388 CTLFLAG_RD, &stats->prc127, 0,
5389 "65-127 byte frames received");
5390 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5391 CTLFLAG_RD, &stats->prc255, 0,
5392 "128-255 byte frames received");
5393 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5394 CTLFLAG_RD, &stats->prc511, 0,
5395 "256-511 byte frames received");
5396 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5397 CTLFLAG_RD, &stats->prc1023, 0,
5398 "512-1023 byte frames received");
5399 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5400 CTLFLAG_RD, &stats->prc1522, 0,
5401 "1023-1522 byte frames received");
5402 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5403 CTLFLAG_RD, &stats->ruc, 0,
5404 "Receive Undersized");
5405 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5406 CTLFLAG_RD, &stats->rfc, 0,
5407 "Fragmented Packets Received ");
5408 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5409 CTLFLAG_RD, &stats->roc, 0,
5410 "Oversized Packets Received");
5411 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5412 CTLFLAG_RD, &stats->rjc, 0,
5414 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5415 CTLFLAG_RD, &stats->mngprc, 0,
5416 "Management Packets Received");
5417 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5418 CTLFLAG_RD, &stats->mngptc, 0,
5419 "Management Packets Dropped");
5420 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5421 CTLFLAG_RD, &stats->xec, 0,
5424 /* Packet Transmission Stats */
5425 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5426 CTLFLAG_RD, &stats->gotc, 0,
5427 "Good Octets Transmitted");
5428 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5429 CTLFLAG_RD, &stats->tpt, 0,
5430 "Total Packets Transmitted");
5431 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5432 CTLFLAG_RD, &stats->gptc, 0,
5433 "Good Packets Transmitted");
5434 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5435 CTLFLAG_RD, &stats->bptc, 0,
5436 "Broadcast Packets Transmitted");
5437 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5438 CTLFLAG_RD, &stats->mptc, 0,
5439 "Multicast Packets Transmitted");
5440 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5441 CTLFLAG_RD, &stats->mngptc, 0,
5442 "Management Packets Transmitted");
5443 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5444 CTLFLAG_RD, &stats->ptc64, 0,
5445 "64 byte frames transmitted ");
5446 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5447 CTLFLAG_RD, &stats->ptc127, 0,
5448 "65-127 byte frames transmitted");
5449 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5450 CTLFLAG_RD, &stats->ptc255, 0,
5451 "128-255 byte frames transmitted");
5452 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5453 CTLFLAG_RD, &stats->ptc511, 0,
5454 "256-511 byte frames transmitted");
5455 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5456 CTLFLAG_RD, &stats->ptc1023, 0,
5457 "512-1023 byte frames transmitted");
5458 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5459 CTLFLAG_RD, &stats->ptc1522, 0,
5460 "1024-1522 byte frames transmitted");
5463 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5464 CTLFLAG_RD, &stats->fccrc, 0,
5466 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5467 CTLFLAG_RD, &stats->fclast, 0,
5469 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5470 CTLFLAG_RD, &stats->fcoerpdc, 0,
5471 "FCoE Packets Dropped");
5472 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5473 CTLFLAG_RD, &stats->fcoeprc, 0,
5474 "FCoE Packets Received");
5475 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5476 CTLFLAG_RD, &stats->fcoeptc, 0,
5477 "FCoE Packets Transmitted");
5478 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5479 CTLFLAG_RD, &stats->fcoedwrc, 0,
5480 "FCoE DWords Received");
5481 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5482 CTLFLAG_RD, &stats->fcoedwtc, 0,
5483 "FCoE DWords Transmitted");
5487 ** Set flow control using sysctl:
5488 ** Flow control values:
5495 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5498 struct adapter *adapter = (struct adapter *) arg1;
5501 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5502 if ((error) || (req->newptr == NULL))
5505 /* Don't bother if it's not changed */
5506 if (adapter->fc == last)
5509 switch (adapter->fc) {
5510 case ixgbe_fc_rx_pause:
5511 case ixgbe_fc_tx_pause:
5513 adapter->hw.fc.requested_mode = adapter->fc;
5517 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5519 /* Don't autoneg if forcing a value */
5520 adapter->hw.fc.disable_fc_autoneg = TRUE;
5521 ixgbe_fc_enable(&adapter->hw);
5526 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5527 const char *description, int *limit, int value)
5530 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5531 SYSCTL_CHILDREN(adapter->sysctl_tree),
5532 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5536 ** Control link advertise speed:
5537 ** 1 - advertise only 1G
5538 ** 2 - advertise 100Mb
5539 ** 3 - advertise normal
5542 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5545 struct adapter *adapter;
5547 struct ixgbe_hw *hw;
5548 ixgbe_link_speed speed, last;
5550 adapter = (struct adapter *) arg1;
5553 last = adapter->advertise;
5555 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5556 if ((error) || (adapter->advertise == -1))
5559 if (adapter->advertise == last) /* no change */
5562 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5563 (hw->phy.multispeed_fiber)))
5566 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5567 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5571 if (adapter->advertise == 1)
5572 speed = IXGBE_LINK_SPEED_1GB_FULL;
5573 else if (adapter->advertise == 2)
5574 speed = IXGBE_LINK_SPEED_100_FULL;
5575 else if (adapter->advertise == 3)
5576 speed = IXGBE_LINK_SPEED_1GB_FULL |
5577 IXGBE_LINK_SPEED_10GB_FULL;
5578 else /* bogus value */
5581 hw->mac.autotry_restart = TRUE;
5582 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5588 ** Thermal Shutdown Trigger
5589 ** - cause a Thermal Overtemp IRQ
5592 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5594 int error, fire = 0;
5595 struct adapter *adapter = (struct adapter *) arg1;
5596 struct ixgbe_hw *hw = &adapter->hw;
5599 if (hw->mac.type != ixgbe_mac_X540)
5602 error = sysctl_handle_int(oidp, &fire, 0, req);
5603 if ((error) || (req->newptr == NULL))
5607 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5608 reg |= IXGBE_EICR_TS;
5609 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5616 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5620 /* convert rate in intr/s to hw representation */
5622 eitr = 1000000 / rate;
5623 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5626 /* Don't disable it */
5627 eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5628 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5629 /* Don't allow it to be too large */
5630 eitr = IXGBE_EITR_INTVL_MASK;
5633 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5637 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5639 struct adapter *sc = (void *)arg1;
5640 struct ifnet *ifp = sc->ifp;
5641 int error, intr_rate, running;
5642 struct ix_queue *que = sc->queues;
5644 intr_rate = sc->intr_rate;
5645 error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5646 if (error || req->newptr == NULL)
5651 ifnet_serialize_all(ifp);
5653 sc->intr_rate = intr_rate;
5654 running = ifp->if_flags & IFF_RUNNING;
5656 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5658 if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5659 for (int i = 0; i < sc->num_queues; i++, que++)
5660 ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5664 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5666 ifnet_deserialize_all(ifp);
5671 /* rearrange mbuf chain to get contiguous bytes */
5673 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5675 int hoff, iphlen, thoff;
5679 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5681 iphlen = m->m_pkthdr.csum_iphlen;
5682 thoff = m->m_pkthdr.csum_thlen;
5683 hoff = m->m_pkthdr.csum_lhlen;
5685 KASSERT(iphlen > 0, ("invalid ip hlen"));
5686 KASSERT(thoff > 0, ("invalid tcp hlen"));
5687 KASSERT(hoff > 0, ("invalid ether hlen"));
5689 if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5690 m = m_pullup(m, hoff + iphlen + thoff);