1 /******************************************************************************
3 Copyright (c) 2001-2012, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
36 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 static void ixgbe_start(struct ifnet *);
106 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111 static void ixgbe_qflush(struct ifnet *);
113 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void ixgbe_init(void *);
115 static void ixgbe_init_locked(struct adapter *);
116 static void ixgbe_stop(void *);
117 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int ixgbe_media_change(struct ifnet *);
119 static void ixgbe_identify_hardware(struct adapter *);
120 static int ixgbe_allocate_pci_resources(struct adapter *);
121 static int ixgbe_allocate_msix(struct adapter *);
122 static int ixgbe_allocate_legacy(struct adapter *);
123 static int ixgbe_allocate_queues(struct adapter *);
124 static int ixgbe_setup_msix(struct adapter *);
125 static void ixgbe_free_pci_resources(struct adapter *);
126 static void ixgbe_local_timer(void *);
127 static int ixgbe_setup_interface(device_t, struct adapter *);
128 static void ixgbe_config_link(struct adapter *);
130 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
131 static int ixgbe_setup_transmit_structures(struct adapter *);
132 static void ixgbe_setup_transmit_ring(struct tx_ring *);
133 static void ixgbe_initialize_transmit_units(struct adapter *);
134 static void ixgbe_free_transmit_structures(struct adapter *);
135 static void ixgbe_free_transmit_buffers(struct tx_ring *);
137 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
138 static int ixgbe_setup_receive_structures(struct adapter *);
139 static int ixgbe_setup_receive_ring(struct rx_ring *);
140 static void ixgbe_initialize_receive_units(struct adapter *);
141 static void ixgbe_free_receive_structures(struct adapter *);
142 static void ixgbe_free_receive_buffers(struct rx_ring *);
144 static void ixgbe_setup_hw_rsc(struct rx_ring *);
147 static void ixgbe_enable_intr(struct adapter *);
148 static void ixgbe_disable_intr(struct adapter *);
149 static void ixgbe_update_stats_counters(struct adapter *);
150 static void ixgbe_txeof(struct tx_ring *);
151 static void ixgbe_rxeof(struct ix_queue *, int);
152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void ixgbe_set_promisc(struct adapter *);
154 static void ixgbe_set_multi(struct adapter *);
155 static void ixgbe_update_link_status(struct adapter *);
156 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
162 struct ixgbe_dma_alloc *, int);
163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
165 const char *, int *, int);
166 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
167 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
168 static int ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
169 static void ixgbe_add_sysctl(struct adapter *);
170 static void ixgbe_set_eitr(struct adapter *, int, int);
171 static int ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
172 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void ixgbe_configure_ivars(struct adapter *);
174 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
176 static void ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
180 static void ixgbe_add_hw_stats(struct adapter *adapter);
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
186 /* Support for pluggable optic modules */
187 static bool ixgbe_sfp_probe(struct adapter *);
188 static void ixgbe_setup_optics(struct adapter *);
190 /* Legacy (single vector interrupt handler */
191 static void ixgbe_legacy_irq(void *);
193 /* The MSI/X Interrupt handlers */
194 static void ixgbe_msix_que(void *);
195 static void ixgbe_msix_link(void *);
197 /* Deferred interrupt tasklets */
198 static void ixgbe_handle_msf(void *, int);
199 static void ixgbe_handle_mod(void *, int);
202 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
203 static void ixgbe_reinit_fdir(void *, int);
206 /*********************************************************************
207 * FreeBSD Device Interface Entry Points
208 *********************************************************************/
210 static device_method_t ixgbe_methods[] = {
211 /* Device interface */
212 DEVMETHOD(device_probe, ixgbe_probe),
213 DEVMETHOD(device_attach, ixgbe_attach),
214 DEVMETHOD(device_detach, ixgbe_detach),
215 DEVMETHOD(device_shutdown, ixgbe_shutdown),
219 static driver_t ixgbe_driver = {
220 "ix", ixgbe_methods, sizeof(struct adapter),
223 devclass_t ixgbe_devclass;
224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230 ** TUNEABLE PARAMETERS:
233 /* How many packets rxeof tries to clean at a time */
234 static int ixgbe_rx_process_limit = 128;
235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
238 ** Smart speed setting, default to on
239 ** this only works as a compile option
240 ** right now as its during attach, set
241 ** this to 'ixgbe_smart_speed_off' to
244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
246 static int ixgbe_msi_enable = 1;
247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
250 * MSIX should be the default for best performance,
251 * but this allows it to be forced off for testing.
253 static int ixgbe_enable_msix = 1;
254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
257 * Header split: this causes the hardware to DMA
258 * the header into a separate mbuf from the payload,
259 * it can be a performance win in some workloads, but
260 * in others it actually hurts, its off by default.
262 static int ixgbe_header_split = FALSE;
263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
266 * Number of Queues, can be set to 0,
267 * it then autoconfigures based on the
268 * number of cpus with a max of 8. This
269 * can be overriden manually here.
271 static int ixgbe_num_queues = 0;
272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
275 ** Number of TX descriptors per ring,
276 ** setting higher than RX as this seems
277 ** the better performing choice.
279 static int ixgbe_txd = PERFORM_TXD;
280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
282 /* Number of RX descriptors per ring */
283 static int ixgbe_rxd = PERFORM_RXD;
284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
286 /* Keep running tab on them for sanity check */
287 static int ixgbe_total_ports;
291 ** For Flow Director: this is the
292 ** number of TX packets we sample
293 ** for the filter pool, this means
294 ** every 20th packet will be probed.
296 ** This feature can be disabled by
297 ** setting this to 0.
299 static int atr_sample_rate = 20;
301 ** Flow Director actually 'steals'
302 ** part of the packet buffer as its
303 ** filter pool, this variable controls
305 ** 0 = 64K, 1 = 128K, 2 = 256K
307 static int fdir_pballoc = 1;
312 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
313 * be a reference on how to implement netmap support in a driver.
314 * Additional comments are in ixgbe_netmap.h .
316 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
317 * that extend the standard driver.
319 #include <dev/netmap/ixgbe_netmap.h>
320 #endif /* DEV_NETMAP */
322 /*********************************************************************
323 * Device identification routine
325 * ixgbe_probe determines if the driver should be loaded on
326 * adapter based on PCI vendor/device id of the adapter.
328 * return BUS_PROBE_DEFAULT on success, positive on failure
329 *********************************************************************/
332 ixgbe_probe(device_t dev)
334 ixgbe_vendor_info_t *ent;
336 u16 pci_vendor_id = 0;
337 u16 pci_device_id = 0;
338 u16 pci_subvendor_id = 0;
339 u16 pci_subdevice_id = 0;
340 char adapter_name[256];
342 INIT_DEBUGOUT("ixgbe_probe: begin");
344 pci_vendor_id = pci_get_vendor(dev);
345 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
348 pci_device_id = pci_get_device(dev);
349 pci_subvendor_id = pci_get_subvendor(dev);
350 pci_subdevice_id = pci_get_subdevice(dev);
352 ent = ixgbe_vendor_info_array;
353 while (ent->vendor_id != 0) {
354 if ((pci_vendor_id == ent->vendor_id) &&
355 (pci_device_id == ent->device_id) &&
357 ((pci_subvendor_id == ent->subvendor_id) ||
358 (ent->subvendor_id == 0)) &&
360 ((pci_subdevice_id == ent->subdevice_id) ||
361 (ent->subdevice_id == 0))) {
362 ksprintf(adapter_name, "%s, Version - %s",
363 ixgbe_strings[ent->index],
364 ixgbe_driver_version);
365 device_set_desc_copy(dev, adapter_name);
367 return (BUS_PROBE_DEFAULT);
374 /*********************************************************************
375 * Device initialization routine
377 * The attach entry point is called when the driver is being loaded.
378 * This routine identifies the type of hardware, allocates all resources
379 * and initializes the hardware.
381 * return 0 on success, positive on failure
382 *********************************************************************/
385 ixgbe_attach(device_t dev)
387 struct adapter *adapter;
393 INIT_DEBUGOUT("ixgbe_attach: begin");
395 if (resource_disabled("ixgbe", device_get_unit(dev))) {
396 device_printf(dev, "Disabled by device hint\n");
400 /* Allocate, clear, and link in our adapter structure */
401 adapter = device_get_softc(dev);
402 adapter->dev = adapter->osdep.dev = dev;
406 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
408 /* Set up the timer callout */
409 callout_init_mp(&adapter->timer);
411 /* Determine hardware revision */
412 ixgbe_identify_hardware(adapter);
414 /* Enable bus mastering */
415 pci_enable_busmaster(dev);
417 /* Do base PCI setup - map BAR0 */
418 if (ixgbe_allocate_pci_resources(adapter)) {
419 device_printf(dev, "Allocation of PCI resources failed\n");
424 /* Do descriptor calc and sanity checks */
425 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
426 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
427 device_printf(dev, "TXD config issue, using default!\n");
428 adapter->num_tx_desc = DEFAULT_TXD;
430 adapter->num_tx_desc = ixgbe_txd;
433 ** With many RX rings it is easy to exceed the
434 ** system mbuf allocation. Tuning nmbclusters
435 ** can alleviate this.
437 if (nmbclusters > 0 ) {
439 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
440 if (s > nmbclusters) {
441 device_printf(dev, "RX Descriptors exceed "
442 "system mbuf max, using default instead!\n");
443 ixgbe_rxd = DEFAULT_RXD;
447 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
448 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
449 device_printf(dev, "RXD config issue, using default!\n");
450 adapter->num_rx_desc = DEFAULT_RXD;
452 adapter->num_rx_desc = ixgbe_rxd;
454 /* Allocate our TX/RX Queues */
455 if (ixgbe_allocate_queues(adapter)) {
460 /* Allocate multicast array memory. */
461 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
462 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
463 if (adapter->mta == NULL) {
464 device_printf(dev, "Can not allocate multicast setup array\n");
469 /* Initialize the shared code */
470 error = ixgbe_init_shared_code(hw);
471 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
473 ** No optics in this port, set up
474 ** so the timer routine will probe
475 ** for later insertion.
477 adapter->sfp_probe = TRUE;
479 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
480 device_printf(dev,"Unsupported SFP+ module detected!\n");
484 device_printf(dev,"Unable to initialize the shared code\n");
489 /* Make sure we have a good EEPROM before we read from it */
490 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
491 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
496 error = ixgbe_init_hw(hw);
498 case IXGBE_ERR_EEPROM_VERSION:
499 device_printf(dev, "This device is a pre-production adapter/"
500 "LOM. Please be aware there may be issues associated "
501 "with your hardware.\n If you are experiencing problems "
502 "please contact your Intel or hardware representative "
503 "who provided you with this hardware.\n");
505 case IXGBE_ERR_SFP_NOT_SUPPORTED:
506 device_printf(dev,"Unsupported SFP+ Module\n");
508 device_printf(dev,"Hardware Initialization Failure\n");
510 case IXGBE_ERR_SFP_NOT_PRESENT:
511 device_printf(dev,"No SFP+ Module found\n");
517 /* Detect and set physical type */
518 ixgbe_setup_optics(adapter);
520 if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
521 adapter->intr_type = PCI_INTR_TYPE_MSIX;
522 error = ixgbe_allocate_msix(adapter);
524 error = ixgbe_allocate_legacy(adapter);
529 /* Setup OS specific network interface */
530 if (ixgbe_setup_interface(dev, adapter) != 0)
533 /* Add sysctl tree */
534 ixgbe_add_sysctl(adapter);
536 /* Initialize statistics */
537 ixgbe_update_stats_counters(adapter);
539 /* Register for VLAN events */
540 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
541 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
542 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
543 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
545 /* Print PCIE bus type/speed/width info */
546 ixgbe_get_bus_info(hw);
547 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
548 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
549 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
550 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
551 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
552 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
555 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
556 (hw->bus.speed == ixgbe_bus_speed_2500)) {
557 device_printf(dev, "PCI-Express bandwidth available"
558 " for this card\n is not sufficient for"
559 " optimal performance.\n");
560 device_printf(dev, "For optimal performance a x8 "
561 "PCIE, or x4 PCIE 2 slot is required.\n");
564 /* let hardware know driver is loaded */
565 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
566 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
567 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
569 ixgbe_add_hw_stats(adapter);
572 ixgbe_netmap_attach(adapter);
573 #endif /* DEV_NETMAP */
574 INIT_DEBUGOUT("ixgbe_attach: end");
577 ixgbe_free_transmit_structures(adapter);
578 ixgbe_free_receive_structures(adapter);
580 if (adapter->ifp != NULL)
581 if_free(adapter->ifp);
582 ixgbe_free_pci_resources(adapter);
583 kfree(adapter->mta, M_DEVBUF);
588 /*********************************************************************
589 * Device removal routine
591 * The detach entry point is called when the driver is being removed.
592 * This routine stops the adapter and deallocates all the resources
593 * that were allocated for driver operation.
595 * return 0 on success, positive on failure
596 *********************************************************************/
599 ixgbe_detach(device_t dev)
601 struct adapter *adapter = device_get_softc(dev);
604 INIT_DEBUGOUT("ixgbe_detach: begin");
606 /* Make sure VLANS are not using driver */
607 if (adapter->ifp->if_vlantrunks != NULL) {
608 device_printf(dev,"Vlan in use, detach first\n");
612 IXGBE_CORE_LOCK(adapter);
614 IXGBE_CORE_UNLOCK(adapter);
616 /* Drain the Link queue */
618 taskqueue_drain(adapter->tq, &adapter->mod_task);
619 taskqueue_drain(adapter->tq, &adapter->msf_task);
621 taskqueue_drain(adapter->tq, &adapter->fdir_task);
623 taskqueue_free(adapter->tq);
626 /* let hardware know driver is unloading */
627 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
628 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
629 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
631 /* Unregister VLAN events */
632 if (adapter->vlan_attach != NULL)
633 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
634 if (adapter->vlan_detach != NULL)
635 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
637 ether_ifdetach(adapter->ifp);
638 callout_stop(&adapter->timer);
640 netmap_detach(adapter->ifp);
641 #endif /* DEV_NETMAP */
642 ixgbe_free_pci_resources(adapter);
643 bus_generic_detach(dev);
644 if_free(adapter->ifp);
646 ixgbe_free_transmit_structures(adapter);
647 ixgbe_free_receive_structures(adapter);
648 kfree(adapter->mta, M_DEVBUF);
649 sysctl_ctx_free(&adapter->sysctl_ctx);
651 IXGBE_CORE_LOCK_DESTROY(adapter);
655 /*********************************************************************
657 * Shutdown entry point
659 **********************************************************************/
662 ixgbe_shutdown(device_t dev)
664 struct adapter *adapter = device_get_softc(dev);
665 IXGBE_CORE_LOCK(adapter);
667 IXGBE_CORE_UNLOCK(adapter);
672 /*********************************************************************
673 * Transmit entry point
675 * ixgbe_start is called by the stack to initiate a transmit.
676 * The driver will remain in this routine as long as there are
677 * packets to transmit and transmit resources are available.
678 * In case resources are not available stack is notified and
679 * the packet is requeued.
680 **********************************************************************/
683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
686 struct adapter *adapter = txr->adapter;
688 IXGBE_TX_LOCK_ASSERT(txr);
690 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
693 if (!adapter->link_active) {
694 ifq_purge(&ifp->if_snd);
698 while (!ifq_is_empty(&ifp->if_snd)) {
699 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
700 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
704 m_head = ifq_dequeue(&ifp->if_snd, NULL);
708 if (ixgbe_xmit(txr, &m_head)) {
709 #if 0 /* XXX: prepend to an ALTQ queue ? */
711 IF_PREPEND(&ifp->if_snd, m_head);
713 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
714 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
717 /* Send a copy of the frame to the BPF listener */
718 ETHER_BPF_MTAP(ifp, m_head);
720 /* Set watchdog on */
721 txr->watchdog_time = ticks;
722 txr->queue_status = IXGBE_QUEUE_WORKING;
729 * Legacy TX start - called by the stack, this
730 * always uses the first tx ring, and should
731 * not be used with multiqueue tx enabled.
734 ixgbe_start(struct ifnet *ifp)
736 struct adapter *adapter = ifp->if_softc;
737 struct tx_ring *txr = adapter->tx_rings;
739 if (ifp->if_flags & IFF_RUNNING) {
741 ixgbe_start_locked(txr, ifp);
742 IXGBE_TX_UNLOCK(txr);
747 #if 0 /* __FreeBSD_version >= 800000 */
749 ** Multiqueue Transmit driver
753 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
755 struct adapter *adapter = ifp->if_softc;
756 struct ix_queue *que;
760 /* Which queue to use */
761 if ((m->m_flags & M_FLOWID) != 0)
762 i = m->m_pkthdr.flowid % adapter->num_queues;
764 i = curcpu % adapter->num_queues;
766 txr = &adapter->tx_rings[i];
767 que = &adapter->queues[i];
769 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
770 IXGBE_TX_TRYLOCK(txr)) {
771 err = ixgbe_mq_start_locked(ifp, txr, m);
772 IXGBE_TX_UNLOCK(txr);
774 err = drbr_enqueue(ifp, txr->br, m);
775 taskqueue_enqueue(que->tq, &que->que_task);
782 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
784 struct adapter *adapter = txr->adapter;
786 int enqueued, err = 0;
788 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
789 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
790 adapter->link_active == 0) {
792 err = drbr_enqueue(ifp, txr->br, m);
798 next = drbr_dequeue(ifp, txr->br);
799 } else if (drbr_needs_enqueue(ifp, txr->br)) {
800 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
802 next = drbr_dequeue(ifp, txr->br);
806 /* Process the queue */
807 while (next != NULL) {
808 if ((err = ixgbe_xmit(txr, &next)) != 0) {
810 err = drbr_enqueue(ifp, txr->br, next);
814 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
815 /* Send a copy of the frame to the BPF listener */
816 ETHER_BPF_MTAP(ifp, next);
817 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
819 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
821 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
822 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
825 next = drbr_dequeue(ifp, txr->br);
829 /* Set watchdog on */
830 txr->queue_status |= IXGBE_QUEUE_WORKING;
831 txr->watchdog_time = ticks;
834 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
841 ** Flush all ring buffers
844 ixgbe_qflush(struct ifnet *ifp)
846 struct adapter *adapter = ifp->if_softc;
847 struct tx_ring *txr = adapter->tx_rings;
850 for (int i = 0; i < adapter->num_queues; i++, txr++) {
852 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
854 IXGBE_TX_UNLOCK(txr);
858 #endif /* __FreeBSD_version >= 800000 */
860 /*********************************************************************
863 * ixgbe_ioctl is called when the user wants to configure the
866 * return 0 on success, positive on failure
867 **********************************************************************/
870 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
872 struct adapter *adapter = ifp->if_softc;
873 struct ifreq *ifr = (struct ifreq *) data;
874 #if defined(INET) || defined(INET6)
875 struct ifaddr *ifa = (struct ifaddr *)data;
876 bool avoid_reset = FALSE;
884 if (ifa->ifa_addr->sa_family == AF_INET)
888 if (ifa->ifa_addr->sa_family == AF_INET6)
891 #if defined(INET) || defined(INET6)
893 ** Calling init results in link renegotiation,
894 ** so we avoid doing it when possible.
897 ifp->if_flags |= IFF_UP;
898 if (!(ifp->if_flags & IFF_RUNNING))
900 if (!(ifp->if_flags & IFF_NOARP))
901 arp_ifinit(ifp, ifa);
903 error = ether_ioctl(ifp, command, data);
907 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
908 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
911 IXGBE_CORE_LOCK(adapter);
912 ifp->if_mtu = ifr->ifr_mtu;
913 adapter->max_frame_size =
914 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
915 ixgbe_init_locked(adapter);
916 IXGBE_CORE_UNLOCK(adapter);
920 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
921 IXGBE_CORE_LOCK(adapter);
922 if (ifp->if_flags & IFF_UP) {
923 if ((ifp->if_flags & IFF_RUNNING)) {
924 if ((ifp->if_flags ^ adapter->if_flags) &
925 (IFF_PROMISC | IFF_ALLMULTI)) {
926 ixgbe_set_promisc(adapter);
929 ixgbe_init_locked(adapter);
931 if (ifp->if_flags & IFF_RUNNING)
933 adapter->if_flags = ifp->if_flags;
934 IXGBE_CORE_UNLOCK(adapter);
938 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
939 if (ifp->if_flags & IFF_RUNNING) {
940 IXGBE_CORE_LOCK(adapter);
941 ixgbe_disable_intr(adapter);
942 ixgbe_set_multi(adapter);
943 ixgbe_enable_intr(adapter);
944 IXGBE_CORE_UNLOCK(adapter);
949 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
950 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
954 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
955 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
956 if (mask & IFCAP_HWCSUM)
957 ifp->if_capenable ^= IFCAP_HWCSUM;
958 if (mask & IFCAP_TSO4)
959 ifp->if_capenable ^= IFCAP_TSO4;
960 if (mask & IFCAP_TSO6)
961 ifp->if_capenable ^= IFCAP_TSO6;
963 if (mask & IFCAP_LRO)
964 ifp->if_capenable ^= IFCAP_LRO;
966 if (mask & IFCAP_VLAN_HWTAGGING)
967 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
968 if (mask & IFCAP_VLAN_HWFILTER)
969 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
971 if (mask & IFCAP_VLAN_HWTSO)
972 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
974 if (ifp->if_flags & IFF_RUNNING) {
975 IXGBE_CORE_LOCK(adapter);
976 ixgbe_init_locked(adapter);
977 IXGBE_CORE_UNLOCK(adapter);
980 VLAN_CAPABILITIES(ifp);
986 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
987 error = ether_ioctl(ifp, command, data);
994 /*********************************************************************
997 * This routine is used in two ways. It is used by the stack as
998 * init entry point in network interface structure. It is also used
999 * by the driver as a hw/sw initialization routine to get to a
1002 * return 0 on success, positive on failure
1003 **********************************************************************/
1004 #define IXGBE_MHADD_MFS_SHIFT 16
1007 ixgbe_init_locked(struct adapter *adapter)
1009 struct ifnet *ifp = adapter->ifp;
1010 device_t dev = adapter->dev;
1011 struct ixgbe_hw *hw = &adapter->hw;
1012 u32 k, txdctl, mhadd, gpie;
1015 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1016 INIT_DEBUGOUT("ixgbe_init: begin");
1017 hw->adapter_stopped = FALSE;
1018 ixgbe_stop_adapter(hw);
1019 callout_stop(&adapter->timer);
1021 /* reprogram the RAR[0] in case user changed it. */
1022 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1024 /* Get the latest mac address, User can use a LAA */
1025 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1026 IXGBE_ETH_LENGTH_OF_ADDRESS);
1027 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1028 hw->addr_ctrl.rar_used_count = 1;
1030 /* Set the various hardware offload abilities */
1031 ifp->if_hwassist = 0;
1032 if (ifp->if_capenable & IFCAP_TSO)
1033 ifp->if_hwassist |= CSUM_TSO;
1034 if (ifp->if_capenable & IFCAP_TXCSUM) {
1035 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1037 if (hw->mac.type != ixgbe_mac_82598EB)
1038 ifp->if_hwassist |= CSUM_SCTP;
1042 /* Prepare transmit descriptors and buffers */
1043 if (ixgbe_setup_transmit_structures(adapter)) {
1044 device_printf(dev,"Could not setup transmit structures\n");
1045 ixgbe_stop(adapter);
1050 ixgbe_initialize_transmit_units(adapter);
1052 /* Setup Multicast table */
1053 ixgbe_set_multi(adapter);
1056 ** Determine the correct mbuf pool
1057 ** for doing jumbo/headersplit
1060 if (adapter->max_frame_size <= 2048)
1061 adapter->rx_mbuf_sz = MCLBYTES;
1062 else if (adapter->max_frame_size <= 4096)
1063 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1064 else if (adapter->max_frame_size <= 9216)
1065 adapter->rx_mbuf_sz = MJUM9BYTES;
1067 adapter->rx_mbuf_sz = MJUM16BYTES;
1069 adapter->rx_mbuf_sz = MCLBYTES;
1072 /* Prepare receive descriptors and buffers */
1073 if (ixgbe_setup_receive_structures(adapter)) {
1074 device_printf(dev,"Could not setup receive structures\n");
1075 ixgbe_stop(adapter);
1079 /* Configure RX settings */
1080 ixgbe_initialize_receive_units(adapter);
1082 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1084 /* Enable Fan Failure Interrupt */
1085 gpie |= IXGBE_SDP1_GPIEN;
1087 /* Add for Module detection */
1088 if (hw->mac.type == ixgbe_mac_82599EB)
1089 gpie |= IXGBE_SDP2_GPIEN;
1091 /* Thermal Failure Detection */
1092 if (hw->mac.type == ixgbe_mac_X540)
1093 gpie |= IXGBE_SDP0_GPIEN;
1095 if (adapter->msix > 1) {
1096 /* Enable Enhanced MSIX mode */
1097 gpie |= IXGBE_GPIE_MSIX_MODE;
1098 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1101 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1104 if (ifp->if_mtu > ETHERMTU) {
1105 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1106 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1107 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1108 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1111 /* Now enable all the queues */
1113 for (int i = 0; i < adapter->num_queues; i++) {
1114 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1115 txdctl |= IXGBE_TXDCTL_ENABLE;
1116 /* Set WTHRESH to 8, burst writeback */
1117 txdctl |= (8 << 16);
1119 * When the internal queue falls below PTHRESH (32),
1120 * start prefetching as long as there are at least
1121 * HTHRESH (1) buffers ready. The values are taken
1122 * from the Intel linux driver 3.8.21.
1123 * Prefetching enables tx line rate even with 1 queue.
1125 txdctl |= (32 << 0) | (1 << 8);
1126 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1129 for (int i = 0; i < adapter->num_queues; i++) {
1130 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1131 if (hw->mac.type == ixgbe_mac_82598EB) {
1137 rxdctl &= ~0x3FFFFF;
1140 rxdctl |= IXGBE_RXDCTL_ENABLE;
1141 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1142 for (k = 0; k < 10; k++) {
1143 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1144 IXGBE_RXDCTL_ENABLE)
1152 * In netmap mode, we must preserve the buffers made
1153 * available to userspace before the if_init()
1154 * (this is true by default on the TX side, because
1155 * init makes all buffers available to userspace).
1157 * netmap_reset() and the device specific routines
1158 * (e.g. ixgbe_setup_receive_rings()) map these
1159 * buffers at the end of the NIC ring, so here we
1160 * must set the RDT (tail) register to make sure
1161 * they are not overwritten.
1163 * In this driver the NIC ring starts at RDH = 0,
1164 * RDT points to the last slot available for reception (?),
1165 * so RDT = num_rx_desc - 1 means the whole ring is available.
1167 if (ifp->if_capenable & IFCAP_NETMAP) {
1168 struct netmap_adapter *na = NA(adapter->ifp);
1169 struct netmap_kring *kring = &na->rx_rings[i];
1170 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1172 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1174 #endif /* DEV_NETMAP */
1175 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1178 /* Set up VLAN support and filter */
1179 ixgbe_setup_vlan_hw_support(adapter);
1181 /* Enable Receive engine */
1182 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1183 if (hw->mac.type == ixgbe_mac_82598EB)
1184 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1185 rxctrl |= IXGBE_RXCTRL_RXEN;
1186 ixgbe_enable_rx_dma(hw, rxctrl);
1188 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1190 /* Set up MSI/X routing */
1191 if (ixgbe_enable_msix) {
1192 ixgbe_configure_ivars(adapter);
1193 /* Set up auto-mask */
1194 if (hw->mac.type == ixgbe_mac_82598EB)
1195 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1197 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1198 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1200 } else { /* Simple settings for Legacy/MSI */
1201 ixgbe_set_ivar(adapter, 0, 0, 0);
1202 ixgbe_set_ivar(adapter, 0, 0, 1);
1203 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1207 /* Init Flow director */
1208 if (hw->mac.type != ixgbe_mac_82598EB) {
1209 u32 hdrm = 32 << fdir_pballoc;
1211 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1212 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1217 ** Check on any SFP devices that
1218 ** need to be kick-started
1220 if (hw->phy.type == ixgbe_phy_none) {
1221 int err = hw->phy.ops.identify(hw);
1222 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1224 "Unsupported SFP+ module type was detected.\n");
1229 /* Set moderation on the Link interrupt */
1230 ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1232 /* Config/Enable Link */
1233 ixgbe_config_link(adapter);
1235 /* Hardware Packet Buffer & Flow Control setup */
1237 u32 rxpb, frame, size, tmp;
1239 frame = adapter->max_frame_size;
1241 /* Calculate High Water */
1242 if (hw->mac.type == ixgbe_mac_X540)
1243 tmp = IXGBE_DV_X540(frame, frame);
1245 tmp = IXGBE_DV(frame, frame);
1246 size = IXGBE_BT2KB(tmp);
1247 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1248 hw->fc.high_water[0] = rxpb - size;
1250 /* Now calculate Low Water */
1251 if (hw->mac.type == ixgbe_mac_X540)
1252 tmp = IXGBE_LOW_DV_X540(frame);
1254 tmp = IXGBE_LOW_DV(frame);
1255 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1257 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1258 hw->fc.pause_time = IXGBE_FC_PAUSE;
1259 hw->fc.send_xon = TRUE;
1261 /* Initialize the FC settings */
1264 /* And now turn on interrupts */
1265 ixgbe_enable_intr(adapter);
1267 /* Now inform the stack we're ready */
1268 ifp->if_flags |= IFF_RUNNING;
1269 ifp->if_flags &= ~IFF_OACTIVE;
1275 ixgbe_init(void *arg)
1277 struct adapter *adapter = arg;
1279 IXGBE_CORE_LOCK(adapter);
1280 ixgbe_init_locked(adapter);
1281 IXGBE_CORE_UNLOCK(adapter);
1288 ** MSIX Interrupt Handlers and Tasklets
1293 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1295 struct ixgbe_hw *hw = &adapter->hw;
1296 u64 queue = (u64)(1 << vector);
1299 if (hw->mac.type == ixgbe_mac_82598EB) {
1300 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1301 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1303 mask = (queue & 0xFFFFFFFF);
1305 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1306 mask = (queue >> 32);
1308 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1313 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1315 struct ixgbe_hw *hw = &adapter->hw;
1316 u64 queue = (u64)(1 << vector);
1319 if (hw->mac.type == ixgbe_mac_82598EB) {
1320 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1321 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1323 mask = (queue & 0xFFFFFFFF);
1325 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1326 mask = (queue >> 32);
1328 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1333 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1337 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1338 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1339 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1341 mask = (queues & 0xFFFFFFFF);
1342 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1343 mask = (queues >> 32);
1344 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1348 /*********************************************************************
1350 * Legacy Interrupt Service routine
1352 **********************************************************************/
1355 ixgbe_legacy_irq(void *arg)
1357 struct ix_queue *que = arg;
1358 struct adapter *adapter = que->adapter;
1359 struct ixgbe_hw *hw = &adapter->hw;
1360 struct tx_ring *txr = adapter->tx_rings;
1364 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1367 if (reg_eicr == 0) {
1368 ixgbe_enable_intr(adapter);
1372 ixgbe_rxeof(que, adapter->rx_process_limit);
1376 if (!ifq_is_empty(&adapter->ifp->if_snd))
1377 ixgbe_start_locked(txr, adapter->ifp);
1378 IXGBE_TX_UNLOCK(txr);
1380 /* Check for fan failure */
1381 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1382 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1383 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1384 "REPLACE IMMEDIATELY!!\n");
1385 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1388 /* Link status change */
1389 if (reg_eicr & IXGBE_EICR_LSC) {
1390 ixgbe_check_link(&adapter->hw,
1391 &adapter->link_speed, &adapter->link_up, 0);
1392 ixgbe_update_link_status(adapter);
1395 ixgbe_enable_intr(adapter);
1399 /*********************************************************************
1401 * MSIX Queue Interrupt Service routine
1403 **********************************************************************/
1405 ixgbe_msix_que(void *arg)
1407 struct ix_queue *que = arg;
1408 struct adapter *adapter = que->adapter;
1409 struct tx_ring *txr = que->txr;
1411 ixgbe_disable_queue(adapter, que->msix);
1414 ixgbe_rxeof(que, adapter->rx_process_limit);
1418 if (!ifq_is_empty(&adapter->ifp->if_snd))
1419 ixgbe_start_locked(txr, adapter->ifp);
1420 IXGBE_TX_UNLOCK(txr);
1422 /* Reenable this interrupt */
1423 ixgbe_enable_queue(adapter, que->msix);
1428 ixgbe_msix_link(void *arg)
1430 struct adapter *adapter = arg;
1431 struct ixgbe_hw *hw = &adapter->hw;
1434 ++adapter->link_irq;
1436 /* First get the cause */
1437 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1438 /* Clear interrupt with write */
1439 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1441 /* Link status change */
1442 if (reg_eicr & IXGBE_EICR_LSC) {
1443 ixgbe_check_link(&adapter->hw,
1444 &adapter->link_speed, &adapter->link_up, 0);
1445 ixgbe_update_link_status(adapter);
1448 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1450 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1451 /* This is probably overkill :) */
1452 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1454 /* Disable the interrupt */
1455 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1456 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1459 if (reg_eicr & IXGBE_EICR_ECC) {
1460 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1461 "Please Reboot!!\n");
1462 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1465 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1466 /* Clear the interrupt */
1467 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1468 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1469 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1470 /* Clear the interrupt */
1471 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1472 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1476 /* Check for fan failure */
1477 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1478 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1479 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1480 "REPLACE IMMEDIATELY!!\n");
1481 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1484 /* Check for over temp condition */
1485 if ((hw->mac.type == ixgbe_mac_X540) &&
1486 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1487 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1488 "PHY IS SHUT DOWN!!\n");
1489 device_printf(adapter->dev, "System shutdown required\n");
1490 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1493 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1497 /*********************************************************************
1499 * Media Ioctl callback
1501 * This routine is called whenever the user queries the status of
1502 * the interface using ifconfig.
1504 **********************************************************************/
1506 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1508 struct adapter *adapter = ifp->if_softc;
1510 ASSERT_IFNET_SERIALIZED_ALL(ifp);
1512 INIT_DEBUGOUT("ixgbe_media_status: begin");
1513 ixgbe_update_link_status(adapter);
1515 ifmr->ifm_status = IFM_AVALID;
1516 ifmr->ifm_active = IFM_ETHER;
1518 if (!adapter->link_active)
1521 ifmr->ifm_status |= IFM_ACTIVE;
1523 switch (adapter->link_speed) {
1524 case IXGBE_LINK_SPEED_100_FULL:
1525 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1527 case IXGBE_LINK_SPEED_1GB_FULL:
1528 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1530 case IXGBE_LINK_SPEED_10GB_FULL:
1531 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1538 /*********************************************************************
1540 * Media Ioctl callback
1542 * This routine is called when the user changes speed/duplex using
1543 * media/mediopt option with ifconfig.
1545 **********************************************************************/
1547 ixgbe_media_change(struct ifnet * ifp)
1549 struct adapter *adapter = ifp->if_softc;
1550 struct ifmedia *ifm = &adapter->media;
1552 INIT_DEBUGOUT("ixgbe_media_change: begin");
1554 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1557 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1559 adapter->hw.phy.autoneg_advertised =
1560 IXGBE_LINK_SPEED_100_FULL |
1561 IXGBE_LINK_SPEED_1GB_FULL |
1562 IXGBE_LINK_SPEED_10GB_FULL;
1565 device_printf(adapter->dev, "Only auto media type\n");
1572 /*********************************************************************
1574 * This routine maps the mbufs to tx descriptors, allowing the
1575 * TX engine to transmit the packets.
1576 * - return 0 on success, positive on failure
1578 **********************************************************************/
1581 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1583 struct adapter *adapter = txr->adapter;
1584 u32 olinfo_status = 0, cmd_type_len;
1586 int i, j, error, nsegs, maxsegs;
1587 int first, last = 0;
1588 struct mbuf *m_head;
1589 bus_dma_segment_t segs[adapter->num_segs];
1591 struct ixgbe_tx_buf *txbuf;
1592 union ixgbe_adv_tx_desc *txd = NULL;
1596 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1597 error = ixgbe_tso_pullup(txr, m_headp);
1603 /* Basic descriptor defines */
1604 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1605 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1607 if (m_head->m_flags & M_VLANTAG)
1608 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1611 * Important to capture the first descriptor
1612 * used because it will contain the index of
1613 * the one we tell the hardware to report back
1615 first = txr->next_avail_desc;
1616 txbuf = &txr->tx_buffers[first];
1620 * Map the packet for DMA.
1622 maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1623 if (maxsegs > adapter->num_segs)
1624 maxsegs = adapter->num_segs;
1626 error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1627 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1629 if (error == ENOBUFS)
1630 adapter->mbuf_defrag_failed++;
1632 adapter->no_tx_dma_setup++;
1639 /* Make certain there are enough descriptors */
1640 if (nsegs > txr->tx_avail - 2) {
1641 txr->no_desc_avail++;
1648 ** Set up the appropriate offload context
1649 ** this becomes the first descriptor of
1652 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1653 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1654 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1655 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1656 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1660 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1661 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1663 #ifdef IXGBE_IEEE1588
1664 /* This is changing soon to an mtag detection */
1665 if (we detect this mbuf has a TSTAMP mtag)
1666 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1670 /* Do the flow director magic */
1671 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1673 if (txr->atr_count >= atr_sample_rate) {
1674 ixgbe_atr(txr, m_head);
1679 /* Record payload length */
1681 olinfo_status |= m_head->m_pkthdr.len <<
1682 IXGBE_ADVTXD_PAYLEN_SHIFT;
1684 i = txr->next_avail_desc;
1685 for (j = 0; j < nsegs; j++) {
1689 txbuf = &txr->tx_buffers[i];
1690 txd = &txr->tx_base[i];
1691 seglen = segs[j].ds_len;
1692 segaddr = htole64(segs[j].ds_addr);
1694 txd->read.buffer_addr = segaddr;
1695 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1696 cmd_type_len |seglen);
1697 txd->read.olinfo_status = htole32(olinfo_status);
1698 last = i; /* descriptor that will get completion IRQ */
1700 if (++i == adapter->num_tx_desc)
1703 txbuf->m_head = NULL;
1704 txbuf->eop_index = -1;
1707 txd->read.cmd_type_len |=
1708 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1709 txr->tx_avail -= nsegs;
1710 txr->next_avail_desc = i;
1712 txbuf->m_head = m_head;
1713 /* Swap the dma map between the first and last descriptor */
1714 txr->tx_buffers[first].map = txbuf->map;
1716 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1718 /* Set the index of the descriptor that will be marked done */
1719 txbuf = &txr->tx_buffers[first];
1720 txbuf->eop_index = last;
1722 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1723 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1725 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1726 * hardware that this frame is available to transmit.
1728 ++txr->total_packets;
1729 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1734 bus_dmamap_unload(txr->txtag, txbuf->map);
1740 ixgbe_set_promisc(struct adapter *adapter)
1743 struct ifnet *ifp = adapter->ifp;
1745 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1746 reg_rctl &= (~IXGBE_FCTRL_UPE);
1747 reg_rctl &= (~IXGBE_FCTRL_MPE);
1748 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1750 if (ifp->if_flags & IFF_PROMISC) {
1751 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1752 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1753 } else if (ifp->if_flags & IFF_ALLMULTI) {
1754 reg_rctl |= IXGBE_FCTRL_MPE;
1755 reg_rctl &= ~IXGBE_FCTRL_UPE;
1756 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1762 /*********************************************************************
1765 * This routine is called whenever multicast address list is updated.
1767 **********************************************************************/
1768 #define IXGBE_RAR_ENTRIES 16
1771 ixgbe_set_multi(struct adapter *adapter)
1776 struct ifmultiaddr *ifma;
1778 struct ifnet *ifp = adapter->ifp;
1780 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1783 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1784 MAX_NUM_MULTICAST_ADDRESSES);
1786 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1787 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1788 if (ifp->if_flags & IFF_PROMISC)
1789 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1790 else if (ifp->if_flags & IFF_ALLMULTI) {
1791 fctrl |= IXGBE_FCTRL_MPE;
1792 fctrl &= ~IXGBE_FCTRL_UPE;
1794 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1796 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1798 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1799 if (ifma->ifma_addr->sa_family != AF_LINK)
1801 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1802 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1803 IXGBE_ETH_LENGTH_OF_ADDRESS);
1808 ixgbe_update_mc_addr_list(&adapter->hw,
1809 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1815 * This is an iterator function now needed by the multicast
1816 * shared code. It simply feeds the shared code routine the
1817 * addresses in the array of ixgbe_set_multi() one by one.
1820 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1822 u8 *addr = *update_ptr;
1826 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1827 *update_ptr = newptr;
1832 /*********************************************************************
1835 * This routine checks for link status,updates statistics,
1836 * and runs the watchdog check.
1838 **********************************************************************/
1841 ixgbe_local_timer(void *arg)
1843 struct adapter *adapter = arg;
1844 device_t dev = adapter->dev;
1845 struct ifnet *ifp = adapter->ifp;
1846 struct ix_queue *que = adapter->queues;
1847 struct tx_ring *txr = adapter->tx_rings;
1848 int hung, busy, paused;
1850 IXGBE_CORE_LOCK(adapter);
1851 hung = busy = paused = 0;
1853 /* Check for pluggable optics */
1854 if (adapter->sfp_probe)
1855 if (!ixgbe_sfp_probe(adapter))
1856 goto out; /* Nothing to do */
1858 ixgbe_update_link_status(adapter);
1859 ixgbe_update_stats_counters(adapter);
1862 * If the interface has been paused
1863 * then don't do the watchdog check
1865 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1869 ** Check the TX queues status
1870 ** - central locked handling of OACTIVE
1871 ** - watchdog only if all queues show hung
1873 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1874 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1877 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1880 /* Only truely watchdog if all queues show hung */
1881 if (hung == adapter->num_queues)
1883 /* Only turn off the stack flow when ALL are depleted */
1884 if (busy == adapter->num_queues)
1885 ifp->if_flags |= IFF_OACTIVE;
1886 else if ((ifp->if_flags & IFF_OACTIVE) &&
1887 (busy < adapter->num_queues))
1888 ifp->if_flags &= ~IFF_OACTIVE;
1891 ixgbe_rearm_queues(adapter, adapter->que_mask);
1892 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1893 IXGBE_CORE_UNLOCK(adapter);
1897 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1898 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1899 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1900 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1901 device_printf(dev,"TX(%d) desc avail = %d,"
1902 "Next TX to Clean = %d\n",
1903 txr->me, txr->tx_avail, txr->next_to_clean);
1904 adapter->ifp->if_flags &= ~IFF_RUNNING;
1905 adapter->watchdog_events++;
1906 ixgbe_init_locked(adapter);
1908 IXGBE_CORE_UNLOCK(adapter);
1912 ** Note: this routine updates the OS on the link state
1913 ** the real check of the hardware only happens with
1914 ** a link interrupt.
1917 ixgbe_update_link_status(struct adapter *adapter)
1919 struct ifnet *ifp = adapter->ifp;
1920 struct tx_ring *txr = adapter->tx_rings;
1921 device_t dev = adapter->dev;
1924 if (adapter->link_up){
1925 if (adapter->link_active == FALSE) {
1927 device_printf(dev,"Link is up %d Gbps %s \n",
1928 ((adapter->link_speed == 128)? 10:1),
1930 adapter->link_active = TRUE;
1931 /* Update any Flow Control changes */
1932 ixgbe_fc_enable(&adapter->hw);
1933 ifp->if_link_state = LINK_STATE_UP;
1934 if_link_state_change(ifp);
1936 } else { /* Link down */
1937 if (adapter->link_active == TRUE) {
1939 device_printf(dev,"Link is Down\n");
1940 ifp->if_link_state = LINK_STATE_DOWN;
1941 if_link_state_change(ifp);
1942 adapter->link_active = FALSE;
1943 for (int i = 0; i < adapter->num_queues;
1945 txr->queue_status = IXGBE_QUEUE_IDLE;
1953 /*********************************************************************
1955 * This routine disables all traffic on the adapter by issuing a
1956 * global reset on the MAC and deallocates TX/RX buffers.
1958 **********************************************************************/
1961 ixgbe_stop(void *arg)
1964 struct adapter *adapter = arg;
1965 struct ixgbe_hw *hw = &adapter->hw;
1968 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1970 INIT_DEBUGOUT("ixgbe_stop: begin\n");
1971 ixgbe_disable_intr(adapter);
1972 callout_stop(&adapter->timer);
1974 /* Let the stack know...*/
1975 ifp->if_flags &= ~IFF_RUNNING;
1976 ifp->if_flags |= IFF_OACTIVE;
1979 hw->adapter_stopped = FALSE;
1980 ixgbe_stop_adapter(hw);
1981 /* Turn off the laser */
1982 if (hw->phy.multispeed_fiber)
1983 ixgbe_disable_tx_laser(hw);
1985 /* reprogram the RAR[0] in case user changed it. */
1986 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1992 /*********************************************************************
1994 * Determine hardware revision.
1996 **********************************************************************/
1998 ixgbe_identify_hardware(struct adapter *adapter)
2000 device_t dev = adapter->dev;
2001 struct ixgbe_hw *hw = &adapter->hw;
2003 /* Save off the information about this board */
2004 hw->vendor_id = pci_get_vendor(dev);
2005 hw->device_id = pci_get_device(dev);
2006 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2007 hw->subsystem_vendor_id =
2008 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2009 hw->subsystem_device_id =
2010 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2012 /* We need this here to set the num_segs below */
2013 ixgbe_set_mac_type(hw);
2015 /* Pick up the 82599 and VF settings */
2016 if (hw->mac.type != ixgbe_mac_82598EB) {
2017 hw->phy.smart_speed = ixgbe_smart_speed;
2018 adapter->num_segs = IXGBE_82599_SCATTER;
2020 adapter->num_segs = IXGBE_82598_SCATTER;
2025 /*********************************************************************
2027 * Determine optic type
2029 **********************************************************************/
2031 ixgbe_setup_optics(struct adapter *adapter)
2033 struct ixgbe_hw *hw = &adapter->hw;
2036 layer = ixgbe_get_supported_physical_layer(hw);
2038 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2039 adapter->optics = IFM_10G_T;
2043 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2044 adapter->optics = IFM_1000_T;
2048 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2049 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2050 adapter->optics = IFM_10G_LR;
2054 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2055 adapter->optics = IFM_10G_SR;
2059 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2060 adapter->optics = IFM_10G_TWINAX;
2064 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2065 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2066 adapter->optics = IFM_10G_CX4;
2070 /* If we get here just set the default */
2071 adapter->optics = IFM_ETHER | IFM_AUTO;
2075 /*********************************************************************
2077 * Setup the Legacy or MSI Interrupt handler
2079 **********************************************************************/
2081 ixgbe_allocate_legacy(struct adapter *adapter)
2083 device_t dev = adapter->dev;
2084 struct ix_queue *que = adapter->queues;
2086 unsigned int intr_flags;
2089 if (adapter->msix == 1)
2092 /* Try allocating a MSI interrupt first */
2093 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2096 /* We allocate a single interrupt resource */
2097 adapter->res = bus_alloc_resource_any(dev,
2098 SYS_RES_IRQ, &rid, intr_flags);
2099 if (adapter->res == NULL) {
2100 device_printf(dev, "Unable to allocate bus resource: "
2105 /* Tasklets for Link, SFP and Multispeed Fiber */
2106 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2107 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2109 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2112 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2113 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2114 device_printf(dev, "Failed to register fast interrupt "
2115 "handler: %d\n", error);
2116 taskqueue_free(adapter->tq);
2120 /* For simplicity in the handlers */
2121 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2127 /*********************************************************************
2129 * Setup MSIX Interrupt resources and handlers
2131 **********************************************************************/
2133 ixgbe_allocate_msix(struct adapter *adapter)
2135 device_t dev = adapter->dev;
2136 struct ix_queue *que = adapter->queues;
2137 int error, rid, vector = 0;
2140 error = pci_setup_msix(dev);
2142 device_printf(dev, "MSI-X setup failed\n");
2146 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2150 ** Bind the msix vector, and thus the
2151 ** ring to the corresponding cpu.
2153 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2155 device_printf(dev, "pci_alloc_msix_vector failed\n");
2159 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2160 RF_SHAREABLE | RF_ACTIVE);
2161 if (que->res == NULL) {
2162 device_printf(dev,"Unable to allocate"
2163 " bus resource: que interrupt [%d]\n", vector);
2166 /* Set the handler function */
2167 ksnprintf(desc, sizeof(desc), "%s que %d",
2168 device_get_nameunit(dev), i);
2169 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2170 ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2173 device_printf(dev, "Failed to register QUE handler");
2177 adapter->que_mask |= (u64)(1 << que->msix);
2180 /* and Link, bind vector to cpu #0 */
2182 error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2184 device_printf(dev, "pci_alloc_msix_vector failed\n");
2187 adapter->res = bus_alloc_resource_any(dev,
2188 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2189 if (!adapter->res) {
2190 device_printf(dev,"Unable to allocate"
2191 " bus resource: Link interrupt [%d]\n", rid);
2194 /* Set the link handler function */
2195 error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2196 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2199 adapter->res = NULL;
2200 device_printf(dev, "Failed to register LINK handler");
2203 pci_enable_msix(dev);
2205 adapter->linkvec = vector;
2206 /* Tasklets for Link, SFP and Multispeed Fiber */
2207 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2208 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2210 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2217 * Setup Either MSI/X or MSI
2220 ixgbe_setup_msix(struct adapter *adapter)
2222 device_t dev = adapter->dev;
2223 int rid, want, queues, msgs;
2225 /* Override by tuneable */
2226 if (ixgbe_enable_msix == 0)
2229 /* First try MSI/X */
2230 rid = PCIR_BAR(MSIX_82598_BAR);
2231 adapter->msix_mem = bus_alloc_resource_any(dev,
2232 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2233 if (!adapter->msix_mem) {
2234 rid += 4; /* 82599 maps in higher BAR */
2235 adapter->msix_mem = bus_alloc_resource_any(dev,
2236 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2238 if (!adapter->msix_mem) {
2239 /* May not be enabled */
2240 device_printf(adapter->dev,
2241 "Unable to map MSIX table \n");
2245 msgs = pci_msix_count(dev);
2246 if (msgs == 0) { /* system has msix disabled */
2247 bus_release_resource(dev, SYS_RES_MEMORY,
2248 rid, adapter->msix_mem);
2249 adapter->msix_mem = NULL;
2253 /* Figure out a reasonable auto config value */
2254 queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2256 if (ixgbe_num_queues != 0)
2257 queues = ixgbe_num_queues;
2258 /* Set max queues to 8 when autoconfiguring */
2259 else if ((ixgbe_num_queues == 0) && (queues > 8))
2263 ** Want one vector (RX/TX pair) per queue
2264 ** plus an additional for Link.
2270 device_printf(adapter->dev,
2271 "MSIX Configuration Problem, "
2272 "%d vectors but %d queues wanted!\n",
2274 return (0); /* Will go to Legacy setup */
2277 device_printf(adapter->dev,
2278 "Using MSIX interrupts with %d vectors\n", msgs);
2279 adapter->num_queues = queues;
2283 msgs = pci_msi_count(dev);
2289 ixgbe_allocate_pci_resources(struct adapter *adapter)
2292 device_t dev = adapter->dev;
2295 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2298 if (!(adapter->pci_mem)) {
2299 device_printf(dev,"Unable to allocate bus resource: memory\n");
2303 adapter->osdep.mem_bus_space_tag =
2304 rman_get_bustag(adapter->pci_mem);
2305 adapter->osdep.mem_bus_space_handle =
2306 rman_get_bushandle(adapter->pci_mem);
2307 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2309 /* Legacy defaults */
2310 adapter->num_queues = 1;
2311 adapter->hw.back = &adapter->osdep;
2314 ** Now setup MSI or MSI/X, should
2315 ** return us the number of supported
2316 ** vectors. (Will be 1 for MSI)
2318 adapter->msix = ixgbe_setup_msix(adapter);
2323 ixgbe_free_pci_resources(struct adapter * adapter)
2325 struct ix_queue *que = adapter->queues;
2326 device_t dev = adapter->dev;
2329 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2330 memrid = PCIR_BAR(MSIX_82598_BAR);
2332 memrid = PCIR_BAR(MSIX_82599_BAR);
2335 ** There is a slight possibility of a failure mode
2336 ** in attach that will result in entering this function
2337 ** before interrupt resources have been initialized, and
2338 ** in that case we do not want to execute the loops below
2339 ** We can detect this reliably by the state of the adapter
2342 if (adapter->res == NULL)
2346 ** Release all msix queue resources:
2348 for (int i = 0; i < adapter->num_queues; i++, que++) {
2349 rid = que->msix + 1;
2350 if (que->tag != NULL) {
2351 bus_teardown_intr(dev, que->res, que->tag);
2354 if (que->res != NULL)
2355 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2359 /* Clean the Legacy or Link interrupt last */
2360 if (adapter->linkvec) /* we are doing MSIX */
2361 rid = adapter->linkvec + 1;
2363 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2365 if (adapter->tag != NULL) {
2366 bus_teardown_intr(dev, adapter->res, adapter->tag);
2367 adapter->tag = NULL;
2369 if (adapter->res != NULL)
2370 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2371 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2372 pci_release_msi(adapter->dev);
2376 pci_release_msi(dev);
2378 if (adapter->msix_mem != NULL)
2379 bus_release_resource(dev, SYS_RES_MEMORY,
2380 memrid, adapter->msix_mem);
2382 if (adapter->pci_mem != NULL)
2383 bus_release_resource(dev, SYS_RES_MEMORY,
2384 PCIR_BAR(0), adapter->pci_mem);
2389 /*********************************************************************
2391 * Setup networking device structure and register an interface.
2393 **********************************************************************/
2395 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2397 struct ixgbe_hw *hw = &adapter->hw;
2400 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2402 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2404 device_printf(dev, "can not allocate ifnet structure\n");
2407 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2408 ifp->if_baudrate = 1000000000;
2409 ifp->if_init = ixgbe_init;
2410 ifp->if_softc = adapter;
2411 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2412 ifp->if_ioctl = ixgbe_ioctl;
2413 ifp->if_start = ixgbe_start;
2414 #if 0 /* __FreeBSD_version >= 800000 */
2415 ifp->if_transmit = ixgbe_mq_start;
2416 ifp->if_qflush = ixgbe_qflush;
2418 ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 2);
2420 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2422 adapter->max_frame_size =
2423 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2426 * Tell the upper layer(s) we support long frames.
2428 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2430 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2431 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2432 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2437 ifp->if_capenable = ifp->if_capabilities;
2439 /* Don't enable LRO by default */
2441 ifp->if_capabilities |= IFCAP_LRO;
2445 ** Don't turn this on by default, if vlans are
2446 ** created on another pseudo device (eg. lagg)
2447 ** then vlan events are not passed thru, breaking
2448 ** operation, but with HW FILTER off it works. If
2449 ** using vlans directly on the ixgbe driver you can
2450 ** enable this and get full hardware tag filtering.
2452 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2455 * Specify the media types supported by this adapter and register
2456 * callbacks to update media and link information
2458 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2459 ixgbe_media_status);
2460 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2461 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2462 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2463 ifmedia_add(&adapter->media,
2464 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2465 ifmedia_add(&adapter->media,
2466 IFM_ETHER | IFM_1000_T, 0, NULL);
2468 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2469 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2475 ixgbe_config_link(struct adapter *adapter)
2477 struct ixgbe_hw *hw = &adapter->hw;
2478 u32 autoneg, err = 0;
2479 bool sfp, negotiate;
2481 sfp = ixgbe_is_sfp(hw);
2484 if (hw->phy.multispeed_fiber) {
2485 hw->mac.ops.setup_sfp(hw);
2486 ixgbe_enable_tx_laser(hw);
2487 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2489 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2491 if (hw->mac.ops.check_link)
2492 err = ixgbe_check_link(hw, &autoneg,
2493 &adapter->link_up, FALSE);
2496 autoneg = hw->phy.autoneg_advertised;
2497 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2498 err = hw->mac.ops.get_link_capabilities(hw,
2499 &autoneg, &negotiate);
2502 if (hw->mac.ops.setup_link)
2503 err = hw->mac.ops.setup_link(hw, autoneg,
2504 negotiate, adapter->link_up);
2510 /********************************************************************
2511 * Manage DMA'able memory.
2512 *******************************************************************/
2514 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2518 *(bus_addr_t *) arg = segs->ds_addr;
2523 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2524 struct ixgbe_dma_alloc *dma, int mapflags)
2526 device_t dev = adapter->dev;
2529 r = bus_dma_tag_create(NULL, /* parent */
2530 DBA_ALIGN, 0, /* alignment, bounds */
2531 BUS_SPACE_MAXADDR, /* lowaddr */
2532 BUS_SPACE_MAXADDR, /* highaddr */
2533 NULL, NULL, /* filter, filterarg */
2536 size, /* maxsegsize */
2537 BUS_DMA_ALLOCNOW, /* flags */
2540 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2544 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2545 BUS_DMA_NOWAIT, &dma->dma_map);
2547 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2551 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2555 mapflags | BUS_DMA_NOWAIT);
2557 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2561 dma->dma_size = size;
2564 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2566 bus_dma_tag_destroy(dma->dma_tag);
2568 dma->dma_map = NULL;
2569 dma->dma_tag = NULL;
2574 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2576 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2577 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2578 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2579 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2580 bus_dma_tag_destroy(dma->dma_tag);
2584 /*********************************************************************
2586 * Allocate memory for the transmit and receive rings, and then
2587 * the descriptors associated with each, called only once at attach.
2589 **********************************************************************/
2591 ixgbe_allocate_queues(struct adapter *adapter)
2593 device_t dev = adapter->dev;
2594 struct ix_queue *que;
2595 struct tx_ring *txr;
2596 struct rx_ring *rxr;
2597 int rsize, tsize, error = IXGBE_SUCCESS;
2598 int txconf = 0, rxconf = 0;
2600 /* First allocate the top level queue structs */
2601 if (!(adapter->queues =
2602 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2603 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2604 device_printf(dev, "Unable to allocate queue memory\n");
2609 /* First allocate the TX ring struct memory */
2610 if (!(adapter->tx_rings =
2611 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2612 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2613 device_printf(dev, "Unable to allocate TX ring memory\n");
2618 /* Next allocate the RX */
2619 if (!(adapter->rx_rings =
2620 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2621 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2622 device_printf(dev, "Unable to allocate RX ring memory\n");
2627 /* For the ring itself */
2628 tsize = roundup2(adapter->num_tx_desc *
2629 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2632 * Now set up the TX queues, txconf is needed to handle the
2633 * possibility that things fail midcourse and we need to
2634 * undo memory gracefully
2636 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2637 /* Set up some basics */
2638 txr = &adapter->tx_rings[i];
2639 txr->adapter = adapter;
2642 /* Initialize the TX side lock */
2643 IXGBE_TX_LOCK_INIT(txr);
2645 if (ixgbe_dma_malloc(adapter, tsize,
2646 &txr->txdma, BUS_DMA_NOWAIT)) {
2648 "Unable to allocate TX Descriptor memory\n");
2652 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2653 bzero((void *)txr->tx_base, tsize);
2655 /* Now allocate transmit buffers for the ring */
2656 if (ixgbe_allocate_transmit_buffers(txr)) {
2658 "Critical Failure setting up transmit buffers\n");
2662 #if 0 /* __FreeBSD_version >= 800000 */
2663 /* Allocate a buf ring */
2664 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2665 M_WAITOK, &txr->tx_mtx);
2666 if (txr->br == NULL) {
2668 "Critical Failure setting up buf ring\n");
2676 * Next the RX queues...
2678 rsize = roundup2(adapter->num_rx_desc *
2679 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2680 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2681 rxr = &adapter->rx_rings[i];
2682 /* Set up some basics */
2683 rxr->adapter = adapter;
2686 /* Initialize the RX side lock */
2687 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2688 device_get_nameunit(dev), rxr->me);
2689 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2691 if (ixgbe_dma_malloc(adapter, rsize,
2692 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2694 "Unable to allocate RxDescriptor memory\n");
2698 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2699 bzero((void *)rxr->rx_base, rsize);
2701 /* Allocate receive buffers for the ring*/
2702 if (ixgbe_allocate_receive_buffers(rxr)) {
2704 "Critical Failure setting up receive buffers\n");
2711 ** Finally set up the queue holding structs
2713 for (int i = 0; i < adapter->num_queues; i++) {
2714 que = &adapter->queues[i];
2715 que->adapter = adapter;
2716 que->txr = &adapter->tx_rings[i];
2717 que->rxr = &adapter->rx_rings[i];
2723 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2724 ixgbe_dma_free(adapter, &rxr->rxdma);
2726 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2727 ixgbe_dma_free(adapter, &txr->txdma);
2728 kfree(adapter->rx_rings, M_DEVBUF);
2730 kfree(adapter->tx_rings, M_DEVBUF);
2732 kfree(adapter->queues, M_DEVBUF);
2737 /*********************************************************************
2739 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2740 * the information needed to transmit a packet on the wire. This is
2741 * called only once at attach, setup is done every reset.
2743 **********************************************************************/
2745 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2747 struct adapter *adapter = txr->adapter;
2748 device_t dev = adapter->dev;
2749 struct ixgbe_tx_buf *txbuf;
2753 * Setup DMA descriptor areas.
2755 if ((error = bus_dma_tag_create(
2757 1, 0, /* alignment, bounds */
2758 BUS_SPACE_MAXADDR, /* lowaddr */
2759 BUS_SPACE_MAXADDR, /* highaddr */
2760 NULL, NULL, /* filter, filterarg */
2761 IXGBE_TSO_SIZE, /* maxsize */
2762 adapter->num_segs, /* nsegments */
2763 PAGE_SIZE, /* maxsegsize */
2766 device_printf(dev,"Unable to allocate TX DMA tag\n");
2770 if (!(txr->tx_buffers =
2771 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2772 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2773 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2778 /* Create the descriptor buffer dma maps */
2779 txbuf = txr->tx_buffers;
2780 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2781 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2783 device_printf(dev, "Unable to create TX DMA map\n");
2790 /* We free all, it handles case where we are in the middle */
2791 ixgbe_free_transmit_structures(adapter);
2795 /*********************************************************************
2797 * Initialize a transmit ring.
2799 **********************************************************************/
2801 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2803 struct adapter *adapter = txr->adapter;
2804 struct ixgbe_tx_buf *txbuf;
2807 struct netmap_adapter *na = NA(adapter->ifp);
2808 struct netmap_slot *slot;
2809 #endif /* DEV_NETMAP */
2811 /* Clear the old ring contents */
2815 * (under lock): if in netmap mode, do some consistency
2816 * checks and set slot to entry 0 of the netmap ring.
2818 slot = netmap_reset(na, NR_TX, txr->me, 0);
2819 #endif /* DEV_NETMAP */
2820 bzero((void *)txr->tx_base,
2821 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2823 txr->next_avail_desc = 0;
2824 txr->next_to_clean = 0;
2826 /* Free any existing tx buffers. */
2827 txbuf = txr->tx_buffers;
2828 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2829 if (txbuf->m_head != NULL) {
2830 bus_dmamap_sync(txr->txtag, txbuf->map,
2831 BUS_DMASYNC_POSTWRITE);
2832 bus_dmamap_unload(txr->txtag, txbuf->map);
2833 m_freem(txbuf->m_head);
2834 txbuf->m_head = NULL;
2838 * In netmap mode, set the map for the packet buffer.
2839 * NOTE: Some drivers (not this one) also need to set
2840 * the physical buffer address in the NIC ring.
2841 * Slots in the netmap ring (indexed by "si") are
2842 * kring->nkr_hwofs positions "ahead" wrt the
2843 * corresponding slot in the NIC ring. In some drivers
2844 * (not here) nkr_hwofs can be negative. Function
2845 * netmap_idx_n2k() handles wraparounds properly.
2848 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2849 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2851 #endif /* DEV_NETMAP */
2852 /* Clear the EOP index */
2853 txbuf->eop_index = -1;
2857 /* Set the rate at which we sample packets */
2858 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2859 txr->atr_sample = atr_sample_rate;
2862 /* Set number of descriptors available */
2863 txr->tx_avail = adapter->num_tx_desc;
2865 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2866 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2867 IXGBE_TX_UNLOCK(txr);
2870 /*********************************************************************
2872 * Initialize all transmit rings.
2874 **********************************************************************/
2876 ixgbe_setup_transmit_structures(struct adapter *adapter)
2878 struct tx_ring *txr = adapter->tx_rings;
2880 for (int i = 0; i < adapter->num_queues; i++, txr++)
2881 ixgbe_setup_transmit_ring(txr);
2886 /*********************************************************************
2888 * Enable transmit unit.
2890 **********************************************************************/
2892 ixgbe_initialize_transmit_units(struct adapter *adapter)
2894 struct tx_ring *txr = adapter->tx_rings;
2895 struct ixgbe_hw *hw = &adapter->hw;
2897 /* Setup the Base and Length of the Tx Descriptor Ring */
2899 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2900 u64 tdba = txr->txdma.dma_paddr;
2903 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2904 (tdba & 0x00000000ffffffffULL));
2905 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2906 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2907 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2909 /* Setup the HW Tx Head and Tail descriptor pointers */
2910 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2911 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2913 /* Setup Transmit Descriptor Cmd Settings */
2914 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2915 txr->queue_status = IXGBE_QUEUE_IDLE;
2917 /* Disable Head Writeback */
2918 switch (hw->mac.type) {
2919 case ixgbe_mac_82598EB:
2920 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2922 case ixgbe_mac_82599EB:
2923 case ixgbe_mac_X540:
2925 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2928 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2929 switch (hw->mac.type) {
2930 case ixgbe_mac_82598EB:
2931 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2933 case ixgbe_mac_82599EB:
2934 case ixgbe_mac_X540:
2936 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2942 if (hw->mac.type != ixgbe_mac_82598EB) {
2943 u32 dmatxctl, rttdcs;
2944 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2945 dmatxctl |= IXGBE_DMATXCTL_TE;
2946 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2947 /* Disable arbiter to set MTQC */
2948 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2949 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2950 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2951 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2952 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2953 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2959 /*********************************************************************
2961 * Free all transmit rings.
2963 **********************************************************************/
2965 ixgbe_free_transmit_structures(struct adapter *adapter)
2967 struct tx_ring *txr = adapter->tx_rings;
2969 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2971 ixgbe_free_transmit_buffers(txr);
2972 ixgbe_dma_free(adapter, &txr->txdma);
2973 IXGBE_TX_UNLOCK(txr);
2974 IXGBE_TX_LOCK_DESTROY(txr);
2976 kfree(adapter->tx_rings, M_DEVBUF);
2979 /*********************************************************************
2981 * Free transmit ring related data structures.
2983 **********************************************************************/
2985 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2987 struct adapter *adapter = txr->adapter;
2988 struct ixgbe_tx_buf *tx_buffer;
2991 INIT_DEBUGOUT("free_transmit_ring: begin");
2993 if (txr->tx_buffers == NULL)
2996 tx_buffer = txr->tx_buffers;
2997 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2998 if (tx_buffer->m_head != NULL) {
2999 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3000 BUS_DMASYNC_POSTWRITE);
3001 bus_dmamap_unload(txr->txtag,
3003 m_freem(tx_buffer->m_head);
3004 tx_buffer->m_head = NULL;
3005 if (tx_buffer->map != NULL) {
3006 bus_dmamap_destroy(txr->txtag,
3008 tx_buffer->map = NULL;
3010 } else if (tx_buffer->map != NULL) {
3011 bus_dmamap_unload(txr->txtag,
3013 bus_dmamap_destroy(txr->txtag,
3015 tx_buffer->map = NULL;
3018 #if 0 /* __FreeBSD_version >= 800000 */
3019 if (txr->br != NULL)
3020 buf_ring_free(txr->br, M_DEVBUF);
3022 if (txr->tx_buffers != NULL) {
3023 kfree(txr->tx_buffers, M_DEVBUF);
3024 txr->tx_buffers = NULL;
3026 if (txr->txtag != NULL) {
3027 bus_dma_tag_destroy(txr->txtag);
3033 /*********************************************************************
3035 * Advanced Context Descriptor setup for VLAN or CSUM
3037 **********************************************************************/
3040 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3042 struct adapter *adapter = txr->adapter;
3043 struct ixgbe_adv_tx_context_desc *TXD;
3044 struct ixgbe_tx_buf *tx_buffer;
3045 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3046 struct ether_vlan_header *eh;
3048 struct ip6_hdr *ip6;
3049 int ehdrlen, ip_hlen = 0;
3052 bool offload = TRUE;
3053 int ctxd = txr->next_avail_desc;
3057 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3060 tx_buffer = &txr->tx_buffers[ctxd];
3061 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3064 ** In advanced descriptors the vlan tag must
3065 ** be placed into the descriptor itself.
3067 if (mp->m_flags & M_VLANTAG) {
3068 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3069 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3070 } else if (offload == FALSE)
3074 * Determine where frame payload starts.
3075 * Jump over vlan headers if already present,
3076 * helpful for QinQ too.
3078 eh = mtod(mp, struct ether_vlan_header *);
3079 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3080 etype = ntohs(eh->evl_proto);
3081 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3083 etype = ntohs(eh->evl_encap_proto);
3084 ehdrlen = ETHER_HDR_LEN;
3087 /* Set the ether header length */
3088 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3092 ip = (struct ip *)(mp->m_data + ehdrlen);
3093 ip_hlen = ip->ip_hl << 2;
3095 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3097 case ETHERTYPE_IPV6:
3098 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3099 ip_hlen = sizeof(struct ip6_hdr);
3100 /* XXX-BZ this will go badly in case of ext hdrs. */
3101 ipproto = ip6->ip6_nxt;
3102 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3109 vlan_macip_lens |= ip_hlen;
3110 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3114 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3115 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3119 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3120 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3125 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3126 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3134 /* Now copy bits into descriptor */
3135 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3136 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3137 TXD->seqnum_seed = htole32(0);
3138 TXD->mss_l4len_idx = htole32(0);
3140 tx_buffer->m_head = NULL;
3141 tx_buffer->eop_index = -1;
3143 /* We've consumed the first desc, adjust counters */
3144 if (++ctxd == adapter->num_tx_desc)
3146 txr->next_avail_desc = ctxd;
3152 /**********************************************************************
3154 * Setup work for hardware segmentation offload (TSO) on
3155 * adapters using advanced tx descriptors
3157 **********************************************************************/
3159 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3162 struct adapter *adapter = txr->adapter;
3163 struct ixgbe_adv_tx_context_desc *TXD;
3164 struct ixgbe_tx_buf *tx_buffer;
3165 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3166 u16 vtag = 0, eh_type;
3167 u32 mss_l4len_idx = 0, len;
3168 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3169 struct ether_vlan_header *eh;
3170 #if 0 /* IPv6 TSO */
3172 struct ip6_hdr *ip6;
3182 * Determine where frame payload starts.
3183 * Jump over vlan headers if already present
3185 eh = mtod(mp, struct ether_vlan_header *);
3186 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3187 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3188 eh_type = eh->evl_proto;
3190 ehdrlen = ETHER_HDR_LEN;
3191 eh_type = eh->evl_encap_proto;
3194 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3195 len = ehdrlen + sizeof(struct tcphdr);
3196 switch (ntohs(eh_type)) {
3197 #if 0 /* IPv6 TSO */
3199 case ETHERTYPE_IPV6:
3200 if (mp->m_len < len + sizeof(struct ip6_hdr))
3202 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3203 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3204 if (ip6->ip6_nxt != IPPROTO_TCP)
3206 ip_hlen = sizeof(struct ip6_hdr);
3207 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3208 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3209 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3215 if (mp->m_len < len + sizeof(struct ip))
3217 ip = (struct ip *)(mp->m_data + ehdrlen);
3218 if (ip->ip_p != IPPROTO_TCP)
3221 ip_hlen = ip->ip_hl << 2;
3222 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3223 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3224 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3225 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3226 /* Tell transmit desc to also do IPv4 checksum. */
3227 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3231 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3232 __func__, ntohs(eh_type));
3236 ctxd = txr->next_avail_desc;
3237 tx_buffer = &txr->tx_buffers[ctxd];
3238 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3240 tcp_hlen = th->th_off << 2;
3242 /* This is used in the transmit desc in encap */
3243 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3245 /* VLAN MACLEN IPLEN */
3246 if (mp->m_flags & M_VLANTAG) {
3247 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3248 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3251 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3252 vlan_macip_lens |= ip_hlen;
3253 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3255 /* ADV DTYPE TUCMD */
3256 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3257 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3258 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3261 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3262 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3263 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3265 TXD->seqnum_seed = htole32(0);
3266 tx_buffer->m_head = NULL;
3267 tx_buffer->eop_index = -1;
3269 if (++ctxd == adapter->num_tx_desc)
3273 txr->next_avail_desc = ctxd;
3279 ** This routine parses packet headers so that Flow
3280 ** Director can make a hashed filter table entry
3281 ** allowing traffic flows to be identified and kept
3282 ** on the same cpu. This would be a performance
3283 ** hit, but we only do it at IXGBE_FDIR_RATE of
3287 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3289 struct adapter *adapter = txr->adapter;
3290 struct ix_queue *que;
3294 struct ether_vlan_header *eh;
3295 union ixgbe_atr_hash_dword input = {.dword = 0};
3296 union ixgbe_atr_hash_dword common = {.dword = 0};
3297 int ehdrlen, ip_hlen;
3300 eh = mtod(mp, struct ether_vlan_header *);
3301 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3302 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3303 etype = eh->evl_proto;
3305 ehdrlen = ETHER_HDR_LEN;
3306 etype = eh->evl_encap_proto;
3309 /* Only handling IPv4 */
3310 if (etype != htons(ETHERTYPE_IP))
3313 ip = (struct ip *)(mp->m_data + ehdrlen);
3314 ip_hlen = ip->ip_hl << 2;
3316 /* check if we're UDP or TCP */
3319 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3320 /* src and dst are inverted */
3321 common.port.dst ^= th->th_sport;
3322 common.port.src ^= th->th_dport;
3323 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3326 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3327 /* src and dst are inverted */
3328 common.port.dst ^= uh->uh_sport;
3329 common.port.src ^= uh->uh_dport;
3330 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3336 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3337 if (mp->m_pkthdr.ether_vtag)
3338 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3340 common.flex_bytes ^= etype;
3341 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3343 que = &adapter->queues[txr->me];
3345 ** This assumes the Rx queue and Tx
3346 ** queue are bound to the same CPU
3348 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3349 input, common, que->msix);
3351 #endif /* IXGBE_FDIR */
3353 /**********************************************************************
3355 * Examine each tx_buffer in the used queue. If the hardware is done
3356 * processing the packet then free associated resources. The
3357 * tx_buffer is put back on the free queue.
3359 **********************************************************************/
3361 ixgbe_txeof(struct tx_ring *txr)
3363 struct adapter *adapter = txr->adapter;
3364 struct ifnet *ifp = adapter->ifp;
3365 u32 first, last, done, processed;
3366 struct ixgbe_tx_buf *tx_buffer;
3367 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3369 IXGBE_TX_LOCK_ASSERT(txr);
3372 if (ifp->if_capenable & IFCAP_NETMAP) {
3373 struct netmap_adapter *na = NA(ifp);
3374 struct netmap_kring *kring = &na->tx_rings[txr->me];
3376 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3378 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3379 BUS_DMASYNC_POSTREAD);
3381 * In netmap mode, all the work is done in the context
3382 * of the client thread. Interrupt handlers only wake up
3383 * clients, which may be sleeping on individual rings
3384 * or on a global resource for all rings.
3385 * To implement tx interrupt mitigation, we wake up the client
3386 * thread roughly every half ring, even if the NIC interrupts
3387 * more frequently. This is implemented as follows:
3388 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3389 * the slot that should wake up the thread (nkr_num_slots
3390 * means the user thread should not be woken up);
3391 * - the driver ignores tx interrupts unless netmap_mitigate=0
3392 * or the slot has the DD bit set.
3394 * When the driver has separate locks, we need to
3395 * release and re-acquire txlock to avoid deadlocks.
3396 * XXX see if we can find a better way.
3398 if (!netmap_mitigate ||
3399 (kring->nr_kflags < kring->nkr_num_slots &&
3400 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3401 kring->nr_kflags = kring->nkr_num_slots;
3402 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3403 IXGBE_TX_UNLOCK(txr);
3404 IXGBE_CORE_LOCK(adapter);
3405 selwakeuppri(&na->tx_si, PI_NET);
3406 IXGBE_CORE_UNLOCK(adapter);
3411 #endif /* DEV_NETMAP */
3413 if (txr->tx_avail == adapter->num_tx_desc) {
3414 txr->queue_status = IXGBE_QUEUE_IDLE;
3419 first = txr->next_to_clean;
3420 tx_buffer = &txr->tx_buffers[first];
3421 /* For cleanup we just use legacy struct */
3422 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3423 last = tx_buffer->eop_index;
3426 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3429 ** Get the index of the first descriptor
3430 ** BEYOND the EOP and call that 'done'.
3431 ** I do this so the comparison in the
3432 ** inner while loop below can be simple
3434 if (++last == adapter->num_tx_desc) last = 0;
3437 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3438 BUS_DMASYNC_POSTREAD);
3440 ** Only the EOP descriptor of a packet now has the DD
3441 ** bit set, this is what we look for...
3443 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3444 /* We clean the range of the packet */
3445 while (first != done) {
3446 tx_desc->upper.data = 0;
3447 tx_desc->lower.data = 0;
3448 tx_desc->buffer_addr = 0;
3452 if (tx_buffer->m_head) {
3454 tx_buffer->m_head->m_pkthdr.len;
3455 bus_dmamap_sync(txr->txtag,
3457 BUS_DMASYNC_POSTWRITE);
3458 bus_dmamap_unload(txr->txtag,
3460 m_freem(tx_buffer->m_head);
3461 tx_buffer->m_head = NULL;
3462 tx_buffer->map = NULL;
3464 tx_buffer->eop_index = -1;
3465 txr->watchdog_time = ticks;
3467 if (++first == adapter->num_tx_desc)
3470 tx_buffer = &txr->tx_buffers[first];
3472 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3476 /* See if there is more work now */
3477 last = tx_buffer->eop_index;
3480 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3481 /* Get next done point */
3482 if (++last == adapter->num_tx_desc) last = 0;
3487 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3488 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3490 txr->next_to_clean = first;
3493 ** Watchdog calculation, we know there's
3494 ** work outstanding or the first return
3495 ** would have been taken, so none processed
3496 ** for too long indicates a hang.
3498 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3499 txr->queue_status = IXGBE_QUEUE_HUNG;
3501 /* With a minimum free clear the depleted state bit. */
3502 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3503 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3505 if (txr->tx_avail == adapter->num_tx_desc) {
3506 txr->queue_status = IXGBE_QUEUE_IDLE;
3510 /*********************************************************************
3512 * Refresh mbuf buffers for RX descriptor rings
3513 * - now keeps its own state so discards due to resource
3514 * exhaustion are unnecessary, if an mbuf cannot be obtained
3515 * it just returns, keeping its placeholder, thus it can simply
3516 * be recalled to try again.
3518 **********************************************************************/
3520 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3522 struct adapter *adapter = rxr->adapter;
3523 bus_dma_segment_t hseg[1];
3524 bus_dma_segment_t pseg[1];
3525 struct ixgbe_rx_buf *rxbuf;
3526 struct mbuf *mh, *mp;
3527 int i, j, nsegs, error;
3528 bool refreshed = FALSE;
3530 i = j = rxr->next_to_refresh;
3531 /* Control the loop with one beyond */
3532 if (++j == adapter->num_rx_desc)
3535 while (j != limit) {
3536 rxbuf = &rxr->rx_buffers[i];
3537 if (rxr->hdr_split == FALSE)
3540 if (rxbuf->m_head == NULL) {
3541 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3547 mh->m_pkthdr.len = mh->m_len = MHLEN;
3549 mh->m_flags |= M_PKTHDR;
3550 /* Get the memory mapping */
3551 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3552 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3554 kprintf("Refresh mbufs: hdr dmamap load"
3555 " failure - %d\n", error);
3557 rxbuf->m_head = NULL;
3561 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3562 BUS_DMASYNC_PREREAD);
3563 rxr->rx_base[i].read.hdr_addr =
3564 htole64(hseg[0].ds_addr);
3567 if (rxbuf->m_pack == NULL) {
3568 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3569 M_PKTHDR, adapter->rx_mbuf_sz);
3575 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3576 /* Get the memory mapping */
3577 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3578 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3580 kprintf("Refresh mbufs: payload dmamap load"
3581 " failure - %d\n", error);
3583 rxbuf->m_pack = NULL;
3587 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3588 BUS_DMASYNC_PREREAD);
3589 rxr->rx_base[i].read.pkt_addr =
3590 htole64(pseg[0].ds_addr);
3593 /* Next is precalculated */
3595 rxr->next_to_refresh = i;
3596 if (++j == adapter->num_rx_desc)
3600 if (refreshed) /* Update hardware tail index */
3601 IXGBE_WRITE_REG(&adapter->hw,
3602 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3606 /*********************************************************************
3608 * Allocate memory for rx_buffer structures. Since we use one
3609 * rx_buffer per received packet, the maximum number of rx_buffer's
3610 * that we'll need is equal to the number of receive descriptors
3611 * that we've allocated.
3613 **********************************************************************/
3615 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3617 struct adapter *adapter = rxr->adapter;
3618 device_t dev = adapter->dev;
3619 struct ixgbe_rx_buf *rxbuf;
3620 int i, bsize, error;
3622 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3623 if (!(rxr->rx_buffers =
3624 (struct ixgbe_rx_buf *) kmalloc(bsize,
3625 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3626 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3631 if ((error = bus_dma_tag_create(NULL, /* parent */
3632 1, 0, /* alignment, bounds */
3633 BUS_SPACE_MAXADDR, /* lowaddr */
3634 BUS_SPACE_MAXADDR, /* highaddr */
3635 NULL, NULL, /* filter, filterarg */
3636 MSIZE, /* maxsize */
3638 MSIZE, /* maxsegsize */
3641 device_printf(dev, "Unable to create RX DMA tag\n");
3645 if ((error = bus_dma_tag_create(NULL, /* parent */
3646 1, 0, /* alignment, bounds */
3647 BUS_SPACE_MAXADDR, /* lowaddr */
3648 BUS_SPACE_MAXADDR, /* highaddr */
3649 NULL, NULL, /* filter, filterarg */
3650 MJUM16BYTES, /* maxsize */
3652 MJUM16BYTES, /* maxsegsize */
3655 device_printf(dev, "Unable to create RX DMA tag\n");
3659 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3660 rxbuf = &rxr->rx_buffers[i];
3661 error = bus_dmamap_create(rxr->htag,
3662 BUS_DMA_NOWAIT, &rxbuf->hmap);
3664 device_printf(dev, "Unable to create RX head map\n");
3667 error = bus_dmamap_create(rxr->ptag,
3668 BUS_DMA_NOWAIT, &rxbuf->pmap);
3670 device_printf(dev, "Unable to create RX pkt map\n");
3678 /* Frees all, but can handle partial completion */
3679 ixgbe_free_receive_structures(adapter);
3684 ** Used to detect a descriptor that has
3685 ** been merged by Hardware RSC.
3688 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3690 return (le32toh(rx->wb.lower.lo_dword.data) &
3691 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3694 /*********************************************************************
3696 * Initialize Hardware RSC (LRO) feature on 82599
3697 * for an RX ring, this is toggled by the LRO capability
3698 * even though it is transparent to the stack.
3700 **********************************************************************/
3703 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3705 struct adapter *adapter = rxr->adapter;
3706 struct ixgbe_hw *hw = &adapter->hw;
3707 u32 rscctrl, rdrxctl;
3709 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3710 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3711 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3712 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3713 #endif /* DEV_NETMAP */
3714 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3715 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3716 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3718 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3719 rscctrl |= IXGBE_RSCCTL_RSCEN;
3721 ** Limit the total number of descriptors that
3722 ** can be combined, so it does not exceed 64K
3724 if (adapter->rx_mbuf_sz == MCLBYTES)
3725 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3726 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3727 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3728 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3729 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3730 else /* Using 16K cluster */
3731 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3733 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3735 /* Enable TCP header recognition */
3736 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3737 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3738 IXGBE_PSRTYPE_TCPHDR));
3740 /* Disable RSC for ACK packets */
3741 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3742 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3749 ixgbe_free_receive_ring(struct rx_ring *rxr)
3751 struct adapter *adapter;
3752 struct ixgbe_rx_buf *rxbuf;
3755 adapter = rxr->adapter;
3756 for (i = 0; i < adapter->num_rx_desc; i++) {
3757 rxbuf = &rxr->rx_buffers[i];
3758 if (rxbuf->m_head != NULL) {
3759 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3760 BUS_DMASYNC_POSTREAD);
3761 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3762 rxbuf->m_head->m_flags |= M_PKTHDR;
3763 m_freem(rxbuf->m_head);
3765 if (rxbuf->m_pack != NULL) {
3766 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3767 BUS_DMASYNC_POSTREAD);
3768 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3769 rxbuf->m_pack->m_flags |= M_PKTHDR;
3770 m_freem(rxbuf->m_pack);
3772 rxbuf->m_head = NULL;
3773 rxbuf->m_pack = NULL;
3778 /*********************************************************************
3780 * Initialize a receive ring and its buffers.
3782 **********************************************************************/
3784 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3786 struct adapter *adapter;
3789 struct ixgbe_rx_buf *rxbuf;
3790 bus_dma_segment_t pseg[1], hseg[1];
3792 struct lro_ctrl *lro = &rxr->lro;
3794 int rsize, nsegs, error = 0;
3796 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3797 struct netmap_slot *slot;
3798 #endif /* DEV_NETMAP */
3800 adapter = rxr->adapter;
3804 /* Clear the ring contents */
3807 /* same as in ixgbe_setup_transmit_ring() */
3808 slot = netmap_reset(na, NR_RX, rxr->me, 0);
3809 #endif /* DEV_NETMAP */
3810 rsize = roundup2(adapter->num_rx_desc *
3811 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3812 bzero((void *)rxr->rx_base, rsize);
3814 /* Free current RX buffer structs and their mbufs */
3815 ixgbe_free_receive_ring(rxr);
3817 /* Configure header split? */
3818 if (ixgbe_header_split)
3819 rxr->hdr_split = TRUE;
3821 /* Now replenish the mbufs */
3822 for (int j = 0; j != adapter->num_rx_desc; ++j) {
3823 struct mbuf *mh, *mp;
3825 rxbuf = &rxr->rx_buffers[j];
3828 * In netmap mode, fill the map and set the buffer
3829 * address in the NIC ring, considering the offset
3830 * between the netmap and NIC rings (see comment in
3831 * ixgbe_setup_transmit_ring() ). No need to allocate
3832 * an mbuf, so end the block with a continue;
3835 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3839 addr = PNMB(slot + sj, &paddr);
3840 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3841 /* Update descriptor */
3842 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3845 #endif /* DEV_NETMAP */
3847 ** Don't allocate mbufs if not
3848 ** doing header split, its wasteful
3850 if (rxr->hdr_split == FALSE)
3853 /* First the header */
3854 rxbuf->m_head = m_gethdr(MB_DONTWAIT, MT_DATA);
3855 if (rxbuf->m_head == NULL) {
3859 m_adj(rxbuf->m_head, ETHER_ALIGN);
3861 mh->m_len = mh->m_pkthdr.len = MHLEN;
3862 mh->m_flags |= M_PKTHDR;
3863 /* Get the memory mapping */
3864 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3865 rxbuf->hmap, rxbuf->m_head, hseg, 1,
3866 &nsegs, BUS_DMA_NOWAIT);
3868 if (error != 0) /* Nothing elegant to do here */
3870 bus_dmamap_sync(rxr->htag,
3871 rxbuf->hmap, BUS_DMASYNC_PREREAD);
3872 /* Update descriptor */
3873 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3876 /* Now the payload cluster */
3877 rxbuf->m_pack = m_getjcl(MB_DONTWAIT, MT_DATA,
3878 M_PKTHDR, adapter->rx_mbuf_sz);
3879 if (rxbuf->m_pack == NULL) {
3884 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3885 /* Get the memory mapping */
3886 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3887 rxbuf->pmap, mp, hseg, 1,
3888 &nsegs, BUS_DMA_NOWAIT);
3891 bus_dmamap_sync(rxr->ptag,
3892 rxbuf->pmap, BUS_DMASYNC_PREREAD);
3893 /* Update descriptor */
3894 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3898 /* Setup our descriptor indices */
3899 rxr->next_to_check = 0;
3900 rxr->next_to_refresh = 0;
3901 rxr->lro_enabled = FALSE;
3902 rxr->rx_split_packets = 0;
3904 rxr->discard = FALSE;
3905 rxr->vtag_strip = FALSE;
3907 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3908 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3911 ** Now set up the LRO interface:
3912 ** 82598 uses software LRO, the
3913 ** 82599 and X540 use a hardware assist.
3916 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3917 (ifp->if_capenable & IFCAP_RXCSUM) &&
3918 (ifp->if_capenable & IFCAP_LRO))
3919 ixgbe_setup_hw_rsc(rxr);
3920 else if (ifp->if_capenable & IFCAP_LRO) {
3921 int err = tcp_lro_init(lro);
3923 device_printf(dev, "LRO Initialization failed!\n");
3926 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3927 rxr->lro_enabled = TRUE;
3928 lro->ifp = adapter->ifp;
3932 IXGBE_RX_UNLOCK(rxr);
3936 ixgbe_free_receive_ring(rxr);
3937 IXGBE_RX_UNLOCK(rxr);
3941 /*********************************************************************
3943 * Initialize all receive rings.
3945 **********************************************************************/
3947 ixgbe_setup_receive_structures(struct adapter *adapter)
3949 struct rx_ring *rxr = adapter->rx_rings;
3952 for (j = 0; j < adapter->num_queues; j++, rxr++)
3953 if (ixgbe_setup_receive_ring(rxr))
3959 * Free RX buffers allocated so far, we will only handle
3960 * the rings that completed, the failing case will have
3961 * cleaned up for itself. 'j' failed, so its the terminus.
3963 for (int i = 0; i < j; ++i) {
3964 rxr = &adapter->rx_rings[i];
3965 ixgbe_free_receive_ring(rxr);
3971 /*********************************************************************
3973 * Setup receive registers and features.
3975 **********************************************************************/
3976 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3978 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3981 ixgbe_initialize_receive_units(struct adapter *adapter)
3983 struct rx_ring *rxr = adapter->rx_rings;
3984 struct ixgbe_hw *hw = &adapter->hw;
3985 struct ifnet *ifp = adapter->ifp;
3986 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
3987 u32 reta, mrqc = 0, hlreg, random[10];
3991 * Make sure receives are disabled while
3992 * setting up the descriptor ring
3994 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3995 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
3996 rxctrl & ~IXGBE_RXCTRL_RXEN);
3998 /* Enable broadcasts */
3999 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4000 fctrl |= IXGBE_FCTRL_BAM;
4001 fctrl |= IXGBE_FCTRL_DPF;
4002 fctrl |= IXGBE_FCTRL_PMCF;
4003 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4005 /* Set for Jumbo Frames? */
4006 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4007 if (ifp->if_mtu > ETHERMTU)
4008 hlreg |= IXGBE_HLREG0_JUMBOEN;
4010 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4012 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4013 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4014 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4016 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4017 #endif /* DEV_NETMAP */
4018 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4020 bufsz = (adapter->rx_mbuf_sz +
4021 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4023 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4024 u64 rdba = rxr->rxdma.dma_paddr;
4026 /* Setup the Base and Length of the Rx Descriptor Ring */
4027 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4028 (rdba & 0x00000000ffffffffULL));
4029 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4030 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4031 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4033 /* Set up the SRRCTL register */
4034 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4035 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4036 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4038 if (rxr->hdr_split) {
4039 /* Use a standard mbuf for the header */
4040 srrctl |= ((IXGBE_RX_HDR <<
4041 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4042 & IXGBE_SRRCTL_BSIZEHDR_MASK);
4043 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4045 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4046 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4048 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4049 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4050 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4053 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4054 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4055 IXGBE_PSRTYPE_UDPHDR |
4056 IXGBE_PSRTYPE_IPV4HDR |
4057 IXGBE_PSRTYPE_IPV6HDR;
4058 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4061 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4064 if (adapter->num_queues > 1) {
4068 /* set up random bits */
4069 karc4rand(&random, sizeof(random));
4071 /* Set up the redirection table */
4072 for (i = 0, j = 0; i < 128; i++, j++) {
4073 if (j == adapter->num_queues) j = 0;
4074 reta = (reta << 8) | (j * 0x11);
4076 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4079 /* Now fill our hash function seeds */
4080 for (int i = 0; i < 10; i++)
4081 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4083 /* Perform hash on these packet types */
4084 mrqc = IXGBE_MRQC_RSSEN
4085 | IXGBE_MRQC_RSS_FIELD_IPV4
4086 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4087 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4088 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4089 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4090 | IXGBE_MRQC_RSS_FIELD_IPV6
4091 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4092 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4093 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4094 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4096 /* RSS and RX IPP Checksum are mutually exclusive */
4097 rxcsum |= IXGBE_RXCSUM_PCSD;
4100 if (ifp->if_capenable & IFCAP_RXCSUM)
4101 rxcsum |= IXGBE_RXCSUM_PCSD;
4103 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4104 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4106 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4111 /*********************************************************************
4113 * Free all receive rings.
4115 **********************************************************************/
4117 ixgbe_free_receive_structures(struct adapter *adapter)
4119 struct rx_ring *rxr = adapter->rx_rings;
4121 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4123 struct lro_ctrl *lro = &rxr->lro;
4125 ixgbe_free_receive_buffers(rxr);
4126 /* Free LRO memory */
4130 /* Free the ring memory as well */
4131 ixgbe_dma_free(adapter, &rxr->rxdma);
4134 kfree(adapter->rx_rings, M_DEVBUF);
4138 /*********************************************************************
4140 * Free receive ring data structures
4142 **********************************************************************/
4144 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4146 struct adapter *adapter = rxr->adapter;
4147 struct ixgbe_rx_buf *rxbuf;
4149 INIT_DEBUGOUT("free_receive_structures: begin");
4151 /* Cleanup any existing buffers */
4152 if (rxr->rx_buffers != NULL) {
4153 for (int i = 0; i < adapter->num_rx_desc; i++) {
4154 rxbuf = &rxr->rx_buffers[i];
4155 if (rxbuf->m_head != NULL) {
4156 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4157 BUS_DMASYNC_POSTREAD);
4158 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4159 rxbuf->m_head->m_flags |= M_PKTHDR;
4160 m_freem(rxbuf->m_head);
4162 if (rxbuf->m_pack != NULL) {
4163 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4164 BUS_DMASYNC_POSTREAD);
4165 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4166 rxbuf->m_pack->m_flags |= M_PKTHDR;
4167 m_freem(rxbuf->m_pack);
4169 rxbuf->m_head = NULL;
4170 rxbuf->m_pack = NULL;
4171 if (rxbuf->hmap != NULL) {
4172 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4175 if (rxbuf->pmap != NULL) {
4176 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4180 if (rxr->rx_buffers != NULL) {
4181 kfree(rxr->rx_buffers, M_DEVBUF);
4182 rxr->rx_buffers = NULL;
4186 if (rxr->htag != NULL) {
4187 bus_dma_tag_destroy(rxr->htag);
4190 if (rxr->ptag != NULL) {
4191 bus_dma_tag_destroy(rxr->ptag);
4198 static __inline void
4199 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4203 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4204 * should be computed by hardware. Also it should not have VLAN tag in
4205 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4208 if (rxr->lro_enabled &&
4209 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4210 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4211 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4212 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4213 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4214 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4215 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4216 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4218 * Send to the stack if:
4219 ** - LRO not enabled, or
4220 ** - no LRO resources, or
4221 ** - lro enqueue fails
4223 if (rxr->lro.lro_cnt != 0)
4224 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4228 IXGBE_RX_UNLOCK(rxr);
4229 (*ifp->if_input)(ifp, m);
4233 static __inline void
4234 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4236 struct ixgbe_rx_buf *rbuf;
4238 rbuf = &rxr->rx_buffers[i];
4240 if (rbuf->fmp != NULL) {/* Partial chain ? */
4241 rbuf->fmp->m_flags |= M_PKTHDR;
4247 ** With advanced descriptors the writeback
4248 ** clobbers the buffer addrs, so its easier
4249 ** to just free the existing mbufs and take
4250 ** the normal refresh path to get new buffers
4254 m_free(rbuf->m_head);
4255 rbuf->m_head = NULL;
4259 m_free(rbuf->m_pack);
4260 rbuf->m_pack = NULL;
4267 ixgbe_add_sysctl(struct adapter *adapter)
4269 sysctl_ctx_init(&adapter->sysctl_ctx);
4270 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4271 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4272 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4273 if (adapter->sysctl_tree == NULL) {
4274 device_printf(adapter->dev, "can't add sysctl node\n");
4277 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4278 SYSCTL_CHILDREN(adapter->sysctl_tree),
4279 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4280 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4282 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4283 SYSCTL_CHILDREN(adapter->sysctl_tree),
4284 OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4285 adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4288 ** Allow a kind of speed control by forcing the autoneg
4289 ** advertised speed list to only a certain value, this
4290 ** supports 1G on 82599 devices, and 100Mb on x540.
4292 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4293 SYSCTL_CHILDREN(adapter->sysctl_tree),
4294 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4295 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4297 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4298 SYSCTL_CHILDREN(adapter->sysctl_tree),
4299 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4300 0, ixgbe_set_thermal_test, "I", "Thermal Test");
4302 /* Sysctl for limiting the amount of work done in the taskqueue */
4303 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4304 "max number of rx packets to process", &adapter->rx_process_limit,
4305 ixgbe_rx_process_limit);
4308 /*********************************************************************
4310 * This routine executes in interrupt context. It replenishes
4311 * the mbufs in the descriptor and sends data which has been
4312 * dma'ed into host memory to upper layer.
4314 * We loop at most count times if count is > 0, or until done if
4317 * Return TRUE for more work, FALSE for all clean.
4318 *********************************************************************/
4320 ixgbe_rxeof(struct ix_queue *que, int count)
4322 struct adapter *adapter = que->adapter;
4323 struct rx_ring *rxr = que->rxr;
4324 struct ifnet *ifp = adapter->ifp;
4326 struct lro_ctrl *lro = &rxr->lro;
4327 struct lro_entry *queued;
4329 int i, nextp, processed = 0;
4331 union ixgbe_adv_rx_desc *cur;
4332 struct ixgbe_rx_buf *rbuf, *nbuf;
4337 if (ifp->if_capenable & IFCAP_NETMAP) {
4339 * Same as the txeof routine: only wakeup clients on intr.
4340 * NKR_PENDINTR in nr_kflags is used to implement interrupt
4341 * mitigation (ixgbe_rxsync() will not look for new packets
4342 * unless NKR_PENDINTR is set).
4344 struct netmap_adapter *na = NA(ifp);
4346 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4347 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4348 IXGBE_RX_UNLOCK(rxr);
4349 IXGBE_CORE_LOCK(adapter);
4350 selwakeuppri(&na->rx_si, PI_NET);
4351 IXGBE_CORE_UNLOCK(adapter);
4354 #endif /* DEV_NETMAP */
4355 for (i = rxr->next_to_check; count != 0;) {
4356 struct mbuf *sendmp, *mh, *mp;
4358 u16 hlen, plen, hdr;
4362 /* Sync the ring. */
4363 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4364 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4366 cur = &rxr->rx_base[i];
4367 staterr = le32toh(cur->wb.upper.status_error);
4369 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4371 if ((ifp->if_flags & IFF_RUNNING) == 0)
4378 cur->wb.upper.status_error = 0;
4379 rbuf = &rxr->rx_buffers[i];
4383 plen = le16toh(cur->wb.upper.length);
4384 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4385 IXGBE_RXDADV_PKTTYPE_MASK;
4386 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4387 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4389 /* Process vlan info */
4390 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4391 vtag = le16toh(cur->wb.upper.vlan);
4393 /* Make sure bad packets are discarded */
4394 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4397 rxr->rx_discarded++;
4399 rxr->discard = FALSE;
4401 rxr->discard = TRUE;
4402 ixgbe_rx_discard(rxr, i);
4407 ** On 82599 which supports a hardware
4408 ** LRO (called HW RSC), packets need
4409 ** not be fragmented across sequential
4410 ** descriptors, rather the next descriptor
4411 ** is indicated in bits of the descriptor.
4412 ** This also means that we might proceses
4413 ** more than one packet at a time, something
4414 ** that has never been true before, it
4415 ** required eliminating global chain pointers
4416 ** in favor of what we are doing here. -jfv
4420 ** Figure out the next descriptor
4423 if (rxr->hw_rsc == TRUE) {
4424 rsc = ixgbe_rsc_count(cur);
4425 rxr->rsc_num += (rsc - 1);
4427 if (rsc) { /* Get hardware index */
4429 IXGBE_RXDADV_NEXTP_MASK) >>
4430 IXGBE_RXDADV_NEXTP_SHIFT);
4431 } else { /* Just sequential */
4433 if (nextp == adapter->num_rx_desc)
4436 nbuf = &rxr->rx_buffers[nextp];
4440 ** The header mbuf is ONLY used when header
4441 ** split is enabled, otherwise we get normal
4442 ** behavior, ie, both header and payload
4443 ** are DMA'd into the payload buffer.
4445 ** Rather than using the fmp/lmp global pointers
4446 ** we now keep the head of a packet chain in the
4447 ** buffer struct and pass this along from one
4448 ** descriptor to the next, until we get EOP.
4450 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4451 /* This must be an initial descriptor */
4452 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4453 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4454 if (hlen > IXGBE_RX_HDR)
4455 hlen = IXGBE_RX_HDR;
4457 mh->m_flags |= M_PKTHDR;
4459 mh->m_pkthdr.len = mh->m_len;
4460 /* Null buf pointer so it is refreshed */
4461 rbuf->m_head = NULL;
4463 ** Check the payload length, this
4464 ** could be zero if its a small
4470 mp->m_flags &= ~M_PKTHDR;
4472 mh->m_pkthdr.len += mp->m_len;
4473 /* Null buf pointer so it is refreshed */
4474 rbuf->m_pack = NULL;
4475 rxr->rx_split_packets++;
4478 ** Now create the forward
4479 ** chain so when complete
4483 /* stash the chain head */
4485 /* Make forward chain */
4487 mp->m_next = nbuf->m_pack;
4489 mh->m_next = nbuf->m_pack;
4491 /* Singlet, prepare to send */
4493 /* If hardware handled vtag */
4495 sendmp->m_pkthdr.ether_vlantag = vtag;
4496 sendmp->m_flags |= M_VLANTAG;
4501 ** Either no header split, or a
4502 ** secondary piece of a fragmented
4507 ** See if there is a stored head
4508 ** that determines what we are
4511 rbuf->m_pack = rbuf->fmp = NULL;
4513 if (sendmp != NULL) { /* secondary frag */
4514 mp->m_flags &= ~M_PKTHDR;
4515 sendmp->m_pkthdr.len += mp->m_len;
4517 /* first desc of a non-ps chain */
4519 sendmp->m_flags |= M_PKTHDR;
4520 sendmp->m_pkthdr.len = mp->m_len;
4521 if (staterr & IXGBE_RXD_STAT_VP) {
4522 sendmp->m_pkthdr.ether_vlantag = vtag;
4523 sendmp->m_flags |= M_VLANTAG;
4526 /* Pass the head pointer on */
4530 mp->m_next = nbuf->m_pack;
4534 /* Sending this frame? */
4536 sendmp->m_pkthdr.rcvif = ifp;
4539 /* capture data for AIM */
4540 rxr->bytes += sendmp->m_pkthdr.len;
4541 rxr->rx_bytes += sendmp->m_pkthdr.len;
4542 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4543 ixgbe_rx_checksum(staterr, sendmp, ptype);
4544 #if 0 /* __FreeBSD_version >= 800000 */
4545 sendmp->m_pkthdr.flowid = que->msix;
4546 sendmp->m_flags |= M_FLOWID;
4550 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4551 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4553 /* Advance our pointers to the next descriptor. */
4554 if (++i == adapter->num_rx_desc)
4557 /* Now send to the stack or do LRO */
4558 if (sendmp != NULL) {
4559 rxr->next_to_check = i;
4560 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4561 i = rxr->next_to_check;
4564 /* Every 8 descriptors we go to refresh mbufs */
4565 if (processed == 8) {
4566 ixgbe_refresh_mbufs(rxr, i);
4571 /* Refresh any remaining buf structs */
4572 if (ixgbe_rx_unrefreshed(rxr))
4573 ixgbe_refresh_mbufs(rxr, i);
4575 rxr->next_to_check = i;
4578 * Flush any outstanding LRO work
4581 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4582 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4583 tcp_lro_flush(lro, queued);
4587 IXGBE_RX_UNLOCK(rxr);
4590 ** We still have cleaning to do?
4591 ** Schedule another interrupt if so.
4593 if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4594 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4599 /*********************************************************************
4601 * Verify that the hardware indicated that the checksum is valid.
4602 * Inform the stack about the status of checksum so that stack
4603 * doesn't spend time verifying the checksum.
4605 *********************************************************************/
4607 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4609 u16 status = (u16) staterr;
4610 u8 errors = (u8) (staterr >> 24);
4613 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4614 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4617 if (status & IXGBE_RXD_STAT_IPCS) {
4618 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4619 /* IP Checksum Good */
4620 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4621 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4624 mp->m_pkthdr.csum_flags = 0;
4626 if (status & IXGBE_RXD_STAT_L4CS) {
4627 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4630 type = CSUM_SCTP_VALID;
4632 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4633 mp->m_pkthdr.csum_flags |= type;
4635 mp->m_pkthdr.csum_data = htons(0xffff);
4643 ** This routine is run via an vlan config EVENT,
4644 ** it enables us to use the HW Filter table since
4645 ** we can get the vlan id. This just creates the
4646 ** entry in the soft version of the VFTA, init will
4647 ** repopulate the real table.
4650 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4652 struct adapter *adapter = ifp->if_softc;
4655 if (ifp->if_softc != arg) /* Not our event */
4658 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4661 IXGBE_CORE_LOCK(adapter);
4662 index = (vtag >> 5) & 0x7F;
4664 adapter->shadow_vfta[index] |= (1 << bit);
4665 ++adapter->num_vlans;
4666 ixgbe_init_locked(adapter);
4667 IXGBE_CORE_UNLOCK(adapter);
4671 ** This routine is run via an vlan
4672 ** unconfig EVENT, remove our entry
4673 ** in the soft vfta.
4676 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4678 struct adapter *adapter = ifp->if_softc;
4681 if (ifp->if_softc != arg)
4684 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4687 IXGBE_CORE_LOCK(adapter);
4688 index = (vtag >> 5) & 0x7F;
4690 adapter->shadow_vfta[index] &= ~(1 << bit);
4691 --adapter->num_vlans;
4692 /* Re-init to load the changes */
4693 ixgbe_init_locked(adapter);
4694 IXGBE_CORE_UNLOCK(adapter);
4698 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4700 struct ifnet *ifp = adapter->ifp;
4701 struct ixgbe_hw *hw = &adapter->hw;
4702 struct rx_ring *rxr;
4706 ** We get here thru init_locked, meaning
4707 ** a soft reset, this has already cleared
4708 ** the VFTA and other state, so if there
4709 ** have been no vlan's registered do nothing.
4711 if (adapter->num_vlans == 0)
4715 ** A soft reset zero's out the VFTA, so
4716 ** we need to repopulate it now.
4718 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4719 if (adapter->shadow_vfta[i] != 0)
4720 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4721 adapter->shadow_vfta[i]);
4723 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4724 /* Enable the Filter Table if enabled */
4725 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4726 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4727 ctrl |= IXGBE_VLNCTRL_VFE;
4729 if (hw->mac.type == ixgbe_mac_82598EB)
4730 ctrl |= IXGBE_VLNCTRL_VME;
4731 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4733 /* Setup the queues for vlans */
4734 for (int i = 0; i < adapter->num_queues; i++) {
4735 rxr = &adapter->rx_rings[i];
4736 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4737 if (hw->mac.type != ixgbe_mac_82598EB) {
4738 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4739 ctrl |= IXGBE_RXDCTL_VME;
4740 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4742 rxr->vtag_strip = TRUE;
4747 ixgbe_enable_intr(struct adapter *adapter)
4749 struct ixgbe_hw *hw = &adapter->hw;
4750 struct ix_queue *que = adapter->queues;
4751 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4754 /* Enable Fan Failure detection */
4755 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4756 mask |= IXGBE_EIMS_GPI_SDP1;
4758 mask |= IXGBE_EIMS_ECC;
4759 mask |= IXGBE_EIMS_GPI_SDP0;
4760 mask |= IXGBE_EIMS_GPI_SDP1;
4761 mask |= IXGBE_EIMS_GPI_SDP2;
4763 mask |= IXGBE_EIMS_FLOW_DIR;
4767 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4769 /* With RSS we use auto clear */
4770 if (adapter->msix_mem) {
4771 mask = IXGBE_EIMS_ENABLE_MASK;
4772 /* Don't autoclear Link */
4773 mask &= ~IXGBE_EIMS_OTHER;
4774 mask &= ~IXGBE_EIMS_LSC;
4775 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4779 ** Now enable all queues, this is done separately to
4780 ** allow for handling the extended (beyond 32) MSIX
4781 ** vectors that can be used by 82599
4783 for (int i = 0; i < adapter->num_queues; i++, que++)
4784 ixgbe_enable_queue(adapter, que->msix);
4786 IXGBE_WRITE_FLUSH(hw);
4792 ixgbe_disable_intr(struct adapter *adapter)
4794 if (adapter->msix_mem)
4795 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4796 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4797 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4799 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4800 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4801 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4803 IXGBE_WRITE_FLUSH(&adapter->hw);
4808 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4812 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4819 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4821 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4828 ** Setup the correct IVAR register for a particular MSIX interrupt
4829 ** (yes this is all very magic and confusing :)
4830 ** - entry is the register array entry
4831 ** - vector is the MSIX vector for this queue
4832 ** - type is RX/TX/MISC
4835 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4837 struct ixgbe_hw *hw = &adapter->hw;
4840 vector |= IXGBE_IVAR_ALLOC_VAL;
4842 switch (hw->mac.type) {
4844 case ixgbe_mac_82598EB:
4846 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4848 entry += (type * 64);
4849 index = (entry >> 2) & 0x1F;
4850 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4851 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4852 ivar |= (vector << (8 * (entry & 0x3)));
4853 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4856 case ixgbe_mac_82599EB:
4857 case ixgbe_mac_X540:
4858 if (type == -1) { /* MISC IVAR */
4859 index = (entry & 1) * 8;
4860 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4861 ivar &= ~(0xFF << index);
4862 ivar |= (vector << index);
4863 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4864 } else { /* RX/TX IVARS */
4865 index = (16 * (entry & 1)) + (8 * type);
4866 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4867 ivar &= ~(0xFF << index);
4868 ivar |= (vector << index);
4869 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4878 ixgbe_configure_ivars(struct adapter *adapter)
4880 struct ix_queue *que = adapter->queues;
4882 for (int i = 0; i < adapter->num_queues; i++, que++) {
4883 /* First the RX queue entry */
4884 ixgbe_set_ivar(adapter, i, que->msix, 0);
4885 /* ... and the TX */
4886 ixgbe_set_ivar(adapter, i, que->msix, 1);
4887 /* Set an Initial EITR value */
4888 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4891 /* For the Link interrupt */
4892 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4896 ** ixgbe_sfp_probe - called in the local timer to
4897 ** determine if a port had optics inserted.
4899 static bool ixgbe_sfp_probe(struct adapter *adapter)
4901 struct ixgbe_hw *hw = &adapter->hw;
4902 device_t dev = adapter->dev;
4903 bool result = FALSE;
4905 if ((hw->phy.type == ixgbe_phy_nl) &&
4906 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4907 s32 ret = hw->phy.ops.identify_sfp(hw);
4910 ret = hw->phy.ops.reset(hw);
4911 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4912 device_printf(dev,"Unsupported SFP+ module detected!");
4913 kprintf(" Reload driver with supported module.\n");
4914 adapter->sfp_probe = FALSE;
4917 device_printf(dev,"SFP+ module detected!\n");
4918 /* We now have supported optics */
4919 adapter->sfp_probe = FALSE;
4920 /* Set the optics type so system reports correctly */
4921 ixgbe_setup_optics(adapter);
4929 ** Tasklet for handling SFP module interrupts
4932 ixgbe_handle_mod(void *context, int pending)
4934 struct adapter *adapter = context;
4935 struct ixgbe_hw *hw = &adapter->hw;
4936 device_t dev = adapter->dev;
4939 err = hw->phy.ops.identify_sfp(hw);
4940 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4942 "Unsupported SFP+ module type was detected.\n");
4945 err = hw->mac.ops.setup_sfp(hw);
4946 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4948 "Setup failure - unsupported SFP+ module type.\n");
4951 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4957 ** Tasklet for handling MSF (multispeed fiber) interrupts
4960 ixgbe_handle_msf(void *context, int pending)
4962 struct adapter *adapter = context;
4963 struct ixgbe_hw *hw = &adapter->hw;
4967 autoneg = hw->phy.autoneg_advertised;
4968 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4969 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4970 if (hw->mac.ops.setup_link)
4971 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4977 ** Tasklet for reinitializing the Flow Director filter table
4980 ixgbe_reinit_fdir(void *context, int pending)
4982 struct adapter *adapter = context;
4983 struct ifnet *ifp = adapter->ifp;
4985 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4987 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4988 adapter->fdir_reinit = 0;
4989 /* re-enable flow director interrupts */
4990 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4991 /* Restart the interface */
4992 ifp->if_drv_flags |= IFF_DRV_RUNNING;
4997 /**********************************************************************
4999 * Update the board statistics counters.
5001 **********************************************************************/
5003 ixgbe_update_stats_counters(struct adapter *adapter)
5005 struct ifnet *ifp = adapter->ifp;
5006 struct ixgbe_hw *hw = &adapter->hw;
5007 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5008 u64 total_missed_rx = 0;
5010 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5011 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5012 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5013 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5015 for (int i = 0; i < 8; i++) {
5017 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5018 /* missed_rx tallies misses for the gprc workaround */
5020 /* global total per queue */
5021 adapter->stats.mpc[i] += mp;
5022 /* Running comprehensive total for stats display */
5023 total_missed_rx += adapter->stats.mpc[i];
5024 if (hw->mac.type == ixgbe_mac_82598EB)
5025 adapter->stats.rnbc[i] +=
5026 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5027 adapter->stats.pxontxc[i] +=
5028 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5029 adapter->stats.pxonrxc[i] +=
5030 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5031 adapter->stats.pxofftxc[i] +=
5032 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5033 adapter->stats.pxoffrxc[i] +=
5034 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5035 adapter->stats.pxon2offc[i] +=
5036 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5038 for (int i = 0; i < 16; i++) {
5039 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5040 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5041 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5042 adapter->stats.qbrc[i] +=
5043 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5044 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5045 adapter->stats.qbtc[i] +=
5046 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5047 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5049 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5050 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5051 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5053 /* Hardware workaround, gprc counts missed packets */
5054 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5055 adapter->stats.gprc -= missed_rx;
5057 if (hw->mac.type != ixgbe_mac_82598EB) {
5058 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5059 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5060 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5061 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5062 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5063 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5064 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5065 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5067 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5068 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5069 /* 82598 only has a counter in the high register */
5070 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5071 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5072 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5076 * Workaround: mprc hardware is incorrectly counting
5077 * broadcasts, so for now we subtract those.
5079 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5080 adapter->stats.bprc += bprc;
5081 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5082 if (hw->mac.type == ixgbe_mac_82598EB)
5083 adapter->stats.mprc -= bprc;
5085 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5086 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5087 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5088 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5089 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5090 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5092 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5093 adapter->stats.lxontxc += lxon;
5094 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5095 adapter->stats.lxofftxc += lxoff;
5096 total = lxon + lxoff;
5098 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5099 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5100 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5101 adapter->stats.gptc -= total;
5102 adapter->stats.mptc -= total;
5103 adapter->stats.ptc64 -= total;
5104 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5106 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5107 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5108 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5109 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5110 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5111 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5112 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5113 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5114 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5115 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5116 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5117 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5118 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5119 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5120 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5121 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5122 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5123 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5124 /* Only read FCOE on 82599 */
5125 if (hw->mac.type != ixgbe_mac_82598EB) {
5126 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5127 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5128 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5129 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5130 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5133 /* Fill out the OS statistics structure */
5134 ifp->if_ipackets = adapter->stats.gprc;
5135 ifp->if_opackets = adapter->stats.gptc;
5136 ifp->if_ibytes = adapter->stats.gorc;
5137 ifp->if_obytes = adapter->stats.gotc;
5138 ifp->if_imcasts = adapter->stats.mprc;
5139 ifp->if_collisions = 0;
5142 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5143 adapter->stats.rlec;
5146 /** ixgbe_sysctl_tdh_handler - Handler function
5147 * Retrieves the TDH value from the hardware
5150 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5154 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5157 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5158 error = sysctl_handle_int(oidp, &val, 0, req);
5159 if (error || !req->newptr)
5164 /** ixgbe_sysctl_tdt_handler - Handler function
5165 * Retrieves the TDT value from the hardware
5168 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5172 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5175 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5176 error = sysctl_handle_int(oidp, &val, 0, req);
5177 if (error || !req->newptr)
5182 /** ixgbe_sysctl_rdh_handler - Handler function
5183 * Retrieves the RDH value from the hardware
5186 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5190 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5193 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5194 error = sysctl_handle_int(oidp, &val, 0, req);
5195 if (error || !req->newptr)
5200 /** ixgbe_sysctl_rdt_handler - Handler function
5201 * Retrieves the RDT value from the hardware
5204 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5208 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5211 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5212 error = sysctl_handle_int(oidp, &val, 0, req);
5213 if (error || !req->newptr)
5219 * Add sysctl variables, one per statistic, to the system.
5222 ixgbe_add_hw_stats(struct adapter *adapter)
5224 struct tx_ring *txr = adapter->tx_rings;
5225 struct rx_ring *rxr = adapter->rx_rings;
5227 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5228 struct sysctl_oid *tree = adapter->sysctl_tree;
5229 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5230 struct ixgbe_hw_stats *stats = &adapter->stats;
5232 struct sysctl_oid *stat_node, *queue_node;
5233 struct sysctl_oid_list *stat_list, *queue_list;
5235 #define QUEUE_NAME_LEN 32
5236 char namebuf[QUEUE_NAME_LEN];
5238 /* Driver Statistics */
5239 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5240 CTLFLAG_RD, &adapter->dropped_pkts,
5241 "Driver dropped packets");
5242 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5243 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5244 "m_defrag() failed");
5245 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5246 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5247 "Driver tx dma failure in xmit");
5248 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5249 CTLFLAG_RD, &adapter->watchdog_events,
5250 "Watchdog timeouts");
5251 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5252 CTLFLAG_RD, &adapter->tso_tx,
5254 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5255 CTLFLAG_RD, &adapter->link_irq,
5256 "Link MSIX IRQ Handled");
5258 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5259 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5260 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5261 CTLFLAG_RD, NULL, "Queue Name");
5262 queue_list = SYSCTL_CHILDREN(queue_node);
5263 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5264 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5265 "irqs on this queue");
5266 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5267 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5268 ixgbe_sysctl_tdh_handler, "IU",
5269 "Transmit Descriptor Head");
5270 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5271 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5272 ixgbe_sysctl_tdt_handler, "IU",
5273 "Transmit Descriptor Tail");
5274 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5275 CTLFLAG_RD, &txr->no_desc_avail, 0,
5276 "Queue No Descriptor Available");
5277 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5278 CTLFLAG_RD, &txr->total_packets, 0,
5279 "Queue Packets Transmitted");
5282 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5283 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5284 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5285 CTLFLAG_RD, NULL, "Queue Name");
5286 queue_list = SYSCTL_CHILDREN(queue_node);
5289 struct lro_ctrl *lro = &rxr->lro;
5292 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5293 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5294 CTLFLAG_RD, NULL, "Queue Name");
5295 queue_list = SYSCTL_CHILDREN(queue_node);
5297 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5298 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5299 ixgbe_sysctl_rdh_handler, "IU",
5300 "Receive Descriptor Head");
5301 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5302 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5303 ixgbe_sysctl_rdt_handler, "IU",
5304 "Receive Descriptor Tail");
5305 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5306 CTLFLAG_RD, &rxr->rx_packets, 0,
5307 "Queue Packets Received");
5308 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5309 CTLFLAG_RD, &rxr->rx_bytes, 0,
5310 "Queue Bytes Received");
5312 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5313 CTLFLAG_RD, &lro->lro_queued, 0,
5315 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5316 CTLFLAG_RD, &lro->lro_flushed, 0,
5321 /* MAC stats get the own sub node */
5323 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5324 CTLFLAG_RD, NULL, "MAC Statistics");
5325 stat_list = SYSCTL_CHILDREN(stat_node);
5327 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5328 CTLFLAG_RD, &stats->crcerrs, 0,
5330 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5331 CTLFLAG_RD, &stats->illerrc, 0,
5332 "Illegal Byte Errors");
5333 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5334 CTLFLAG_RD, &stats->errbc, 0,
5336 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5337 CTLFLAG_RD, &stats->mspdc, 0,
5338 "MAC Short Packets Discarded");
5339 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5340 CTLFLAG_RD, &stats->mlfc, 0,
5341 "MAC Local Faults");
5342 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5343 CTLFLAG_RD, &stats->mrfc, 0,
5344 "MAC Remote Faults");
5345 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5346 CTLFLAG_RD, &stats->rlec, 0,
5347 "Receive Length Errors");
5348 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5349 CTLFLAG_RD, &stats->lxontxc, 0,
5350 "Link XON Transmitted");
5351 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5352 CTLFLAG_RD, &stats->lxonrxc, 0,
5353 "Link XON Received");
5354 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5355 CTLFLAG_RD, &stats->lxofftxc, 0,
5356 "Link XOFF Transmitted");
5357 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5358 CTLFLAG_RD, &stats->lxoffrxc, 0,
5359 "Link XOFF Received");
5361 /* Packet Reception Stats */
5362 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5363 CTLFLAG_RD, &stats->tor, 0,
5364 "Total Octets Received");
5365 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5366 CTLFLAG_RD, &stats->gorc, 0,
5367 "Good Octets Received");
5368 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5369 CTLFLAG_RD, &stats->tpr, 0,
5370 "Total Packets Received");
5371 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5372 CTLFLAG_RD, &stats->gprc, 0,
5373 "Good Packets Received");
5374 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5375 CTLFLAG_RD, &stats->mprc, 0,
5376 "Multicast Packets Received");
5377 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5378 CTLFLAG_RD, &stats->bprc, 0,
5379 "Broadcast Packets Received");
5380 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5381 CTLFLAG_RD, &stats->prc64, 0,
5382 "64 byte frames received ");
5383 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5384 CTLFLAG_RD, &stats->prc127, 0,
5385 "65-127 byte frames received");
5386 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5387 CTLFLAG_RD, &stats->prc255, 0,
5388 "128-255 byte frames received");
5389 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5390 CTLFLAG_RD, &stats->prc511, 0,
5391 "256-511 byte frames received");
5392 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5393 CTLFLAG_RD, &stats->prc1023, 0,
5394 "512-1023 byte frames received");
5395 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5396 CTLFLAG_RD, &stats->prc1522, 0,
5397 "1023-1522 byte frames received");
5398 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5399 CTLFLAG_RD, &stats->ruc, 0,
5400 "Receive Undersized");
5401 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5402 CTLFLAG_RD, &stats->rfc, 0,
5403 "Fragmented Packets Received ");
5404 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5405 CTLFLAG_RD, &stats->roc, 0,
5406 "Oversized Packets Received");
5407 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5408 CTLFLAG_RD, &stats->rjc, 0,
5410 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5411 CTLFLAG_RD, &stats->mngprc, 0,
5412 "Management Packets Received");
5413 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5414 CTLFLAG_RD, &stats->mngptc, 0,
5415 "Management Packets Dropped");
5416 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5417 CTLFLAG_RD, &stats->xec, 0,
5420 /* Packet Transmission Stats */
5421 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5422 CTLFLAG_RD, &stats->gotc, 0,
5423 "Good Octets Transmitted");
5424 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5425 CTLFLAG_RD, &stats->tpt, 0,
5426 "Total Packets Transmitted");
5427 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5428 CTLFLAG_RD, &stats->gptc, 0,
5429 "Good Packets Transmitted");
5430 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5431 CTLFLAG_RD, &stats->bptc, 0,
5432 "Broadcast Packets Transmitted");
5433 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5434 CTLFLAG_RD, &stats->mptc, 0,
5435 "Multicast Packets Transmitted");
5436 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5437 CTLFLAG_RD, &stats->mngptc, 0,
5438 "Management Packets Transmitted");
5439 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5440 CTLFLAG_RD, &stats->ptc64, 0,
5441 "64 byte frames transmitted ");
5442 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5443 CTLFLAG_RD, &stats->ptc127, 0,
5444 "65-127 byte frames transmitted");
5445 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5446 CTLFLAG_RD, &stats->ptc255, 0,
5447 "128-255 byte frames transmitted");
5448 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5449 CTLFLAG_RD, &stats->ptc511, 0,
5450 "256-511 byte frames transmitted");
5451 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5452 CTLFLAG_RD, &stats->ptc1023, 0,
5453 "512-1023 byte frames transmitted");
5454 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5455 CTLFLAG_RD, &stats->ptc1522, 0,
5456 "1024-1522 byte frames transmitted");
5459 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5460 CTLFLAG_RD, &stats->fccrc, 0,
5462 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5463 CTLFLAG_RD, &stats->fclast, 0,
5465 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5466 CTLFLAG_RD, &stats->fcoerpdc, 0,
5467 "FCoE Packets Dropped");
5468 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5469 CTLFLAG_RD, &stats->fcoeprc, 0,
5470 "FCoE Packets Received");
5471 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5472 CTLFLAG_RD, &stats->fcoeptc, 0,
5473 "FCoE Packets Transmitted");
5474 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5475 CTLFLAG_RD, &stats->fcoedwrc, 0,
5476 "FCoE DWords Received");
5477 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5478 CTLFLAG_RD, &stats->fcoedwtc, 0,
5479 "FCoE DWords Transmitted");
5483 ** Set flow control using sysctl:
5484 ** Flow control values:
5491 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5494 struct adapter *adapter = (struct adapter *) arg1;
5497 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5498 if ((error) || (req->newptr == NULL))
5501 /* Don't bother if it's not changed */
5502 if (adapter->fc == last)
5505 switch (adapter->fc) {
5506 case ixgbe_fc_rx_pause:
5507 case ixgbe_fc_tx_pause:
5509 adapter->hw.fc.requested_mode = adapter->fc;
5513 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5515 /* Don't autoneg if forcing a value */
5516 adapter->hw.fc.disable_fc_autoneg = TRUE;
5517 ixgbe_fc_enable(&adapter->hw);
5522 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5523 const char *description, int *limit, int value)
5526 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5527 SYSCTL_CHILDREN(adapter->sysctl_tree),
5528 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5532 ** Control link advertise speed:
5533 ** 1 - advertise only 1G
5534 ** 2 - advertise 100Mb
5535 ** 3 - advertise normal
5538 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5541 struct adapter *adapter;
5543 struct ixgbe_hw *hw;
5544 ixgbe_link_speed speed, last;
5546 adapter = (struct adapter *) arg1;
5549 last = adapter->advertise;
5551 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5552 if ((error) || (adapter->advertise == -1))
5555 if (adapter->advertise == last) /* no change */
5558 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5559 (hw->phy.multispeed_fiber)))
5562 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5563 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5567 if (adapter->advertise == 1)
5568 speed = IXGBE_LINK_SPEED_1GB_FULL;
5569 else if (adapter->advertise == 2)
5570 speed = IXGBE_LINK_SPEED_100_FULL;
5571 else if (adapter->advertise == 3)
5572 speed = IXGBE_LINK_SPEED_1GB_FULL |
5573 IXGBE_LINK_SPEED_10GB_FULL;
5574 else /* bogus value */
5577 hw->mac.autotry_restart = TRUE;
5578 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5584 ** Thermal Shutdown Trigger
5585 ** - cause a Thermal Overtemp IRQ
5588 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5590 int error, fire = 0;
5591 struct adapter *adapter = (struct adapter *) arg1;
5592 struct ixgbe_hw *hw = &adapter->hw;
5595 if (hw->mac.type != ixgbe_mac_X540)
5598 error = sysctl_handle_int(oidp, &fire, 0, req);
5599 if ((error) || (req->newptr == NULL))
5603 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5604 reg |= IXGBE_EICR_TS;
5605 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5612 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5616 /* convert rate in intr/s to hw representation */
5618 eitr = 1000000 / rate;
5619 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5622 /* Don't disable it */
5623 eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5624 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5625 /* Don't allow it to be too large */
5626 eitr = IXGBE_EITR_INTVL_MASK;
5629 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5633 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5635 struct adapter *sc = (void *)arg1;
5636 struct ifnet *ifp = sc->ifp;
5637 int error, intr_rate, running;
5638 struct ix_queue *que = sc->queues;
5640 intr_rate = sc->intr_rate;
5641 error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5642 if (error || req->newptr == NULL)
5647 ifnet_serialize_all(ifp);
5649 sc->intr_rate = intr_rate;
5650 running = ifp->if_flags & IFF_RUNNING;
5652 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5654 if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5655 for (int i = 0; i < sc->num_queues; i++, que++)
5656 ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5660 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5662 ifnet_deserialize_all(ifp);
5667 /* rearrange mbuf chain to get contiguous bytes */
5669 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5671 int hoff, iphlen, thoff;
5675 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5677 iphlen = m->m_pkthdr.csum_iphlen;
5678 thoff = m->m_pkthdr.csum_thlen;
5679 hoff = m->m_pkthdr.csum_lhlen;
5681 KASSERT(iphlen > 0, ("invalid ip hlen"));
5682 KASSERT(thoff > 0, ("invalid tcp hlen"));
5683 KASSERT(hoff > 0, ("invalid ether hlen"));
5685 if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5686 m = m_pullup(m, hoff + iphlen + thoff);