1 /******************************************************************************
3 Copyright (c) 2001-2012, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
36 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 static void ixgbe_start(struct ifnet *);
106 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111 static void ixgbe_qflush(struct ifnet *);
113 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void ixgbe_init(void *);
115 static void ixgbe_init_locked(struct adapter *);
116 static void ixgbe_stop(void *);
117 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int ixgbe_media_change(struct ifnet *);
119 static void ixgbe_identify_hardware(struct adapter *);
120 static int ixgbe_allocate_pci_resources(struct adapter *);
121 static int ixgbe_allocate_msix(struct adapter *);
122 static int ixgbe_allocate_legacy(struct adapter *);
123 static int ixgbe_allocate_queues(struct adapter *);
124 static int ixgbe_setup_msix(struct adapter *);
125 static void ixgbe_free_pci_resources(struct adapter *);
126 static void ixgbe_local_timer(void *);
127 static int ixgbe_setup_interface(device_t, struct adapter *);
128 static void ixgbe_config_link(struct adapter *);
130 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
131 static int ixgbe_setup_transmit_structures(struct adapter *);
132 static void ixgbe_setup_transmit_ring(struct tx_ring *);
133 static void ixgbe_initialize_transmit_units(struct adapter *);
134 static void ixgbe_free_transmit_structures(struct adapter *);
135 static void ixgbe_free_transmit_buffers(struct tx_ring *);
137 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
138 static int ixgbe_setup_receive_structures(struct adapter *);
139 static int ixgbe_setup_receive_ring(struct rx_ring *);
140 static void ixgbe_initialize_receive_units(struct adapter *);
141 static void ixgbe_free_receive_structures(struct adapter *);
142 static void ixgbe_free_receive_buffers(struct rx_ring *);
144 static void ixgbe_setup_hw_rsc(struct rx_ring *);
147 static void ixgbe_enable_intr(struct adapter *);
148 static void ixgbe_disable_intr(struct adapter *);
149 static void ixgbe_update_stats_counters(struct adapter *);
150 static void ixgbe_txeof(struct tx_ring *);
151 static void ixgbe_rxeof(struct ix_queue *, int);
152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void ixgbe_set_promisc(struct adapter *);
154 static void ixgbe_set_multi(struct adapter *);
155 static void ixgbe_update_link_status(struct adapter *);
156 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
162 struct ixgbe_dma_alloc *, int);
163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
165 const char *, int *, int);
166 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
167 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
168 static int ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
169 static void ixgbe_add_sysctl(struct adapter *);
170 static void ixgbe_set_eitr(struct adapter *, int, int);
171 static int ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
172 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void ixgbe_configure_ivars(struct adapter *);
174 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
176 static void ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
180 static void ixgbe_add_hw_stats(struct adapter *adapter);
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
186 /* Support for pluggable optic modules */
187 static bool ixgbe_sfp_probe(struct adapter *);
188 static void ixgbe_setup_optics(struct adapter *);
190 /* Legacy (single vector interrupt handler */
191 static void ixgbe_legacy_irq(void *);
193 /* The MSI/X Interrupt handlers */
194 static void ixgbe_msix_que(void *);
195 static void ixgbe_msix_link(void *);
197 /* Deferred interrupt tasklets */
198 static void ixgbe_handle_msf(void *, int);
199 static void ixgbe_handle_mod(void *, int);
202 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
203 static void ixgbe_reinit_fdir(void *, int);
206 /*********************************************************************
207 * FreeBSD Device Interface Entry Points
208 *********************************************************************/
210 static device_method_t ixgbe_methods[] = {
211 /* Device interface */
212 DEVMETHOD(device_probe, ixgbe_probe),
213 DEVMETHOD(device_attach, ixgbe_attach),
214 DEVMETHOD(device_detach, ixgbe_detach),
215 DEVMETHOD(device_shutdown, ixgbe_shutdown),
219 static driver_t ixgbe_driver = {
220 "ix", ixgbe_methods, sizeof(struct adapter),
223 devclass_t ixgbe_devclass;
224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230 ** TUNEABLE PARAMETERS:
233 /* How many packets rxeof tries to clean at a time */
234 static int ixgbe_rx_process_limit = 128;
235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
238 ** Smart speed setting, default to on
239 ** this only works as a compile option
240 ** right now as its during attach, set
241 ** this to 'ixgbe_smart_speed_off' to
244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
246 static int ixgbe_msi_enable = 1;
247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
250 * MSIX should be the default for best performance,
251 * but this allows it to be forced off for testing.
253 static int ixgbe_enable_msix = 1;
254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
257 * Header split: this causes the hardware to DMA
258 * the header into a separate mbuf from the payload,
259 * it can be a performance win in some workloads, but
260 * in others it actually hurts, its off by default.
262 static int ixgbe_header_split = FALSE;
263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
266 * Number of Queues, can be set to 0,
267 * it then autoconfigures based on the
268 * number of cpus with a max of 8. This
269 * can be overriden manually here.
271 static int ixgbe_num_queues = 0;
272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
275 ** Number of TX descriptors per ring,
276 ** setting higher than RX as this seems
277 ** the better performing choice.
279 static int ixgbe_txd = PERFORM_TXD;
280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
282 /* Number of RX descriptors per ring */
283 static int ixgbe_rxd = PERFORM_RXD;
284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
286 /* Keep running tab on them for sanity check */
287 static int ixgbe_total_ports;
291 ** For Flow Director: this is the
292 ** number of TX packets we sample
293 ** for the filter pool, this means
294 ** every 20th packet will be probed.
296 ** This feature can be disabled by
297 ** setting this to 0.
299 static int atr_sample_rate = 20;
301 ** Flow Director actually 'steals'
302 ** part of the packet buffer as its
303 ** filter pool, this variable controls
305 ** 0 = 64K, 1 = 128K, 2 = 256K
307 static int fdir_pballoc = 1;
312 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
313 * be a reference on how to implement netmap support in a driver.
314 * Additional comments are in ixgbe_netmap.h .
316 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
317 * that extend the standard driver.
319 #include <dev/netmap/ixgbe_netmap.h>
320 #endif /* DEV_NETMAP */
322 /*********************************************************************
323 * Device identification routine
325 * ixgbe_probe determines if the driver should be loaded on
326 * adapter based on PCI vendor/device id of the adapter.
328 * return BUS_PROBE_DEFAULT on success, positive on failure
329 *********************************************************************/
332 ixgbe_probe(device_t dev)
334 ixgbe_vendor_info_t *ent;
336 u16 pci_vendor_id = 0;
337 u16 pci_device_id = 0;
338 u16 pci_subvendor_id = 0;
339 u16 pci_subdevice_id = 0;
340 char adapter_name[256];
342 INIT_DEBUGOUT("ixgbe_probe: begin");
344 pci_vendor_id = pci_get_vendor(dev);
345 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
348 pci_device_id = pci_get_device(dev);
349 pci_subvendor_id = pci_get_subvendor(dev);
350 pci_subdevice_id = pci_get_subdevice(dev);
352 ent = ixgbe_vendor_info_array;
353 while (ent->vendor_id != 0) {
354 if ((pci_vendor_id == ent->vendor_id) &&
355 (pci_device_id == ent->device_id) &&
357 ((pci_subvendor_id == ent->subvendor_id) ||
358 (ent->subvendor_id == 0)) &&
360 ((pci_subdevice_id == ent->subdevice_id) ||
361 (ent->subdevice_id == 0))) {
362 ksprintf(adapter_name, "%s, Version - %s",
363 ixgbe_strings[ent->index],
364 ixgbe_driver_version);
365 device_set_desc_copy(dev, adapter_name);
367 return (BUS_PROBE_DEFAULT);
374 /*********************************************************************
375 * Device initialization routine
377 * The attach entry point is called when the driver is being loaded.
378 * This routine identifies the type of hardware, allocates all resources
379 * and initializes the hardware.
381 * return 0 on success, positive on failure
382 *********************************************************************/
385 ixgbe_attach(device_t dev)
387 struct adapter *adapter;
393 INIT_DEBUGOUT("ixgbe_attach: begin");
395 if (resource_disabled("ixgbe", device_get_unit(dev))) {
396 device_printf(dev, "Disabled by device hint\n");
400 /* Allocate, clear, and link in our adapter structure */
401 adapter = device_get_softc(dev);
402 adapter->dev = adapter->osdep.dev = dev;
406 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
408 /* Set up the timer callout */
409 callout_init_mp(&adapter->timer);
411 /* Determine hardware revision */
412 ixgbe_identify_hardware(adapter);
414 /* Enable bus mastering */
415 pci_enable_busmaster(dev);
417 /* Do base PCI setup - map BAR0 */
418 if (ixgbe_allocate_pci_resources(adapter)) {
419 device_printf(dev, "Allocation of PCI resources failed\n");
424 /* Do descriptor calc and sanity checks */
425 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
426 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
427 device_printf(dev, "TXD config issue, using default!\n");
428 adapter->num_tx_desc = DEFAULT_TXD;
430 adapter->num_tx_desc = ixgbe_txd;
433 ** With many RX rings it is easy to exceed the
434 ** system mbuf allocation. Tuning nmbclusters
435 ** can alleviate this.
437 if (nmbclusters > 0 ) {
439 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
440 if (s > nmbclusters) {
441 device_printf(dev, "RX Descriptors exceed "
442 "system mbuf max, using default instead!\n");
443 ixgbe_rxd = DEFAULT_RXD;
447 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
448 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
449 device_printf(dev, "RXD config issue, using default!\n");
450 adapter->num_rx_desc = DEFAULT_RXD;
452 adapter->num_rx_desc = ixgbe_rxd;
454 /* Allocate our TX/RX Queues */
455 if (ixgbe_allocate_queues(adapter)) {
460 /* Allocate multicast array memory. */
461 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
462 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
463 if (adapter->mta == NULL) {
464 device_printf(dev, "Can not allocate multicast setup array\n");
469 /* Initialize the shared code */
470 error = ixgbe_init_shared_code(hw);
471 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
473 ** No optics in this port, set up
474 ** so the timer routine will probe
475 ** for later insertion.
477 adapter->sfp_probe = TRUE;
479 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
480 device_printf(dev,"Unsupported SFP+ module detected!\n");
484 device_printf(dev,"Unable to initialize the shared code\n");
489 /* Make sure we have a good EEPROM before we read from it */
490 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
491 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
496 error = ixgbe_init_hw(hw);
498 case IXGBE_ERR_EEPROM_VERSION:
499 device_printf(dev, "This device is a pre-production adapter/"
500 "LOM. Please be aware there may be issues associated "
501 "with your hardware.\n If you are experiencing problems "
502 "please contact your Intel or hardware representative "
503 "who provided you with this hardware.\n");
505 case IXGBE_ERR_SFP_NOT_SUPPORTED:
506 device_printf(dev,"Unsupported SFP+ Module\n");
508 device_printf(dev,"Hardware Initialization Failure\n");
510 case IXGBE_ERR_SFP_NOT_PRESENT:
511 device_printf(dev,"No SFP+ Module found\n");
517 /* Detect and set physical type */
518 ixgbe_setup_optics(adapter);
520 if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
521 adapter->intr_type = PCI_INTR_TYPE_MSIX;
522 error = ixgbe_allocate_msix(adapter);
524 error = ixgbe_allocate_legacy(adapter);
529 /* Setup OS specific network interface */
530 if (ixgbe_setup_interface(dev, adapter) != 0)
533 /* Add sysctl tree */
534 ixgbe_add_sysctl(adapter);
536 /* Initialize statistics */
537 ixgbe_update_stats_counters(adapter);
539 /* Register for VLAN events */
540 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
541 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
542 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
543 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
545 /* Print PCIE bus type/speed/width info */
546 ixgbe_get_bus_info(hw);
547 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
548 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
549 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
550 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
551 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
552 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
555 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
556 (hw->bus.speed == ixgbe_bus_speed_2500)) {
557 device_printf(dev, "PCI-Express bandwidth available"
558 " for this card\n is not sufficient for"
559 " optimal performance.\n");
560 device_printf(dev, "For optimal performance a x8 "
561 "PCIE, or x4 PCIE 2 slot is required.\n");
564 /* let hardware know driver is loaded */
565 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
566 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
567 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
569 ixgbe_add_hw_stats(adapter);
572 ixgbe_netmap_attach(adapter);
573 #endif /* DEV_NETMAP */
574 INIT_DEBUGOUT("ixgbe_attach: end");
577 ixgbe_free_transmit_structures(adapter);
578 ixgbe_free_receive_structures(adapter);
580 if (adapter->ifp != NULL)
581 if_free(adapter->ifp);
582 ixgbe_free_pci_resources(adapter);
583 kfree(adapter->mta, M_DEVBUF);
588 /*********************************************************************
589 * Device removal routine
591 * The detach entry point is called when the driver is being removed.
592 * This routine stops the adapter and deallocates all the resources
593 * that were allocated for driver operation.
595 * return 0 on success, positive on failure
596 *********************************************************************/
599 ixgbe_detach(device_t dev)
601 struct adapter *adapter = device_get_softc(dev);
604 INIT_DEBUGOUT("ixgbe_detach: begin");
606 /* Make sure VLANS are not using driver */
607 if (adapter->ifp->if_vlantrunks != NULL) {
608 device_printf(dev,"Vlan in use, detach first\n");
612 IXGBE_CORE_LOCK(adapter);
614 IXGBE_CORE_UNLOCK(adapter);
616 /* Drain the Link queue */
618 taskqueue_drain(adapter->tq, &adapter->mod_task);
619 taskqueue_drain(adapter->tq, &adapter->msf_task);
621 taskqueue_drain(adapter->tq, &adapter->fdir_task);
623 taskqueue_free(adapter->tq);
626 /* let hardware know driver is unloading */
627 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
628 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
629 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
631 /* Unregister VLAN events */
632 if (adapter->vlan_attach != NULL)
633 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
634 if (adapter->vlan_detach != NULL)
635 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
637 ether_ifdetach(adapter->ifp);
638 callout_stop(&adapter->timer);
640 netmap_detach(adapter->ifp);
641 #endif /* DEV_NETMAP */
642 ixgbe_free_pci_resources(adapter);
643 bus_generic_detach(dev);
644 if_free(adapter->ifp);
646 ixgbe_free_transmit_structures(adapter);
647 ixgbe_free_receive_structures(adapter);
648 kfree(adapter->mta, M_DEVBUF);
649 sysctl_ctx_free(&adapter->sysctl_ctx);
651 IXGBE_CORE_LOCK_DESTROY(adapter);
655 /*********************************************************************
657 * Shutdown entry point
659 **********************************************************************/
662 ixgbe_shutdown(device_t dev)
664 struct adapter *adapter = device_get_softc(dev);
665 IXGBE_CORE_LOCK(adapter);
667 IXGBE_CORE_UNLOCK(adapter);
672 /*********************************************************************
673 * Transmit entry point
675 * ixgbe_start is called by the stack to initiate a transmit.
676 * The driver will remain in this routine as long as there are
677 * packets to transmit and transmit resources are available.
678 * In case resources are not available stack is notified and
679 * the packet is requeued.
680 **********************************************************************/
683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
686 struct adapter *adapter = txr->adapter;
688 IXGBE_TX_LOCK_ASSERT(txr);
690 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
692 if (!adapter->link_active)
695 while (!ifq_is_empty(&ifp->if_snd)) {
696 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
697 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
701 m_head = ifq_dequeue(&ifp->if_snd, NULL);
705 if (ixgbe_xmit(txr, &m_head)) {
706 #if 0 /* XXX: prepend to an ALTQ queue ? */
708 IF_PREPEND(&ifp->if_snd, m_head);
710 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
711 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
714 /* Send a copy of the frame to the BPF listener */
715 ETHER_BPF_MTAP(ifp, m_head);
717 /* Set watchdog on */
718 txr->watchdog_time = ticks;
719 txr->queue_status = IXGBE_QUEUE_WORKING;
726 * Legacy TX start - called by the stack, this
727 * always uses the first tx ring, and should
728 * not be used with multiqueue tx enabled.
731 ixgbe_start(struct ifnet *ifp)
733 struct adapter *adapter = ifp->if_softc;
734 struct tx_ring *txr = adapter->tx_rings;
736 if (ifp->if_flags & IFF_RUNNING) {
738 ixgbe_start_locked(txr, ifp);
739 IXGBE_TX_UNLOCK(txr);
744 #if 0 /* __FreeBSD_version >= 800000 */
746 ** Multiqueue Transmit driver
750 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
752 struct adapter *adapter = ifp->if_softc;
753 struct ix_queue *que;
757 /* Which queue to use */
758 if ((m->m_flags & M_FLOWID) != 0)
759 i = m->m_pkthdr.flowid % adapter->num_queues;
761 i = curcpu % adapter->num_queues;
763 txr = &adapter->tx_rings[i];
764 que = &adapter->queues[i];
766 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
767 IXGBE_TX_TRYLOCK(txr)) {
768 err = ixgbe_mq_start_locked(ifp, txr, m);
769 IXGBE_TX_UNLOCK(txr);
771 err = drbr_enqueue(ifp, txr->br, m);
772 taskqueue_enqueue(que->tq, &que->que_task);
779 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
781 struct adapter *adapter = txr->adapter;
783 int enqueued, err = 0;
785 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
786 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
787 adapter->link_active == 0) {
789 err = drbr_enqueue(ifp, txr->br, m);
795 next = drbr_dequeue(ifp, txr->br);
796 } else if (drbr_needs_enqueue(ifp, txr->br)) {
797 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
799 next = drbr_dequeue(ifp, txr->br);
803 /* Process the queue */
804 while (next != NULL) {
805 if ((err = ixgbe_xmit(txr, &next)) != 0) {
807 err = drbr_enqueue(ifp, txr->br, next);
811 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
812 /* Send a copy of the frame to the BPF listener */
813 ETHER_BPF_MTAP(ifp, next);
814 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
816 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
818 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
819 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
822 next = drbr_dequeue(ifp, txr->br);
826 /* Set watchdog on */
827 txr->queue_status |= IXGBE_QUEUE_WORKING;
828 txr->watchdog_time = ticks;
831 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
838 ** Flush all ring buffers
841 ixgbe_qflush(struct ifnet *ifp)
843 struct adapter *adapter = ifp->if_softc;
844 struct tx_ring *txr = adapter->tx_rings;
847 for (int i = 0; i < adapter->num_queues; i++, txr++) {
849 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
851 IXGBE_TX_UNLOCK(txr);
855 #endif /* __FreeBSD_version >= 800000 */
857 /*********************************************************************
860 * ixgbe_ioctl is called when the user wants to configure the
863 * return 0 on success, positive on failure
864 **********************************************************************/
867 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
869 struct adapter *adapter = ifp->if_softc;
870 struct ifreq *ifr = (struct ifreq *) data;
871 #if defined(INET) || defined(INET6)
872 struct ifaddr *ifa = (struct ifaddr *)data;
873 bool avoid_reset = FALSE;
881 if (ifa->ifa_addr->sa_family == AF_INET)
885 if (ifa->ifa_addr->sa_family == AF_INET6)
888 #if defined(INET) || defined(INET6)
890 ** Calling init results in link renegotiation,
891 ** so we avoid doing it when possible.
894 ifp->if_flags |= IFF_UP;
895 if (!(ifp->if_flags & IFF_RUNNING))
897 if (!(ifp->if_flags & IFF_NOARP))
898 arp_ifinit(ifp, ifa);
900 error = ether_ioctl(ifp, command, data);
904 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
905 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
908 IXGBE_CORE_LOCK(adapter);
909 ifp->if_mtu = ifr->ifr_mtu;
910 adapter->max_frame_size =
911 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
912 ixgbe_init_locked(adapter);
913 IXGBE_CORE_UNLOCK(adapter);
917 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
918 IXGBE_CORE_LOCK(adapter);
919 if (ifp->if_flags & IFF_UP) {
920 if ((ifp->if_flags & IFF_RUNNING)) {
921 if ((ifp->if_flags ^ adapter->if_flags) &
922 (IFF_PROMISC | IFF_ALLMULTI)) {
923 ixgbe_set_promisc(adapter);
926 ixgbe_init_locked(adapter);
928 if (ifp->if_flags & IFF_RUNNING)
930 adapter->if_flags = ifp->if_flags;
931 IXGBE_CORE_UNLOCK(adapter);
935 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
936 if (ifp->if_flags & IFF_RUNNING) {
937 IXGBE_CORE_LOCK(adapter);
938 ixgbe_disable_intr(adapter);
939 ixgbe_set_multi(adapter);
940 ixgbe_enable_intr(adapter);
941 IXGBE_CORE_UNLOCK(adapter);
946 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
947 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
951 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
952 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
953 if (mask & IFCAP_HWCSUM)
954 ifp->if_capenable ^= IFCAP_HWCSUM;
955 if (mask & IFCAP_TSO4)
956 ifp->if_capenable ^= IFCAP_TSO4;
957 if (mask & IFCAP_TSO6)
958 ifp->if_capenable ^= IFCAP_TSO6;
960 if (mask & IFCAP_LRO)
961 ifp->if_capenable ^= IFCAP_LRO;
963 if (mask & IFCAP_VLAN_HWTAGGING)
964 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
965 if (mask & IFCAP_VLAN_HWFILTER)
966 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
968 if (mask & IFCAP_VLAN_HWTSO)
969 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
971 if (ifp->if_flags & IFF_RUNNING) {
972 IXGBE_CORE_LOCK(adapter);
973 ixgbe_init_locked(adapter);
974 IXGBE_CORE_UNLOCK(adapter);
977 VLAN_CAPABILITIES(ifp);
983 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
984 error = ether_ioctl(ifp, command, data);
991 /*********************************************************************
994 * This routine is used in two ways. It is used by the stack as
995 * init entry point in network interface structure. It is also used
996 * by the driver as a hw/sw initialization routine to get to a
999 * return 0 on success, positive on failure
1000 **********************************************************************/
1001 #define IXGBE_MHADD_MFS_SHIFT 16
1004 ixgbe_init_locked(struct adapter *adapter)
1006 struct ifnet *ifp = adapter->ifp;
1007 device_t dev = adapter->dev;
1008 struct ixgbe_hw *hw = &adapter->hw;
1009 u32 k, txdctl, mhadd, gpie;
1012 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1013 INIT_DEBUGOUT("ixgbe_init: begin");
1014 hw->adapter_stopped = FALSE;
1015 ixgbe_stop_adapter(hw);
1016 callout_stop(&adapter->timer);
1018 /* reprogram the RAR[0] in case user changed it. */
1019 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1021 /* Get the latest mac address, User can use a LAA */
1022 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1023 IXGBE_ETH_LENGTH_OF_ADDRESS);
1024 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1025 hw->addr_ctrl.rar_used_count = 1;
1027 /* Set the various hardware offload abilities */
1028 ifp->if_hwassist = 0;
1029 if (ifp->if_capenable & IFCAP_TSO)
1030 ifp->if_hwassist |= CSUM_TSO;
1031 if (ifp->if_capenable & IFCAP_TXCSUM) {
1032 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1034 if (hw->mac.type != ixgbe_mac_82598EB)
1035 ifp->if_hwassist |= CSUM_SCTP;
1039 /* Prepare transmit descriptors and buffers */
1040 if (ixgbe_setup_transmit_structures(adapter)) {
1041 device_printf(dev,"Could not setup transmit structures\n");
1042 ixgbe_stop(adapter);
1047 ixgbe_initialize_transmit_units(adapter);
1049 /* Setup Multicast table */
1050 ixgbe_set_multi(adapter);
1053 ** Determine the correct mbuf pool
1054 ** for doing jumbo/headersplit
1056 if (adapter->max_frame_size <= 2048)
1057 adapter->rx_mbuf_sz = MCLBYTES;
1058 else if (adapter->max_frame_size <= 4096)
1059 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1060 else if (adapter->max_frame_size <= 9216)
1061 adapter->rx_mbuf_sz = MJUM9BYTES;
1063 adapter->rx_mbuf_sz = MJUM16BYTES;
1065 /* Prepare receive descriptors and buffers */
1066 if (ixgbe_setup_receive_structures(adapter)) {
1067 device_printf(dev,"Could not setup receive structures\n");
1068 ixgbe_stop(adapter);
1072 /* Configure RX settings */
1073 ixgbe_initialize_receive_units(adapter);
1075 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1077 /* Enable Fan Failure Interrupt */
1078 gpie |= IXGBE_SDP1_GPIEN;
1080 /* Add for Module detection */
1081 if (hw->mac.type == ixgbe_mac_82599EB)
1082 gpie |= IXGBE_SDP2_GPIEN;
1084 /* Thermal Failure Detection */
1085 if (hw->mac.type == ixgbe_mac_X540)
1086 gpie |= IXGBE_SDP0_GPIEN;
1088 if (adapter->msix > 1) {
1089 /* Enable Enhanced MSIX mode */
1090 gpie |= IXGBE_GPIE_MSIX_MODE;
1091 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1094 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1097 if (ifp->if_mtu > ETHERMTU) {
1098 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1099 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1100 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1101 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1104 /* Now enable all the queues */
1106 for (int i = 0; i < adapter->num_queues; i++) {
1107 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1108 txdctl |= IXGBE_TXDCTL_ENABLE;
1109 /* Set WTHRESH to 8, burst writeback */
1110 txdctl |= (8 << 16);
1112 * When the internal queue falls below PTHRESH (32),
1113 * start prefetching as long as there are at least
1114 * HTHRESH (1) buffers ready. The values are taken
1115 * from the Intel linux driver 3.8.21.
1116 * Prefetching enables tx line rate even with 1 queue.
1118 txdctl |= (32 << 0) | (1 << 8);
1119 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1122 for (int i = 0; i < adapter->num_queues; i++) {
1123 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1124 if (hw->mac.type == ixgbe_mac_82598EB) {
1130 rxdctl &= ~0x3FFFFF;
1133 rxdctl |= IXGBE_RXDCTL_ENABLE;
1134 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1135 for (k = 0; k < 10; k++) {
1136 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1137 IXGBE_RXDCTL_ENABLE)
1145 * In netmap mode, we must preserve the buffers made
1146 * available to userspace before the if_init()
1147 * (this is true by default on the TX side, because
1148 * init makes all buffers available to userspace).
1150 * netmap_reset() and the device specific routines
1151 * (e.g. ixgbe_setup_receive_rings()) map these
1152 * buffers at the end of the NIC ring, so here we
1153 * must set the RDT (tail) register to make sure
1154 * they are not overwritten.
1156 * In this driver the NIC ring starts at RDH = 0,
1157 * RDT points to the last slot available for reception (?),
1158 * so RDT = num_rx_desc - 1 means the whole ring is available.
1160 if (ifp->if_capenable & IFCAP_NETMAP) {
1161 struct netmap_adapter *na = NA(adapter->ifp);
1162 struct netmap_kring *kring = &na->rx_rings[i];
1163 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1165 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1167 #endif /* DEV_NETMAP */
1168 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1171 /* Set up VLAN support and filter */
1172 ixgbe_setup_vlan_hw_support(adapter);
1174 /* Enable Receive engine */
1175 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1176 if (hw->mac.type == ixgbe_mac_82598EB)
1177 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1178 rxctrl |= IXGBE_RXCTRL_RXEN;
1179 ixgbe_enable_rx_dma(hw, rxctrl);
1181 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1183 /* Set up MSI/X routing */
1184 if (ixgbe_enable_msix) {
1185 ixgbe_configure_ivars(adapter);
1186 /* Set up auto-mask */
1187 if (hw->mac.type == ixgbe_mac_82598EB)
1188 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1190 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1191 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1193 } else { /* Simple settings for Legacy/MSI */
1194 ixgbe_set_ivar(adapter, 0, 0, 0);
1195 ixgbe_set_ivar(adapter, 0, 0, 1);
1196 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1200 /* Init Flow director */
1201 if (hw->mac.type != ixgbe_mac_82598EB) {
1202 u32 hdrm = 32 << fdir_pballoc;
1204 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1205 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1210 ** Check on any SFP devices that
1211 ** need to be kick-started
1213 if (hw->phy.type == ixgbe_phy_none) {
1214 int err = hw->phy.ops.identify(hw);
1215 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1217 "Unsupported SFP+ module type was detected.\n");
1222 /* Set moderation on the Link interrupt */
1223 ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1225 /* Config/Enable Link */
1226 ixgbe_config_link(adapter);
1228 /* Hardware Packet Buffer & Flow Control setup */
1230 u32 rxpb, frame, size, tmp;
1232 frame = adapter->max_frame_size;
1234 /* Calculate High Water */
1235 if (hw->mac.type == ixgbe_mac_X540)
1236 tmp = IXGBE_DV_X540(frame, frame);
1238 tmp = IXGBE_DV(frame, frame);
1239 size = IXGBE_BT2KB(tmp);
1240 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1241 hw->fc.high_water[0] = rxpb - size;
1243 /* Now calculate Low Water */
1244 if (hw->mac.type == ixgbe_mac_X540)
1245 tmp = IXGBE_LOW_DV_X540(frame);
1247 tmp = IXGBE_LOW_DV(frame);
1248 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1250 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1251 hw->fc.pause_time = IXGBE_FC_PAUSE;
1252 hw->fc.send_xon = TRUE;
1254 /* Initialize the FC settings */
1257 /* And now turn on interrupts */
1258 ixgbe_enable_intr(adapter);
1260 /* Now inform the stack we're ready */
1261 ifp->if_flags |= IFF_RUNNING;
1262 ifp->if_flags &= ~IFF_OACTIVE;
1268 ixgbe_init(void *arg)
1270 struct adapter *adapter = arg;
1272 IXGBE_CORE_LOCK(adapter);
1273 ixgbe_init_locked(adapter);
1274 IXGBE_CORE_UNLOCK(adapter);
1281 ** MSIX Interrupt Handlers and Tasklets
1286 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1288 struct ixgbe_hw *hw = &adapter->hw;
1289 u64 queue = (u64)(1 << vector);
1292 if (hw->mac.type == ixgbe_mac_82598EB) {
1293 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1294 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1296 mask = (queue & 0xFFFFFFFF);
1298 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1299 mask = (queue >> 32);
1301 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1306 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1308 struct ixgbe_hw *hw = &adapter->hw;
1309 u64 queue = (u64)(1 << vector);
1312 if (hw->mac.type == ixgbe_mac_82598EB) {
1313 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1314 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1316 mask = (queue & 0xFFFFFFFF);
1318 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1319 mask = (queue >> 32);
1321 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1326 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1330 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1331 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1332 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1334 mask = (queues & 0xFFFFFFFF);
1335 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1336 mask = (queues >> 32);
1337 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1341 /*********************************************************************
1343 * Legacy Interrupt Service routine
1345 **********************************************************************/
1348 ixgbe_legacy_irq(void *arg)
1350 struct ix_queue *que = arg;
1351 struct adapter *adapter = que->adapter;
1352 struct ixgbe_hw *hw = &adapter->hw;
1353 struct tx_ring *txr = adapter->tx_rings;
1357 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1360 if (reg_eicr == 0) {
1361 ixgbe_enable_intr(adapter);
1365 ixgbe_rxeof(que, adapter->rx_process_limit);
1369 if (!ifq_is_empty(&adapter->ifp->if_snd))
1370 ixgbe_start_locked(txr, adapter->ifp);
1371 IXGBE_TX_UNLOCK(txr);
1373 /* Check for fan failure */
1374 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1375 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1376 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1377 "REPLACE IMMEDIATELY!!\n");
1378 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1381 /* Link status change */
1382 if (reg_eicr & IXGBE_EICR_LSC) {
1383 ixgbe_check_link(&adapter->hw,
1384 &adapter->link_speed, &adapter->link_up, 0);
1385 ixgbe_update_link_status(adapter);
1388 ixgbe_enable_intr(adapter);
1392 /*********************************************************************
1394 * MSIX Queue Interrupt Service routine
1396 **********************************************************************/
1398 ixgbe_msix_que(void *arg)
1400 struct ix_queue *que = arg;
1401 struct adapter *adapter = que->adapter;
1402 struct tx_ring *txr = que->txr;
1404 ixgbe_disable_queue(adapter, que->msix);
1407 ixgbe_rxeof(que, adapter->rx_process_limit);
1411 if (!ifq_is_empty(&adapter->ifp->if_snd))
1412 ixgbe_start_locked(txr, adapter->ifp);
1413 IXGBE_TX_UNLOCK(txr);
1415 /* Reenable this interrupt */
1416 ixgbe_enable_queue(adapter, que->msix);
1421 ixgbe_msix_link(void *arg)
1423 struct adapter *adapter = arg;
1424 struct ixgbe_hw *hw = &adapter->hw;
1427 ++adapter->link_irq;
1429 /* First get the cause */
1430 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1431 /* Clear interrupt with write */
1432 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1434 /* Link status change */
1435 if (reg_eicr & IXGBE_EICR_LSC) {
1436 ixgbe_check_link(&adapter->hw,
1437 &adapter->link_speed, &adapter->link_up, 0);
1438 ixgbe_update_link_status(adapter);
1441 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1443 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1444 /* This is probably overkill :) */
1445 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1447 /* Disable the interrupt */
1448 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1449 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1452 if (reg_eicr & IXGBE_EICR_ECC) {
1453 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1454 "Please Reboot!!\n");
1455 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1458 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1459 /* Clear the interrupt */
1460 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1461 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1462 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1463 /* Clear the interrupt */
1464 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1465 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1469 /* Check for fan failure */
1470 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1471 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1472 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1473 "REPLACE IMMEDIATELY!!\n");
1474 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1477 /* Check for over temp condition */
1478 if ((hw->mac.type == ixgbe_mac_X540) &&
1479 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1480 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1481 "PHY IS SHUT DOWN!!\n");
1482 device_printf(adapter->dev, "System shutdown required\n");
1483 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1486 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1490 /*********************************************************************
1492 * Media Ioctl callback
1494 * This routine is called whenever the user queries the status of
1495 * the interface using ifconfig.
1497 **********************************************************************/
1499 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1501 struct adapter *adapter = ifp->if_softc;
1503 INIT_DEBUGOUT("ixgbe_media_status: begin");
1504 IXGBE_CORE_LOCK(adapter);
1505 ixgbe_update_link_status(adapter);
1507 ifmr->ifm_status = IFM_AVALID;
1508 ifmr->ifm_active = IFM_ETHER;
1510 if (!adapter->link_active) {
1511 IXGBE_CORE_UNLOCK(adapter);
1515 ifmr->ifm_status |= IFM_ACTIVE;
1517 switch (adapter->link_speed) {
1518 case IXGBE_LINK_SPEED_100_FULL:
1519 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1521 case IXGBE_LINK_SPEED_1GB_FULL:
1522 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1524 case IXGBE_LINK_SPEED_10GB_FULL:
1525 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1529 IXGBE_CORE_UNLOCK(adapter);
1534 /*********************************************************************
1536 * Media Ioctl callback
1538 * This routine is called when the user changes speed/duplex using
1539 * media/mediopt option with ifconfig.
1541 **********************************************************************/
1543 ixgbe_media_change(struct ifnet * ifp)
1545 struct adapter *adapter = ifp->if_softc;
1546 struct ifmedia *ifm = &adapter->media;
1548 INIT_DEBUGOUT("ixgbe_media_change: begin");
1550 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1553 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1555 adapter->hw.phy.autoneg_advertised =
1556 IXGBE_LINK_SPEED_100_FULL |
1557 IXGBE_LINK_SPEED_1GB_FULL |
1558 IXGBE_LINK_SPEED_10GB_FULL;
1561 device_printf(adapter->dev, "Only auto media type\n");
1568 /*********************************************************************
1570 * This routine maps the mbufs to tx descriptors, allowing the
1571 * TX engine to transmit the packets.
1572 * - return 0 on success, positive on failure
1574 **********************************************************************/
1577 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1579 struct adapter *adapter = txr->adapter;
1580 u32 olinfo_status = 0, cmd_type_len;
1582 int i, j, error, nsegs, maxsegs;
1583 int first, last = 0;
1584 struct mbuf *m_head;
1585 bus_dma_segment_t segs[adapter->num_segs];
1587 struct ixgbe_tx_buf *txbuf;
1588 union ixgbe_adv_tx_desc *txd = NULL;
1592 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1593 error = ixgbe_tso_pullup(txr, m_headp);
1599 /* Basic descriptor defines */
1600 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1601 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1603 if (m_head->m_flags & M_VLANTAG)
1604 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1607 * Important to capture the first descriptor
1608 * used because it will contain the index of
1609 * the one we tell the hardware to report back
1611 first = txr->next_avail_desc;
1612 txbuf = &txr->tx_buffers[first];
1616 * Map the packet for DMA.
1618 maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1619 if (maxsegs > adapter->num_segs)
1620 maxsegs = adapter->num_segs;
1622 error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1623 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1625 if (error == ENOBUFS)
1626 adapter->mbuf_defrag_failed++;
1628 adapter->no_tx_dma_setup++;
1635 /* Make certain there are enough descriptors */
1636 if (nsegs > txr->tx_avail - 2) {
1637 txr->no_desc_avail++;
1644 ** Set up the appropriate offload context
1645 ** this becomes the first descriptor of
1648 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1649 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1650 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1651 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1652 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1656 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1657 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1659 #ifdef IXGBE_IEEE1588
1660 /* This is changing soon to an mtag detection */
1661 if (we detect this mbuf has a TSTAMP mtag)
1662 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1666 /* Do the flow director magic */
1667 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1669 if (txr->atr_count >= atr_sample_rate) {
1670 ixgbe_atr(txr, m_head);
1675 /* Record payload length */
1677 olinfo_status |= m_head->m_pkthdr.len <<
1678 IXGBE_ADVTXD_PAYLEN_SHIFT;
1680 i = txr->next_avail_desc;
1681 for (j = 0; j < nsegs; j++) {
1685 txbuf = &txr->tx_buffers[i];
1686 txd = &txr->tx_base[i];
1687 seglen = segs[j].ds_len;
1688 segaddr = htole64(segs[j].ds_addr);
1690 txd->read.buffer_addr = segaddr;
1691 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1692 cmd_type_len |seglen);
1693 txd->read.olinfo_status = htole32(olinfo_status);
1694 last = i; /* descriptor that will get completion IRQ */
1696 if (++i == adapter->num_tx_desc)
1699 txbuf->m_head = NULL;
1700 txbuf->eop_index = -1;
1703 txd->read.cmd_type_len |=
1704 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1705 txr->tx_avail -= nsegs;
1706 txr->next_avail_desc = i;
1708 txbuf->m_head = m_head;
1709 /* Swap the dma map between the first and last descriptor */
1710 txr->tx_buffers[first].map = txbuf->map;
1712 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1714 /* Set the index of the descriptor that will be marked done */
1715 txbuf = &txr->tx_buffers[first];
1716 txbuf->eop_index = last;
1718 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1719 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1721 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1722 * hardware that this frame is available to transmit.
1724 ++txr->total_packets;
1725 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1730 bus_dmamap_unload(txr->txtag, txbuf->map);
1736 ixgbe_set_promisc(struct adapter *adapter)
1739 struct ifnet *ifp = adapter->ifp;
1741 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1742 reg_rctl &= (~IXGBE_FCTRL_UPE);
1743 reg_rctl &= (~IXGBE_FCTRL_MPE);
1744 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1746 if (ifp->if_flags & IFF_PROMISC) {
1747 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1748 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1749 } else if (ifp->if_flags & IFF_ALLMULTI) {
1750 reg_rctl |= IXGBE_FCTRL_MPE;
1751 reg_rctl &= ~IXGBE_FCTRL_UPE;
1752 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1758 /*********************************************************************
1761 * This routine is called whenever multicast address list is updated.
1763 **********************************************************************/
1764 #define IXGBE_RAR_ENTRIES 16
1767 ixgbe_set_multi(struct adapter *adapter)
1772 struct ifmultiaddr *ifma;
1774 struct ifnet *ifp = adapter->ifp;
1776 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1779 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1780 MAX_NUM_MULTICAST_ADDRESSES);
1782 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1783 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1784 if (ifp->if_flags & IFF_PROMISC)
1785 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1786 else if (ifp->if_flags & IFF_ALLMULTI) {
1787 fctrl |= IXGBE_FCTRL_MPE;
1788 fctrl &= ~IXGBE_FCTRL_UPE;
1790 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1792 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1794 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1795 if (ifma->ifma_addr->sa_family != AF_LINK)
1797 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1798 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1799 IXGBE_ETH_LENGTH_OF_ADDRESS);
1804 ixgbe_update_mc_addr_list(&adapter->hw,
1805 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1811 * This is an iterator function now needed by the multicast
1812 * shared code. It simply feeds the shared code routine the
1813 * addresses in the array of ixgbe_set_multi() one by one.
1816 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1818 u8 *addr = *update_ptr;
1822 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1823 *update_ptr = newptr;
1828 /*********************************************************************
1831 * This routine checks for link status,updates statistics,
1832 * and runs the watchdog check.
1834 **********************************************************************/
1837 ixgbe_local_timer(void *arg)
1839 struct adapter *adapter = arg;
1840 device_t dev = adapter->dev;
1841 struct ifnet *ifp = adapter->ifp;
1842 struct ix_queue *que = adapter->queues;
1843 struct tx_ring *txr = adapter->tx_rings;
1844 int hung, busy, paused;
1846 IXGBE_CORE_LOCK(adapter);
1847 hung = busy = paused = 0;
1849 /* Check for pluggable optics */
1850 if (adapter->sfp_probe)
1851 if (!ixgbe_sfp_probe(adapter))
1852 goto out; /* Nothing to do */
1854 ixgbe_update_link_status(adapter);
1855 ixgbe_update_stats_counters(adapter);
1858 * If the interface has been paused
1859 * then don't do the watchdog check
1861 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1865 ** Check the TX queues status
1866 ** - central locked handling of OACTIVE
1867 ** - watchdog only if all queues show hung
1869 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1870 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1873 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1876 /* Only truely watchdog if all queues show hung */
1877 if (hung == adapter->num_queues)
1879 /* Only turn off the stack flow when ALL are depleted */
1880 if (busy == adapter->num_queues)
1881 ifp->if_flags |= IFF_OACTIVE;
1882 else if ((ifp->if_flags & IFF_OACTIVE) &&
1883 (busy < adapter->num_queues))
1884 ifp->if_flags &= ~IFF_OACTIVE;
1887 ixgbe_rearm_queues(adapter, adapter->que_mask);
1888 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1889 IXGBE_CORE_UNLOCK(adapter);
1893 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1894 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1895 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1896 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1897 device_printf(dev,"TX(%d) desc avail = %d,"
1898 "Next TX to Clean = %d\n",
1899 txr->me, txr->tx_avail, txr->next_to_clean);
1900 adapter->ifp->if_flags &= ~IFF_RUNNING;
1901 adapter->watchdog_events++;
1902 ixgbe_init_locked(adapter);
1904 IXGBE_CORE_UNLOCK(adapter);
1908 ** Note: this routine updates the OS on the link state
1909 ** the real check of the hardware only happens with
1910 ** a link interrupt.
1913 ixgbe_update_link_status(struct adapter *adapter)
1915 struct ifnet *ifp = adapter->ifp;
1916 struct tx_ring *txr = adapter->tx_rings;
1917 device_t dev = adapter->dev;
1920 if (adapter->link_up){
1921 if (adapter->link_active == FALSE) {
1923 device_printf(dev,"Link is up %d Gbps %s \n",
1924 ((adapter->link_speed == 128)? 10:1),
1926 adapter->link_active = TRUE;
1927 /* Update any Flow Control changes */
1928 ixgbe_fc_enable(&adapter->hw);
1929 ifp->if_link_state = LINK_STATE_UP;
1930 if_link_state_change(ifp);
1932 } else { /* Link down */
1933 if (adapter->link_active == TRUE) {
1935 device_printf(dev,"Link is Down\n");
1936 ifp->if_link_state = LINK_STATE_DOWN;
1937 if_link_state_change(ifp);
1938 adapter->link_active = FALSE;
1939 for (int i = 0; i < adapter->num_queues;
1941 txr->queue_status = IXGBE_QUEUE_IDLE;
1949 /*********************************************************************
1951 * This routine disables all traffic on the adapter by issuing a
1952 * global reset on the MAC and deallocates TX/RX buffers.
1954 **********************************************************************/
1957 ixgbe_stop(void *arg)
1960 struct adapter *adapter = arg;
1961 struct ixgbe_hw *hw = &adapter->hw;
1964 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1966 INIT_DEBUGOUT("ixgbe_stop: begin\n");
1967 ixgbe_disable_intr(adapter);
1968 callout_stop(&adapter->timer);
1970 /* Let the stack know...*/
1971 ifp->if_flags &= ~IFF_RUNNING;
1972 ifp->if_flags |= IFF_OACTIVE;
1975 hw->adapter_stopped = FALSE;
1976 ixgbe_stop_adapter(hw);
1977 /* Turn off the laser */
1978 if (hw->phy.multispeed_fiber)
1979 ixgbe_disable_tx_laser(hw);
1981 /* reprogram the RAR[0] in case user changed it. */
1982 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1988 /*********************************************************************
1990 * Determine hardware revision.
1992 **********************************************************************/
1994 ixgbe_identify_hardware(struct adapter *adapter)
1996 device_t dev = adapter->dev;
1997 struct ixgbe_hw *hw = &adapter->hw;
1999 /* Save off the information about this board */
2000 hw->vendor_id = pci_get_vendor(dev);
2001 hw->device_id = pci_get_device(dev);
2002 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2003 hw->subsystem_vendor_id =
2004 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2005 hw->subsystem_device_id =
2006 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2008 /* We need this here to set the num_segs below */
2009 ixgbe_set_mac_type(hw);
2011 /* Pick up the 82599 and VF settings */
2012 if (hw->mac.type != ixgbe_mac_82598EB) {
2013 hw->phy.smart_speed = ixgbe_smart_speed;
2014 adapter->num_segs = IXGBE_82599_SCATTER;
2016 adapter->num_segs = IXGBE_82598_SCATTER;
2021 /*********************************************************************
2023 * Determine optic type
2025 **********************************************************************/
2027 ixgbe_setup_optics(struct adapter *adapter)
2029 struct ixgbe_hw *hw = &adapter->hw;
2032 layer = ixgbe_get_supported_physical_layer(hw);
2034 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2035 adapter->optics = IFM_10G_T;
2039 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2040 adapter->optics = IFM_1000_T;
2044 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2045 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2046 adapter->optics = IFM_10G_LR;
2050 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2051 adapter->optics = IFM_10G_SR;
2055 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2056 adapter->optics = IFM_10G_TWINAX;
2060 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2061 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2062 adapter->optics = IFM_10G_CX4;
2066 /* If we get here just set the default */
2067 adapter->optics = IFM_ETHER | IFM_AUTO;
2071 /*********************************************************************
2073 * Setup the Legacy or MSI Interrupt handler
2075 **********************************************************************/
2077 ixgbe_allocate_legacy(struct adapter *adapter)
2079 device_t dev = adapter->dev;
2080 struct ix_queue *que = adapter->queues;
2082 unsigned int intr_flags;
2085 if (adapter->msix == 1)
2088 /* Try allocating a MSI interrupt first */
2089 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2092 /* We allocate a single interrupt resource */
2093 adapter->res = bus_alloc_resource_any(dev,
2094 SYS_RES_IRQ, &rid, intr_flags);
2095 if (adapter->res == NULL) {
2096 device_printf(dev, "Unable to allocate bus resource: "
2101 /* Tasklets for Link, SFP and Multispeed Fiber */
2102 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2103 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2105 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2108 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2109 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2110 device_printf(dev, "Failed to register fast interrupt "
2111 "handler: %d\n", error);
2112 taskqueue_free(adapter->tq);
2116 /* For simplicity in the handlers */
2117 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2123 /*********************************************************************
2125 * Setup MSIX Interrupt resources and handlers
2127 **********************************************************************/
2129 ixgbe_allocate_msix(struct adapter *adapter)
2131 device_t dev = adapter->dev;
2132 struct ix_queue *que = adapter->queues;
2133 int error, rid, vector = 0;
2136 error = pci_setup_msix(dev);
2138 device_printf(dev, "MSI-X setup failed\n");
2142 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2146 ** Bind the msix vector, and thus the
2147 ** ring to the corresponding cpu.
2149 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2151 device_printf(dev, "pci_alloc_msix_vector failed\n");
2155 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2156 RF_SHAREABLE | RF_ACTIVE);
2157 if (que->res == NULL) {
2158 device_printf(dev,"Unable to allocate"
2159 " bus resource: que interrupt [%d]\n", vector);
2162 /* Set the handler function */
2163 ksnprintf(desc, sizeof(desc), "%s que %d",
2164 device_get_nameunit(dev), i);
2165 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2166 ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2169 device_printf(dev, "Failed to register QUE handler");
2173 adapter->que_mask |= (u64)(1 << que->msix);
2176 /* and Link, bind vector to cpu #0 */
2178 error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2180 device_printf(dev, "pci_alloc_msix_vector failed\n");
2183 adapter->res = bus_alloc_resource_any(dev,
2184 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2185 if (!adapter->res) {
2186 device_printf(dev,"Unable to allocate"
2187 " bus resource: Link interrupt [%d]\n", rid);
2190 /* Set the link handler function */
2191 error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2192 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2195 adapter->res = NULL;
2196 device_printf(dev, "Failed to register LINK handler");
2199 pci_enable_msix(dev);
2201 adapter->linkvec = vector;
2202 /* Tasklets for Link, SFP and Multispeed Fiber */
2203 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2204 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2206 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2213 * Setup Either MSI/X or MSI
2216 ixgbe_setup_msix(struct adapter *adapter)
2218 device_t dev = adapter->dev;
2219 int rid, want, queues, msgs;
2221 /* Override by tuneable */
2222 if (ixgbe_enable_msix == 0)
2225 /* First try MSI/X */
2226 rid = PCIR_BAR(MSIX_82598_BAR);
2227 adapter->msix_mem = bus_alloc_resource_any(dev,
2228 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2229 if (!adapter->msix_mem) {
2230 rid += 4; /* 82599 maps in higher BAR */
2231 adapter->msix_mem = bus_alloc_resource_any(dev,
2232 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2234 if (!adapter->msix_mem) {
2235 /* May not be enabled */
2236 device_printf(adapter->dev,
2237 "Unable to map MSIX table \n");
2241 msgs = pci_msix_count(dev);
2242 if (msgs == 0) { /* system has msix disabled */
2243 bus_release_resource(dev, SYS_RES_MEMORY,
2244 rid, adapter->msix_mem);
2245 adapter->msix_mem = NULL;
2249 /* Figure out a reasonable auto config value */
2250 queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2252 if (ixgbe_num_queues != 0)
2253 queues = ixgbe_num_queues;
2254 /* Set max queues to 8 when autoconfiguring */
2255 else if ((ixgbe_num_queues == 0) && (queues > 8))
2259 ** Want one vector (RX/TX pair) per queue
2260 ** plus an additional for Link.
2266 device_printf(adapter->dev,
2267 "MSIX Configuration Problem, "
2268 "%d vectors but %d queues wanted!\n",
2270 return (0); /* Will go to Legacy setup */
2273 device_printf(adapter->dev,
2274 "Using MSIX interrupts with %d vectors\n", msgs);
2275 adapter->num_queues = queues;
2279 msgs = pci_msi_count(dev);
2285 ixgbe_allocate_pci_resources(struct adapter *adapter)
2288 device_t dev = adapter->dev;
2291 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2294 if (!(adapter->pci_mem)) {
2295 device_printf(dev,"Unable to allocate bus resource: memory\n");
2299 adapter->osdep.mem_bus_space_tag =
2300 rman_get_bustag(adapter->pci_mem);
2301 adapter->osdep.mem_bus_space_handle =
2302 rman_get_bushandle(adapter->pci_mem);
2303 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2305 /* Legacy defaults */
2306 adapter->num_queues = 1;
2307 adapter->hw.back = &adapter->osdep;
2310 ** Now setup MSI or MSI/X, should
2311 ** return us the number of supported
2312 ** vectors. (Will be 1 for MSI)
2314 adapter->msix = ixgbe_setup_msix(adapter);
2319 ixgbe_free_pci_resources(struct adapter * adapter)
2321 struct ix_queue *que = adapter->queues;
2322 device_t dev = adapter->dev;
2325 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2326 memrid = PCIR_BAR(MSIX_82598_BAR);
2328 memrid = PCIR_BAR(MSIX_82599_BAR);
2331 ** There is a slight possibility of a failure mode
2332 ** in attach that will result in entering this function
2333 ** before interrupt resources have been initialized, and
2334 ** in that case we do not want to execute the loops below
2335 ** We can detect this reliably by the state of the adapter
2338 if (adapter->res == NULL)
2342 ** Release all msix queue resources:
2344 for (int i = 0; i < adapter->num_queues; i++, que++) {
2345 rid = que->msix + 1;
2346 if (que->tag != NULL) {
2347 bus_teardown_intr(dev, que->res, que->tag);
2350 if (que->res != NULL)
2351 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2355 /* Clean the Legacy or Link interrupt last */
2356 if (adapter->linkvec) /* we are doing MSIX */
2357 rid = adapter->linkvec + 1;
2359 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2361 if (adapter->tag != NULL) {
2362 bus_teardown_intr(dev, adapter->res, adapter->tag);
2363 adapter->tag = NULL;
2365 if (adapter->res != NULL)
2366 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2367 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2368 pci_release_msi(adapter->dev);
2372 pci_release_msi(dev);
2374 if (adapter->msix_mem != NULL)
2375 bus_release_resource(dev, SYS_RES_MEMORY,
2376 memrid, adapter->msix_mem);
2378 if (adapter->pci_mem != NULL)
2379 bus_release_resource(dev, SYS_RES_MEMORY,
2380 PCIR_BAR(0), adapter->pci_mem);
2385 /*********************************************************************
2387 * Setup networking device structure and register an interface.
2389 **********************************************************************/
2391 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2393 struct ixgbe_hw *hw = &adapter->hw;
2396 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2398 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2400 device_printf(dev, "can not allocate ifnet structure\n");
2403 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2404 ifp->if_baudrate = 1000000000;
2405 ifp->if_init = ixgbe_init;
2406 ifp->if_softc = adapter;
2407 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2408 ifp->if_ioctl = ixgbe_ioctl;
2409 ifp->if_start = ixgbe_start;
2410 #if 0 /* __FreeBSD_version >= 800000 */
2411 ifp->if_transmit = ixgbe_mq_start;
2412 ifp->if_qflush = ixgbe_qflush;
2414 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2416 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2418 adapter->max_frame_size =
2419 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2422 * Tell the upper layer(s) we support long frames.
2424 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2426 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2427 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2428 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2433 ifp->if_capenable = ifp->if_capabilities;
2435 /* Don't enable LRO by default */
2437 ifp->if_capabilities |= IFCAP_LRO;
2441 ** Don't turn this on by default, if vlans are
2442 ** created on another pseudo device (eg. lagg)
2443 ** then vlan events are not passed thru, breaking
2444 ** operation, but with HW FILTER off it works. If
2445 ** using vlans directly on the ixgbe driver you can
2446 ** enable this and get full hardware tag filtering.
2448 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2451 * Specify the media types supported by this adapter and register
2452 * callbacks to update media and link information
2454 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2455 ixgbe_media_status);
2456 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2457 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2458 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2459 ifmedia_add(&adapter->media,
2460 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2461 ifmedia_add(&adapter->media,
2462 IFM_ETHER | IFM_1000_T, 0, NULL);
2464 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2465 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2471 ixgbe_config_link(struct adapter *adapter)
2473 struct ixgbe_hw *hw = &adapter->hw;
2474 u32 autoneg, err = 0;
2475 bool sfp, negotiate;
2477 sfp = ixgbe_is_sfp(hw);
2480 if (hw->phy.multispeed_fiber) {
2481 hw->mac.ops.setup_sfp(hw);
2482 ixgbe_enable_tx_laser(hw);
2483 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2485 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2487 if (hw->mac.ops.check_link)
2488 err = ixgbe_check_link(hw, &autoneg,
2489 &adapter->link_up, FALSE);
2492 autoneg = hw->phy.autoneg_advertised;
2493 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2494 err = hw->mac.ops.get_link_capabilities(hw,
2495 &autoneg, &negotiate);
2498 if (hw->mac.ops.setup_link)
2499 err = hw->mac.ops.setup_link(hw, autoneg,
2500 negotiate, adapter->link_up);
2506 /********************************************************************
2507 * Manage DMA'able memory.
2508 *******************************************************************/
2510 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2514 *(bus_addr_t *) arg = segs->ds_addr;
2519 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2520 struct ixgbe_dma_alloc *dma, int mapflags)
2522 device_t dev = adapter->dev;
2525 r = bus_dma_tag_create(NULL, /* parent */
2526 DBA_ALIGN, 0, /* alignment, bounds */
2527 BUS_SPACE_MAXADDR, /* lowaddr */
2528 BUS_SPACE_MAXADDR, /* highaddr */
2529 NULL, NULL, /* filter, filterarg */
2532 size, /* maxsegsize */
2533 BUS_DMA_ALLOCNOW, /* flags */
2536 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2540 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2541 BUS_DMA_NOWAIT, &dma->dma_map);
2543 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2547 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2551 mapflags | BUS_DMA_NOWAIT);
2553 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2557 dma->dma_size = size;
2560 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2562 bus_dma_tag_destroy(dma->dma_tag);
2564 dma->dma_map = NULL;
2565 dma->dma_tag = NULL;
2570 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2572 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2573 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2574 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2575 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2576 bus_dma_tag_destroy(dma->dma_tag);
2580 /*********************************************************************
2582 * Allocate memory for the transmit and receive rings, and then
2583 * the descriptors associated with each, called only once at attach.
2585 **********************************************************************/
2587 ixgbe_allocate_queues(struct adapter *adapter)
2589 device_t dev = adapter->dev;
2590 struct ix_queue *que;
2591 struct tx_ring *txr;
2592 struct rx_ring *rxr;
2593 int rsize, tsize, error = IXGBE_SUCCESS;
2594 int txconf = 0, rxconf = 0;
2596 /* First allocate the top level queue structs */
2597 if (!(adapter->queues =
2598 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2599 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2600 device_printf(dev, "Unable to allocate queue memory\n");
2605 /* First allocate the TX ring struct memory */
2606 if (!(adapter->tx_rings =
2607 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2608 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2609 device_printf(dev, "Unable to allocate TX ring memory\n");
2614 /* Next allocate the RX */
2615 if (!(adapter->rx_rings =
2616 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2617 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2618 device_printf(dev, "Unable to allocate RX ring memory\n");
2623 /* For the ring itself */
2624 tsize = roundup2(adapter->num_tx_desc *
2625 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2628 * Now set up the TX queues, txconf is needed to handle the
2629 * possibility that things fail midcourse and we need to
2630 * undo memory gracefully
2632 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2633 /* Set up some basics */
2634 txr = &adapter->tx_rings[i];
2635 txr->adapter = adapter;
2638 /* Initialize the TX side lock */
2639 IXGBE_TX_LOCK_INIT(txr);
2641 if (ixgbe_dma_malloc(adapter, tsize,
2642 &txr->txdma, BUS_DMA_NOWAIT)) {
2644 "Unable to allocate TX Descriptor memory\n");
2648 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2649 bzero((void *)txr->tx_base, tsize);
2651 /* Now allocate transmit buffers for the ring */
2652 if (ixgbe_allocate_transmit_buffers(txr)) {
2654 "Critical Failure setting up transmit buffers\n");
2658 #if 0 /* __FreeBSD_version >= 800000 */
2659 /* Allocate a buf ring */
2660 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2661 M_WAITOK, &txr->tx_mtx);
2662 if (txr->br == NULL) {
2664 "Critical Failure setting up buf ring\n");
2672 * Next the RX queues...
2674 rsize = roundup2(adapter->num_rx_desc *
2675 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2676 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2677 rxr = &adapter->rx_rings[i];
2678 /* Set up some basics */
2679 rxr->adapter = adapter;
2682 /* Initialize the RX side lock */
2683 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2684 device_get_nameunit(dev), rxr->me);
2685 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2687 if (ixgbe_dma_malloc(adapter, rsize,
2688 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2690 "Unable to allocate RxDescriptor memory\n");
2694 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2695 bzero((void *)rxr->rx_base, rsize);
2697 /* Allocate receive buffers for the ring*/
2698 if (ixgbe_allocate_receive_buffers(rxr)) {
2700 "Critical Failure setting up receive buffers\n");
2707 ** Finally set up the queue holding structs
2709 for (int i = 0; i < adapter->num_queues; i++) {
2710 que = &adapter->queues[i];
2711 que->adapter = adapter;
2712 que->txr = &adapter->tx_rings[i];
2713 que->rxr = &adapter->rx_rings[i];
2719 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2720 ixgbe_dma_free(adapter, &rxr->rxdma);
2722 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2723 ixgbe_dma_free(adapter, &txr->txdma);
2724 kfree(adapter->rx_rings, M_DEVBUF);
2726 kfree(adapter->tx_rings, M_DEVBUF);
2728 kfree(adapter->queues, M_DEVBUF);
2733 /*********************************************************************
2735 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2736 * the information needed to transmit a packet on the wire. This is
2737 * called only once at attach, setup is done every reset.
2739 **********************************************************************/
2741 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2743 struct adapter *adapter = txr->adapter;
2744 device_t dev = adapter->dev;
2745 struct ixgbe_tx_buf *txbuf;
2749 * Setup DMA descriptor areas.
2751 if ((error = bus_dma_tag_create(
2753 1, 0, /* alignment, bounds */
2754 BUS_SPACE_MAXADDR, /* lowaddr */
2755 BUS_SPACE_MAXADDR, /* highaddr */
2756 NULL, NULL, /* filter, filterarg */
2757 IXGBE_TSO_SIZE, /* maxsize */
2758 adapter->num_segs, /* nsegments */
2759 PAGE_SIZE, /* maxsegsize */
2762 device_printf(dev,"Unable to allocate TX DMA tag\n");
2766 if (!(txr->tx_buffers =
2767 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2768 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2769 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2774 /* Create the descriptor buffer dma maps */
2775 txbuf = txr->tx_buffers;
2776 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2777 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2779 device_printf(dev, "Unable to create TX DMA map\n");
2786 /* We free all, it handles case where we are in the middle */
2787 ixgbe_free_transmit_structures(adapter);
2791 /*********************************************************************
2793 * Initialize a transmit ring.
2795 **********************************************************************/
2797 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2799 struct adapter *adapter = txr->adapter;
2800 struct ixgbe_tx_buf *txbuf;
2803 struct netmap_adapter *na = NA(adapter->ifp);
2804 struct netmap_slot *slot;
2805 #endif /* DEV_NETMAP */
2807 /* Clear the old ring contents */
2811 * (under lock): if in netmap mode, do some consistency
2812 * checks and set slot to entry 0 of the netmap ring.
2814 slot = netmap_reset(na, NR_TX, txr->me, 0);
2815 #endif /* DEV_NETMAP */
2816 bzero((void *)txr->tx_base,
2817 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2819 txr->next_avail_desc = 0;
2820 txr->next_to_clean = 0;
2822 /* Free any existing tx buffers. */
2823 txbuf = txr->tx_buffers;
2824 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2825 if (txbuf->m_head != NULL) {
2826 bus_dmamap_sync(txr->txtag, txbuf->map,
2827 BUS_DMASYNC_POSTWRITE);
2828 bus_dmamap_unload(txr->txtag, txbuf->map);
2829 m_freem(txbuf->m_head);
2830 txbuf->m_head = NULL;
2834 * In netmap mode, set the map for the packet buffer.
2835 * NOTE: Some drivers (not this one) also need to set
2836 * the physical buffer address in the NIC ring.
2837 * Slots in the netmap ring (indexed by "si") are
2838 * kring->nkr_hwofs positions "ahead" wrt the
2839 * corresponding slot in the NIC ring. In some drivers
2840 * (not here) nkr_hwofs can be negative. Function
2841 * netmap_idx_n2k() handles wraparounds properly.
2844 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2845 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2847 #endif /* DEV_NETMAP */
2848 /* Clear the EOP index */
2849 txbuf->eop_index = -1;
2853 /* Set the rate at which we sample packets */
2854 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2855 txr->atr_sample = atr_sample_rate;
2858 /* Set number of descriptors available */
2859 txr->tx_avail = adapter->num_tx_desc;
2861 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2862 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2863 IXGBE_TX_UNLOCK(txr);
2866 /*********************************************************************
2868 * Initialize all transmit rings.
2870 **********************************************************************/
2872 ixgbe_setup_transmit_structures(struct adapter *adapter)
2874 struct tx_ring *txr = adapter->tx_rings;
2876 for (int i = 0; i < adapter->num_queues; i++, txr++)
2877 ixgbe_setup_transmit_ring(txr);
2882 /*********************************************************************
2884 * Enable transmit unit.
2886 **********************************************************************/
2888 ixgbe_initialize_transmit_units(struct adapter *adapter)
2890 struct tx_ring *txr = adapter->tx_rings;
2891 struct ixgbe_hw *hw = &adapter->hw;
2893 /* Setup the Base and Length of the Tx Descriptor Ring */
2895 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2896 u64 tdba = txr->txdma.dma_paddr;
2899 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2900 (tdba & 0x00000000ffffffffULL));
2901 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2902 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2903 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2905 /* Setup the HW Tx Head and Tail descriptor pointers */
2906 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2907 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2909 /* Setup Transmit Descriptor Cmd Settings */
2910 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2911 txr->queue_status = IXGBE_QUEUE_IDLE;
2913 /* Disable Head Writeback */
2914 switch (hw->mac.type) {
2915 case ixgbe_mac_82598EB:
2916 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2918 case ixgbe_mac_82599EB:
2919 case ixgbe_mac_X540:
2921 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2924 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2925 switch (hw->mac.type) {
2926 case ixgbe_mac_82598EB:
2927 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2929 case ixgbe_mac_82599EB:
2930 case ixgbe_mac_X540:
2932 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2938 if (hw->mac.type != ixgbe_mac_82598EB) {
2939 u32 dmatxctl, rttdcs;
2940 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2941 dmatxctl |= IXGBE_DMATXCTL_TE;
2942 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2943 /* Disable arbiter to set MTQC */
2944 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2945 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2946 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2947 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2948 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2949 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2955 /*********************************************************************
2957 * Free all transmit rings.
2959 **********************************************************************/
2961 ixgbe_free_transmit_structures(struct adapter *adapter)
2963 struct tx_ring *txr = adapter->tx_rings;
2965 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2967 ixgbe_free_transmit_buffers(txr);
2968 ixgbe_dma_free(adapter, &txr->txdma);
2969 IXGBE_TX_UNLOCK(txr);
2970 IXGBE_TX_LOCK_DESTROY(txr);
2972 kfree(adapter->tx_rings, M_DEVBUF);
2975 /*********************************************************************
2977 * Free transmit ring related data structures.
2979 **********************************************************************/
2981 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2983 struct adapter *adapter = txr->adapter;
2984 struct ixgbe_tx_buf *tx_buffer;
2987 INIT_DEBUGOUT("free_transmit_ring: begin");
2989 if (txr->tx_buffers == NULL)
2992 tx_buffer = txr->tx_buffers;
2993 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2994 if (tx_buffer->m_head != NULL) {
2995 bus_dmamap_sync(txr->txtag, tx_buffer->map,
2996 BUS_DMASYNC_POSTWRITE);
2997 bus_dmamap_unload(txr->txtag,
2999 m_freem(tx_buffer->m_head);
3000 tx_buffer->m_head = NULL;
3001 if (tx_buffer->map != NULL) {
3002 bus_dmamap_destroy(txr->txtag,
3004 tx_buffer->map = NULL;
3006 } else if (tx_buffer->map != NULL) {
3007 bus_dmamap_unload(txr->txtag,
3009 bus_dmamap_destroy(txr->txtag,
3011 tx_buffer->map = NULL;
3014 #if 0 /* __FreeBSD_version >= 800000 */
3015 if (txr->br != NULL)
3016 buf_ring_free(txr->br, M_DEVBUF);
3018 if (txr->tx_buffers != NULL) {
3019 kfree(txr->tx_buffers, M_DEVBUF);
3020 txr->tx_buffers = NULL;
3022 if (txr->txtag != NULL) {
3023 bus_dma_tag_destroy(txr->txtag);
3029 /*********************************************************************
3031 * Advanced Context Descriptor setup for VLAN or CSUM
3033 **********************************************************************/
3036 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3038 struct adapter *adapter = txr->adapter;
3039 struct ixgbe_adv_tx_context_desc *TXD;
3040 struct ixgbe_tx_buf *tx_buffer;
3041 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3042 struct ether_vlan_header *eh;
3044 struct ip6_hdr *ip6;
3045 int ehdrlen, ip_hlen = 0;
3048 bool offload = TRUE;
3049 int ctxd = txr->next_avail_desc;
3053 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3056 tx_buffer = &txr->tx_buffers[ctxd];
3057 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3060 ** In advanced descriptors the vlan tag must
3061 ** be placed into the descriptor itself.
3063 if (mp->m_flags & M_VLANTAG) {
3064 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3065 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3066 } else if (offload == FALSE)
3070 * Determine where frame payload starts.
3071 * Jump over vlan headers if already present,
3072 * helpful for QinQ too.
3074 eh = mtod(mp, struct ether_vlan_header *);
3075 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3076 etype = ntohs(eh->evl_proto);
3077 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3079 etype = ntohs(eh->evl_encap_proto);
3080 ehdrlen = ETHER_HDR_LEN;
3083 /* Set the ether header length */
3084 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3088 ip = (struct ip *)(mp->m_data + ehdrlen);
3089 ip_hlen = ip->ip_hl << 2;
3091 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3093 case ETHERTYPE_IPV6:
3094 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3095 ip_hlen = sizeof(struct ip6_hdr);
3096 /* XXX-BZ this will go badly in case of ext hdrs. */
3097 ipproto = ip6->ip6_nxt;
3098 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3105 vlan_macip_lens |= ip_hlen;
3106 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3110 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3111 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3115 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3116 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3121 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3122 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3130 /* Now copy bits into descriptor */
3131 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3132 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3133 TXD->seqnum_seed = htole32(0);
3134 TXD->mss_l4len_idx = htole32(0);
3136 tx_buffer->m_head = NULL;
3137 tx_buffer->eop_index = -1;
3139 /* We've consumed the first desc, adjust counters */
3140 if (++ctxd == adapter->num_tx_desc)
3142 txr->next_avail_desc = ctxd;
3148 /**********************************************************************
3150 * Setup work for hardware segmentation offload (TSO) on
3151 * adapters using advanced tx descriptors
3153 **********************************************************************/
3155 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3158 struct adapter *adapter = txr->adapter;
3159 struct ixgbe_adv_tx_context_desc *TXD;
3160 struct ixgbe_tx_buf *tx_buffer;
3161 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3162 u16 vtag = 0, eh_type;
3163 u32 mss_l4len_idx = 0, len;
3164 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3165 struct ether_vlan_header *eh;
3166 #if 0 /* IPv6 TSO */
3168 struct ip6_hdr *ip6;
3178 * Determine where frame payload starts.
3179 * Jump over vlan headers if already present
3181 eh = mtod(mp, struct ether_vlan_header *);
3182 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3183 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3184 eh_type = eh->evl_proto;
3186 ehdrlen = ETHER_HDR_LEN;
3187 eh_type = eh->evl_encap_proto;
3190 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3191 len = ehdrlen + sizeof(struct tcphdr);
3192 switch (ntohs(eh_type)) {
3193 #if 0 /* IPv6 TSO */
3195 case ETHERTYPE_IPV6:
3196 if (mp->m_len < len + sizeof(struct ip6_hdr))
3198 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3199 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3200 if (ip6->ip6_nxt != IPPROTO_TCP)
3202 ip_hlen = sizeof(struct ip6_hdr);
3203 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3204 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3205 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3211 if (mp->m_len < len + sizeof(struct ip))
3213 ip = (struct ip *)(mp->m_data + ehdrlen);
3214 if (ip->ip_p != IPPROTO_TCP)
3217 ip_hlen = ip->ip_hl << 2;
3218 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3219 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3220 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3221 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3222 /* Tell transmit desc to also do IPv4 checksum. */
3223 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3227 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3228 __func__, ntohs(eh_type));
3232 ctxd = txr->next_avail_desc;
3233 tx_buffer = &txr->tx_buffers[ctxd];
3234 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3236 tcp_hlen = th->th_off << 2;
3238 /* This is used in the transmit desc in encap */
3239 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3241 /* VLAN MACLEN IPLEN */
3242 if (mp->m_flags & M_VLANTAG) {
3243 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3244 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3247 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3248 vlan_macip_lens |= ip_hlen;
3249 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3251 /* ADV DTYPE TUCMD */
3252 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3253 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3254 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3257 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3258 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3259 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3261 TXD->seqnum_seed = htole32(0);
3262 tx_buffer->m_head = NULL;
3263 tx_buffer->eop_index = -1;
3265 if (++ctxd == adapter->num_tx_desc)
3269 txr->next_avail_desc = ctxd;
3275 ** This routine parses packet headers so that Flow
3276 ** Director can make a hashed filter table entry
3277 ** allowing traffic flows to be identified and kept
3278 ** on the same cpu. This would be a performance
3279 ** hit, but we only do it at IXGBE_FDIR_RATE of
3283 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3285 struct adapter *adapter = txr->adapter;
3286 struct ix_queue *que;
3290 struct ether_vlan_header *eh;
3291 union ixgbe_atr_hash_dword input = {.dword = 0};
3292 union ixgbe_atr_hash_dword common = {.dword = 0};
3293 int ehdrlen, ip_hlen;
3296 eh = mtod(mp, struct ether_vlan_header *);
3297 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3298 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3299 etype = eh->evl_proto;
3301 ehdrlen = ETHER_HDR_LEN;
3302 etype = eh->evl_encap_proto;
3305 /* Only handling IPv4 */
3306 if (etype != htons(ETHERTYPE_IP))
3309 ip = (struct ip *)(mp->m_data + ehdrlen);
3310 ip_hlen = ip->ip_hl << 2;
3312 /* check if we're UDP or TCP */
3315 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3316 /* src and dst are inverted */
3317 common.port.dst ^= th->th_sport;
3318 common.port.src ^= th->th_dport;
3319 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3322 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3323 /* src and dst are inverted */
3324 common.port.dst ^= uh->uh_sport;
3325 common.port.src ^= uh->uh_dport;
3326 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3332 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3333 if (mp->m_pkthdr.ether_vtag)
3334 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3336 common.flex_bytes ^= etype;
3337 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3339 que = &adapter->queues[txr->me];
3341 ** This assumes the Rx queue and Tx
3342 ** queue are bound to the same CPU
3344 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3345 input, common, que->msix);
3347 #endif /* IXGBE_FDIR */
3349 /**********************************************************************
3351 * Examine each tx_buffer in the used queue. If the hardware is done
3352 * processing the packet then free associated resources. The
3353 * tx_buffer is put back on the free queue.
3355 **********************************************************************/
3357 ixgbe_txeof(struct tx_ring *txr)
3359 struct adapter *adapter = txr->adapter;
3360 struct ifnet *ifp = adapter->ifp;
3361 u32 first, last, done, processed;
3362 struct ixgbe_tx_buf *tx_buffer;
3363 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3365 IXGBE_TX_LOCK_ASSERT(txr);
3368 if (ifp->if_capenable & IFCAP_NETMAP) {
3369 struct netmap_adapter *na = NA(ifp);
3370 struct netmap_kring *kring = &na->tx_rings[txr->me];
3372 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3374 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3375 BUS_DMASYNC_POSTREAD);
3377 * In netmap mode, all the work is done in the context
3378 * of the client thread. Interrupt handlers only wake up
3379 * clients, which may be sleeping on individual rings
3380 * or on a global resource for all rings.
3381 * To implement tx interrupt mitigation, we wake up the client
3382 * thread roughly every half ring, even if the NIC interrupts
3383 * more frequently. This is implemented as follows:
3384 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3385 * the slot that should wake up the thread (nkr_num_slots
3386 * means the user thread should not be woken up);
3387 * - the driver ignores tx interrupts unless netmap_mitigate=0
3388 * or the slot has the DD bit set.
3390 * When the driver has separate locks, we need to
3391 * release and re-acquire txlock to avoid deadlocks.
3392 * XXX see if we can find a better way.
3394 if (!netmap_mitigate ||
3395 (kring->nr_kflags < kring->nkr_num_slots &&
3396 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3397 kring->nr_kflags = kring->nkr_num_slots;
3398 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3399 IXGBE_TX_UNLOCK(txr);
3400 IXGBE_CORE_LOCK(adapter);
3401 selwakeuppri(&na->tx_si, PI_NET);
3402 IXGBE_CORE_UNLOCK(adapter);
3407 #endif /* DEV_NETMAP */
3409 if (txr->tx_avail == adapter->num_tx_desc) {
3410 txr->queue_status = IXGBE_QUEUE_IDLE;
3415 first = txr->next_to_clean;
3416 tx_buffer = &txr->tx_buffers[first];
3417 /* For cleanup we just use legacy struct */
3418 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3419 last = tx_buffer->eop_index;
3422 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3425 ** Get the index of the first descriptor
3426 ** BEYOND the EOP and call that 'done'.
3427 ** I do this so the comparison in the
3428 ** inner while loop below can be simple
3430 if (++last == adapter->num_tx_desc) last = 0;
3433 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3434 BUS_DMASYNC_POSTREAD);
3436 ** Only the EOP descriptor of a packet now has the DD
3437 ** bit set, this is what we look for...
3439 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3440 /* We clean the range of the packet */
3441 while (first != done) {
3442 tx_desc->upper.data = 0;
3443 tx_desc->lower.data = 0;
3444 tx_desc->buffer_addr = 0;
3448 if (tx_buffer->m_head) {
3450 tx_buffer->m_head->m_pkthdr.len;
3451 bus_dmamap_sync(txr->txtag,
3453 BUS_DMASYNC_POSTWRITE);
3454 bus_dmamap_unload(txr->txtag,
3456 m_freem(tx_buffer->m_head);
3457 tx_buffer->m_head = NULL;
3458 tx_buffer->map = NULL;
3460 tx_buffer->eop_index = -1;
3461 txr->watchdog_time = ticks;
3463 if (++first == adapter->num_tx_desc)
3466 tx_buffer = &txr->tx_buffers[first];
3468 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3472 /* See if there is more work now */
3473 last = tx_buffer->eop_index;
3476 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3477 /* Get next done point */
3478 if (++last == adapter->num_tx_desc) last = 0;
3483 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3484 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3486 txr->next_to_clean = first;
3489 ** Watchdog calculation, we know there's
3490 ** work outstanding or the first return
3491 ** would have been taken, so none processed
3492 ** for too long indicates a hang.
3494 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3495 txr->queue_status = IXGBE_QUEUE_HUNG;
3497 /* With a minimum free clear the depleted state bit. */
3498 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3499 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3501 if (txr->tx_avail == adapter->num_tx_desc) {
3502 txr->queue_status = IXGBE_QUEUE_IDLE;
3506 /*********************************************************************
3508 * Refresh mbuf buffers for RX descriptor rings
3509 * - now keeps its own state so discards due to resource
3510 * exhaustion are unnecessary, if an mbuf cannot be obtained
3511 * it just returns, keeping its placeholder, thus it can simply
3512 * be recalled to try again.
3514 **********************************************************************/
3516 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3518 struct adapter *adapter = rxr->adapter;
3519 bus_dma_segment_t hseg[1];
3520 bus_dma_segment_t pseg[1];
3521 struct ixgbe_rx_buf *rxbuf;
3522 struct mbuf *mh, *mp;
3523 int i, j, nsegs, error;
3524 bool refreshed = FALSE;
3526 i = j = rxr->next_to_refresh;
3527 /* Control the loop with one beyond */
3528 if (++j == adapter->num_rx_desc)
3531 while (j != limit) {
3532 rxbuf = &rxr->rx_buffers[i];
3533 if (rxr->hdr_split == FALSE)
3536 if (rxbuf->m_head == NULL) {
3537 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3543 mh->m_pkthdr.len = mh->m_len = MHLEN;
3545 mh->m_flags |= M_PKTHDR;
3546 /* Get the memory mapping */
3547 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3548 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3550 kprintf("Refresh mbufs: hdr dmamap load"
3551 " failure - %d\n", error);
3553 rxbuf->m_head = NULL;
3557 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3558 BUS_DMASYNC_PREREAD);
3559 rxr->rx_base[i].read.hdr_addr =
3560 htole64(hseg[0].ds_addr);
3563 if (rxbuf->m_pack == NULL) {
3564 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3565 M_PKTHDR, adapter->rx_mbuf_sz);
3571 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3572 /* Get the memory mapping */
3573 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3574 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3576 kprintf("Refresh mbufs: payload dmamap load"
3577 " failure - %d\n", error);
3579 rxbuf->m_pack = NULL;
3583 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3584 BUS_DMASYNC_PREREAD);
3585 rxr->rx_base[i].read.pkt_addr =
3586 htole64(pseg[0].ds_addr);
3589 /* Next is precalculated */
3591 rxr->next_to_refresh = i;
3592 if (++j == adapter->num_rx_desc)
3596 if (refreshed) /* Update hardware tail index */
3597 IXGBE_WRITE_REG(&adapter->hw,
3598 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3602 /*********************************************************************
3604 * Allocate memory for rx_buffer structures. Since we use one
3605 * rx_buffer per received packet, the maximum number of rx_buffer's
3606 * that we'll need is equal to the number of receive descriptors
3607 * that we've allocated.
3609 **********************************************************************/
3611 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3613 struct adapter *adapter = rxr->adapter;
3614 device_t dev = adapter->dev;
3615 struct ixgbe_rx_buf *rxbuf;
3616 int i, bsize, error;
3618 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3619 if (!(rxr->rx_buffers =
3620 (struct ixgbe_rx_buf *) kmalloc(bsize,
3621 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3622 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3627 if ((error = bus_dma_tag_create(NULL, /* parent */
3628 1, 0, /* alignment, bounds */
3629 BUS_SPACE_MAXADDR, /* lowaddr */
3630 BUS_SPACE_MAXADDR, /* highaddr */
3631 NULL, NULL, /* filter, filterarg */
3632 MSIZE, /* maxsize */
3634 MSIZE, /* maxsegsize */
3637 device_printf(dev, "Unable to create RX DMA tag\n");
3641 if ((error = bus_dma_tag_create(NULL, /* parent */
3642 1, 0, /* alignment, bounds */
3643 BUS_SPACE_MAXADDR, /* lowaddr */
3644 BUS_SPACE_MAXADDR, /* highaddr */
3645 NULL, NULL, /* filter, filterarg */
3646 MJUM16BYTES, /* maxsize */
3648 MJUM16BYTES, /* maxsegsize */
3651 device_printf(dev, "Unable to create RX DMA tag\n");
3655 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3656 rxbuf = &rxr->rx_buffers[i];
3657 error = bus_dmamap_create(rxr->htag,
3658 BUS_DMA_NOWAIT, &rxbuf->hmap);
3660 device_printf(dev, "Unable to create RX head map\n");
3663 error = bus_dmamap_create(rxr->ptag,
3664 BUS_DMA_NOWAIT, &rxbuf->pmap);
3666 device_printf(dev, "Unable to create RX pkt map\n");
3674 /* Frees all, but can handle partial completion */
3675 ixgbe_free_receive_structures(adapter);
3680 ** Used to detect a descriptor that has
3681 ** been merged by Hardware RSC.
3684 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3686 return (le32toh(rx->wb.lower.lo_dword.data) &
3687 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3690 /*********************************************************************
3692 * Initialize Hardware RSC (LRO) feature on 82599
3693 * for an RX ring, this is toggled by the LRO capability
3694 * even though it is transparent to the stack.
3696 **********************************************************************/
3699 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3701 struct adapter *adapter = rxr->adapter;
3702 struct ixgbe_hw *hw = &adapter->hw;
3703 u32 rscctrl, rdrxctl;
3705 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3706 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3707 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3708 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3709 #endif /* DEV_NETMAP */
3710 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3711 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3712 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3714 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3715 rscctrl |= IXGBE_RSCCTL_RSCEN;
3717 ** Limit the total number of descriptors that
3718 ** can be combined, so it does not exceed 64K
3720 if (adapter->rx_mbuf_sz == MCLBYTES)
3721 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3722 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3723 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3724 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3725 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3726 else /* Using 16K cluster */
3727 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3729 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3731 /* Enable TCP header recognition */
3732 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3733 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3734 IXGBE_PSRTYPE_TCPHDR));
3736 /* Disable RSC for ACK packets */
3737 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3738 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3745 ixgbe_free_receive_ring(struct rx_ring *rxr)
3747 struct adapter *adapter;
3748 struct ixgbe_rx_buf *rxbuf;
3751 adapter = rxr->adapter;
3752 for (i = 0; i < adapter->num_rx_desc; i++) {
3753 rxbuf = &rxr->rx_buffers[i];
3754 if (rxbuf->m_head != NULL) {
3755 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3756 BUS_DMASYNC_POSTREAD);
3757 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3758 rxbuf->m_head->m_flags |= M_PKTHDR;
3759 m_freem(rxbuf->m_head);
3761 if (rxbuf->m_pack != NULL) {
3762 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3763 BUS_DMASYNC_POSTREAD);
3764 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3765 rxbuf->m_pack->m_flags |= M_PKTHDR;
3766 m_freem(rxbuf->m_pack);
3768 rxbuf->m_head = NULL;
3769 rxbuf->m_pack = NULL;
3774 /*********************************************************************
3776 * Initialize a receive ring and its buffers.
3778 **********************************************************************/
3780 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3782 struct adapter *adapter;
3785 struct ixgbe_rx_buf *rxbuf;
3786 bus_dma_segment_t pseg[1], hseg[1];
3788 struct lro_ctrl *lro = &rxr->lro;
3790 int rsize, nsegs, error = 0;
3792 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3793 struct netmap_slot *slot;
3794 #endif /* DEV_NETMAP */
3796 adapter = rxr->adapter;
3800 /* Clear the ring contents */
3803 /* same as in ixgbe_setup_transmit_ring() */
3804 slot = netmap_reset(na, NR_RX, rxr->me, 0);
3805 #endif /* DEV_NETMAP */
3806 rsize = roundup2(adapter->num_rx_desc *
3807 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3808 bzero((void *)rxr->rx_base, rsize);
3810 /* Free current RX buffer structs and their mbufs */
3811 ixgbe_free_receive_ring(rxr);
3813 /* Configure header split? */
3814 if (ixgbe_header_split)
3815 rxr->hdr_split = TRUE;
3817 /* Now replenish the mbufs */
3818 for (int j = 0; j != adapter->num_rx_desc; ++j) {
3819 struct mbuf *mh, *mp;
3821 rxbuf = &rxr->rx_buffers[j];
3824 * In netmap mode, fill the map and set the buffer
3825 * address in the NIC ring, considering the offset
3826 * between the netmap and NIC rings (see comment in
3827 * ixgbe_setup_transmit_ring() ). No need to allocate
3828 * an mbuf, so end the block with a continue;
3831 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3835 addr = PNMB(slot + sj, &paddr);
3836 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3837 /* Update descriptor */
3838 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3841 #endif /* DEV_NETMAP */
3843 ** Don't allocate mbufs if not
3844 ** doing header split, its wasteful
3846 if (rxr->hdr_split == FALSE)
3849 /* First the header */
3850 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
3851 if (rxbuf->m_head == NULL) {
3855 m_adj(rxbuf->m_head, ETHER_ALIGN);
3857 mh->m_len = mh->m_pkthdr.len = MHLEN;
3858 mh->m_flags |= M_PKTHDR;
3859 /* Get the memory mapping */
3860 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3861 rxbuf->hmap, rxbuf->m_head, hseg, 1,
3862 &nsegs, BUS_DMA_NOWAIT);
3864 if (error != 0) /* Nothing elegant to do here */
3866 bus_dmamap_sync(rxr->htag,
3867 rxbuf->hmap, BUS_DMASYNC_PREREAD);
3868 /* Update descriptor */
3869 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3872 /* Now the payload cluster */
3873 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
3874 M_PKTHDR, adapter->rx_mbuf_sz);
3875 if (rxbuf->m_pack == NULL) {
3880 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3881 /* Get the memory mapping */
3882 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3883 rxbuf->pmap, mp, hseg, 1,
3884 &nsegs, BUS_DMA_NOWAIT);
3887 bus_dmamap_sync(rxr->ptag,
3888 rxbuf->pmap, BUS_DMASYNC_PREREAD);
3889 /* Update descriptor */
3890 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3894 /* Setup our descriptor indices */
3895 rxr->next_to_check = 0;
3896 rxr->next_to_refresh = 0;
3897 rxr->lro_enabled = FALSE;
3898 rxr->rx_split_packets = 0;
3900 rxr->discard = FALSE;
3901 rxr->vtag_strip = FALSE;
3903 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3904 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3907 ** Now set up the LRO interface:
3908 ** 82598 uses software LRO, the
3909 ** 82599 and X540 use a hardware assist.
3912 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3913 (ifp->if_capenable & IFCAP_RXCSUM) &&
3914 (ifp->if_capenable & IFCAP_LRO))
3915 ixgbe_setup_hw_rsc(rxr);
3916 else if (ifp->if_capenable & IFCAP_LRO) {
3917 int err = tcp_lro_init(lro);
3919 device_printf(dev, "LRO Initialization failed!\n");
3922 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3923 rxr->lro_enabled = TRUE;
3924 lro->ifp = adapter->ifp;
3928 IXGBE_RX_UNLOCK(rxr);
3932 ixgbe_free_receive_ring(rxr);
3933 IXGBE_RX_UNLOCK(rxr);
3937 /*********************************************************************
3939 * Initialize all receive rings.
3941 **********************************************************************/
3943 ixgbe_setup_receive_structures(struct adapter *adapter)
3945 struct rx_ring *rxr = adapter->rx_rings;
3948 for (j = 0; j < adapter->num_queues; j++, rxr++)
3949 if (ixgbe_setup_receive_ring(rxr))
3955 * Free RX buffers allocated so far, we will only handle
3956 * the rings that completed, the failing case will have
3957 * cleaned up for itself. 'j' failed, so its the terminus.
3959 for (int i = 0; i < j; ++i) {
3960 rxr = &adapter->rx_rings[i];
3961 ixgbe_free_receive_ring(rxr);
3967 /*********************************************************************
3969 * Setup receive registers and features.
3971 **********************************************************************/
3972 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3974 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3977 ixgbe_initialize_receive_units(struct adapter *adapter)
3979 struct rx_ring *rxr = adapter->rx_rings;
3980 struct ixgbe_hw *hw = &adapter->hw;
3981 struct ifnet *ifp = adapter->ifp;
3982 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
3983 u32 reta, mrqc = 0, hlreg, random[10];
3987 * Make sure receives are disabled while
3988 * setting up the descriptor ring
3990 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3991 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
3992 rxctrl & ~IXGBE_RXCTRL_RXEN);
3994 /* Enable broadcasts */
3995 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3996 fctrl |= IXGBE_FCTRL_BAM;
3997 fctrl |= IXGBE_FCTRL_DPF;
3998 fctrl |= IXGBE_FCTRL_PMCF;
3999 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4001 /* Set for Jumbo Frames? */
4002 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4003 if (ifp->if_mtu > ETHERMTU)
4004 hlreg |= IXGBE_HLREG0_JUMBOEN;
4006 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4008 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4009 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4010 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4012 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4013 #endif /* DEV_NETMAP */
4014 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4016 bufsz = (adapter->rx_mbuf_sz +
4017 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4019 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4020 u64 rdba = rxr->rxdma.dma_paddr;
4022 /* Setup the Base and Length of the Rx Descriptor Ring */
4023 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4024 (rdba & 0x00000000ffffffffULL));
4025 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4026 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4027 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4029 /* Set up the SRRCTL register */
4030 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4031 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4032 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4034 if (rxr->hdr_split) {
4035 /* Use a standard mbuf for the header */
4036 srrctl |= ((IXGBE_RX_HDR <<
4037 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4038 & IXGBE_SRRCTL_BSIZEHDR_MASK);
4039 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4041 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4042 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4044 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4045 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4046 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4049 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4050 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4051 IXGBE_PSRTYPE_UDPHDR |
4052 IXGBE_PSRTYPE_IPV4HDR |
4053 IXGBE_PSRTYPE_IPV6HDR;
4054 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4057 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4060 if (adapter->num_queues > 1) {
4064 /* set up random bits */
4065 karc4rand(&random, sizeof(random));
4067 /* Set up the redirection table */
4068 for (i = 0, j = 0; i < 128; i++, j++) {
4069 if (j == adapter->num_queues) j = 0;
4070 reta = (reta << 8) | (j * 0x11);
4072 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4075 /* Now fill our hash function seeds */
4076 for (int i = 0; i < 10; i++)
4077 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4079 /* Perform hash on these packet types */
4080 mrqc = IXGBE_MRQC_RSSEN
4081 | IXGBE_MRQC_RSS_FIELD_IPV4
4082 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4083 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4084 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4085 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4086 | IXGBE_MRQC_RSS_FIELD_IPV6
4087 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4088 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4089 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4090 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4092 /* RSS and RX IPP Checksum are mutually exclusive */
4093 rxcsum |= IXGBE_RXCSUM_PCSD;
4096 if (ifp->if_capenable & IFCAP_RXCSUM)
4097 rxcsum |= IXGBE_RXCSUM_PCSD;
4099 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4100 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4102 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4107 /*********************************************************************
4109 * Free all receive rings.
4111 **********************************************************************/
4113 ixgbe_free_receive_structures(struct adapter *adapter)
4115 struct rx_ring *rxr = adapter->rx_rings;
4117 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4119 struct lro_ctrl *lro = &rxr->lro;
4121 ixgbe_free_receive_buffers(rxr);
4122 /* Free LRO memory */
4126 /* Free the ring memory as well */
4127 ixgbe_dma_free(adapter, &rxr->rxdma);
4130 kfree(adapter->rx_rings, M_DEVBUF);
4134 /*********************************************************************
4136 * Free receive ring data structures
4138 **********************************************************************/
4140 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4142 struct adapter *adapter = rxr->adapter;
4143 struct ixgbe_rx_buf *rxbuf;
4145 INIT_DEBUGOUT("free_receive_structures: begin");
4147 /* Cleanup any existing buffers */
4148 if (rxr->rx_buffers != NULL) {
4149 for (int i = 0; i < adapter->num_rx_desc; i++) {
4150 rxbuf = &rxr->rx_buffers[i];
4151 if (rxbuf->m_head != NULL) {
4152 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4153 BUS_DMASYNC_POSTREAD);
4154 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4155 rxbuf->m_head->m_flags |= M_PKTHDR;
4156 m_freem(rxbuf->m_head);
4158 if (rxbuf->m_pack != NULL) {
4159 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4160 BUS_DMASYNC_POSTREAD);
4161 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4162 rxbuf->m_pack->m_flags |= M_PKTHDR;
4163 m_freem(rxbuf->m_pack);
4165 rxbuf->m_head = NULL;
4166 rxbuf->m_pack = NULL;
4167 if (rxbuf->hmap != NULL) {
4168 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4171 if (rxbuf->pmap != NULL) {
4172 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4176 if (rxr->rx_buffers != NULL) {
4177 kfree(rxr->rx_buffers, M_DEVBUF);
4178 rxr->rx_buffers = NULL;
4182 if (rxr->htag != NULL) {
4183 bus_dma_tag_destroy(rxr->htag);
4186 if (rxr->ptag != NULL) {
4187 bus_dma_tag_destroy(rxr->ptag);
4194 static __inline void
4195 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4199 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4200 * should be computed by hardware. Also it should not have VLAN tag in
4201 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4204 if (rxr->lro_enabled &&
4205 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4206 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4207 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4208 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4209 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4210 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4211 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4212 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4214 * Send to the stack if:
4215 ** - LRO not enabled, or
4216 ** - no LRO resources, or
4217 ** - lro enqueue fails
4219 if (rxr->lro.lro_cnt != 0)
4220 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4224 IXGBE_RX_UNLOCK(rxr);
4225 (*ifp->if_input)(ifp, m);
4229 static __inline void
4230 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4232 struct ixgbe_rx_buf *rbuf;
4234 rbuf = &rxr->rx_buffers[i];
4236 if (rbuf->fmp != NULL) {/* Partial chain ? */
4237 rbuf->fmp->m_flags |= M_PKTHDR;
4243 ** With advanced descriptors the writeback
4244 ** clobbers the buffer addrs, so its easier
4245 ** to just free the existing mbufs and take
4246 ** the normal refresh path to get new buffers
4250 m_free(rbuf->m_head);
4251 rbuf->m_head = NULL;
4255 m_free(rbuf->m_pack);
4256 rbuf->m_pack = NULL;
4263 ixgbe_add_sysctl(struct adapter *adapter)
4265 sysctl_ctx_init(&adapter->sysctl_ctx);
4266 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4267 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4268 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4269 if (adapter->sysctl_tree == NULL) {
4270 device_printf(adapter->dev, "can't add sysctl node\n");
4273 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4274 SYSCTL_CHILDREN(adapter->sysctl_tree),
4275 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4276 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4278 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4279 SYSCTL_CHILDREN(adapter->sysctl_tree),
4280 OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4281 adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4284 ** Allow a kind of speed control by forcing the autoneg
4285 ** advertised speed list to only a certain value, this
4286 ** supports 1G on 82599 devices, and 100Mb on x540.
4288 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4289 SYSCTL_CHILDREN(adapter->sysctl_tree),
4290 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4291 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4293 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4294 SYSCTL_CHILDREN(adapter->sysctl_tree),
4295 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4296 0, ixgbe_set_thermal_test, "I", "Thermal Test");
4298 /* Sysctl for limiting the amount of work done in the taskqueue */
4299 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4300 "max number of rx packets to process", &adapter->rx_process_limit,
4301 ixgbe_rx_process_limit);
4304 /*********************************************************************
4306 * This routine executes in interrupt context. It replenishes
4307 * the mbufs in the descriptor and sends data which has been
4308 * dma'ed into host memory to upper layer.
4310 * We loop at most count times if count is > 0, or until done if
4313 * Return TRUE for more work, FALSE for all clean.
4314 *********************************************************************/
4316 ixgbe_rxeof(struct ix_queue *que, int count)
4318 struct adapter *adapter = que->adapter;
4319 struct rx_ring *rxr = que->rxr;
4320 struct ifnet *ifp = adapter->ifp;
4322 struct lro_ctrl *lro = &rxr->lro;
4323 struct lro_entry *queued;
4325 int i, nextp, processed = 0;
4327 union ixgbe_adv_rx_desc *cur;
4328 struct ixgbe_rx_buf *rbuf, *nbuf;
4333 if (ifp->if_capenable & IFCAP_NETMAP) {
4335 * Same as the txeof routine: only wakeup clients on intr.
4336 * NKR_PENDINTR in nr_kflags is used to implement interrupt
4337 * mitigation (ixgbe_rxsync() will not look for new packets
4338 * unless NKR_PENDINTR is set).
4340 struct netmap_adapter *na = NA(ifp);
4342 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4343 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4344 IXGBE_RX_UNLOCK(rxr);
4345 IXGBE_CORE_LOCK(adapter);
4346 selwakeuppri(&na->rx_si, PI_NET);
4347 IXGBE_CORE_UNLOCK(adapter);
4350 #endif /* DEV_NETMAP */
4351 for (i = rxr->next_to_check; count != 0;) {
4352 struct mbuf *sendmp, *mh, *mp;
4354 u16 hlen, plen, hdr;
4358 /* Sync the ring. */
4359 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4360 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4362 cur = &rxr->rx_base[i];
4363 staterr = le32toh(cur->wb.upper.status_error);
4365 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4367 if ((ifp->if_flags & IFF_RUNNING) == 0)
4374 cur->wb.upper.status_error = 0;
4375 rbuf = &rxr->rx_buffers[i];
4379 plen = le16toh(cur->wb.upper.length);
4380 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4381 IXGBE_RXDADV_PKTTYPE_MASK;
4382 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4383 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4385 /* Process vlan info */
4386 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4387 vtag = le16toh(cur->wb.upper.vlan);
4389 /* Make sure bad packets are discarded */
4390 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4393 rxr->rx_discarded++;
4395 rxr->discard = FALSE;
4397 rxr->discard = TRUE;
4398 ixgbe_rx_discard(rxr, i);
4403 ** On 82599 which supports a hardware
4404 ** LRO (called HW RSC), packets need
4405 ** not be fragmented across sequential
4406 ** descriptors, rather the next descriptor
4407 ** is indicated in bits of the descriptor.
4408 ** This also means that we might proceses
4409 ** more than one packet at a time, something
4410 ** that has never been true before, it
4411 ** required eliminating global chain pointers
4412 ** in favor of what we are doing here. -jfv
4416 ** Figure out the next descriptor
4419 if (rxr->hw_rsc == TRUE) {
4420 rsc = ixgbe_rsc_count(cur);
4421 rxr->rsc_num += (rsc - 1);
4423 if (rsc) { /* Get hardware index */
4425 IXGBE_RXDADV_NEXTP_MASK) >>
4426 IXGBE_RXDADV_NEXTP_SHIFT);
4427 } else { /* Just sequential */
4429 if (nextp == adapter->num_rx_desc)
4432 nbuf = &rxr->rx_buffers[nextp];
4436 ** The header mbuf is ONLY used when header
4437 ** split is enabled, otherwise we get normal
4438 ** behavior, ie, both header and payload
4439 ** are DMA'd into the payload buffer.
4441 ** Rather than using the fmp/lmp global pointers
4442 ** we now keep the head of a packet chain in the
4443 ** buffer struct and pass this along from one
4444 ** descriptor to the next, until we get EOP.
4446 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4447 /* This must be an initial descriptor */
4448 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4449 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4450 if (hlen > IXGBE_RX_HDR)
4451 hlen = IXGBE_RX_HDR;
4453 mh->m_flags |= M_PKTHDR;
4455 mh->m_pkthdr.len = mh->m_len;
4456 /* Null buf pointer so it is refreshed */
4457 rbuf->m_head = NULL;
4459 ** Check the payload length, this
4460 ** could be zero if its a small
4466 mp->m_flags &= ~M_PKTHDR;
4468 mh->m_pkthdr.len += mp->m_len;
4469 /* Null buf pointer so it is refreshed */
4470 rbuf->m_pack = NULL;
4471 rxr->rx_split_packets++;
4474 ** Now create the forward
4475 ** chain so when complete
4479 /* stash the chain head */
4481 /* Make forward chain */
4483 mp->m_next = nbuf->m_pack;
4485 mh->m_next = nbuf->m_pack;
4487 /* Singlet, prepare to send */
4489 /* If hardware handled vtag */
4491 sendmp->m_pkthdr.ether_vlantag = vtag;
4492 sendmp->m_flags |= M_VLANTAG;
4497 ** Either no header split, or a
4498 ** secondary piece of a fragmented
4503 ** See if there is a stored head
4504 ** that determines what we are
4507 rbuf->m_pack = rbuf->fmp = NULL;
4509 if (sendmp != NULL) { /* secondary frag */
4510 mp->m_flags &= ~M_PKTHDR;
4511 sendmp->m_pkthdr.len += mp->m_len;
4513 /* first desc of a non-ps chain */
4515 sendmp->m_flags |= M_PKTHDR;
4516 sendmp->m_pkthdr.len = mp->m_len;
4517 if (staterr & IXGBE_RXD_STAT_VP) {
4518 sendmp->m_pkthdr.ether_vlantag = vtag;
4519 sendmp->m_flags |= M_VLANTAG;
4522 /* Pass the head pointer on */
4526 mp->m_next = nbuf->m_pack;
4530 /* Sending this frame? */
4532 sendmp->m_pkthdr.rcvif = ifp;
4535 /* capture data for AIM */
4536 rxr->bytes += sendmp->m_pkthdr.len;
4537 rxr->rx_bytes += sendmp->m_pkthdr.len;
4538 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4539 ixgbe_rx_checksum(staterr, sendmp, ptype);
4540 #if 0 /* __FreeBSD_version >= 800000 */
4541 sendmp->m_pkthdr.flowid = que->msix;
4542 sendmp->m_flags |= M_FLOWID;
4546 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4547 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4549 /* Advance our pointers to the next descriptor. */
4550 if (++i == adapter->num_rx_desc)
4553 /* Now send to the stack or do LRO */
4554 if (sendmp != NULL) {
4555 rxr->next_to_check = i;
4556 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4557 i = rxr->next_to_check;
4560 /* Every 8 descriptors we go to refresh mbufs */
4561 if (processed == 8) {
4562 ixgbe_refresh_mbufs(rxr, i);
4567 /* Refresh any remaining buf structs */
4568 if (ixgbe_rx_unrefreshed(rxr))
4569 ixgbe_refresh_mbufs(rxr, i);
4571 rxr->next_to_check = i;
4574 * Flush any outstanding LRO work
4577 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4578 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4579 tcp_lro_flush(lro, queued);
4583 IXGBE_RX_UNLOCK(rxr);
4586 ** We still have cleaning to do?
4587 ** Schedule another interrupt if so.
4589 if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4590 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4595 /*********************************************************************
4597 * Verify that the hardware indicated that the checksum is valid.
4598 * Inform the stack about the status of checksum so that stack
4599 * doesn't spend time verifying the checksum.
4601 *********************************************************************/
4603 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4605 u16 status = (u16) staterr;
4606 u8 errors = (u8) (staterr >> 24);
4609 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4610 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4613 if (status & IXGBE_RXD_STAT_IPCS) {
4614 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4615 /* IP Checksum Good */
4616 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4617 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4620 mp->m_pkthdr.csum_flags = 0;
4622 if (status & IXGBE_RXD_STAT_L4CS) {
4623 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4626 type = CSUM_SCTP_VALID;
4628 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4629 mp->m_pkthdr.csum_flags |= type;
4631 mp->m_pkthdr.csum_data = htons(0xffff);
4639 ** This routine is run via an vlan config EVENT,
4640 ** it enables us to use the HW Filter table since
4641 ** we can get the vlan id. This just creates the
4642 ** entry in the soft version of the VFTA, init will
4643 ** repopulate the real table.
4646 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4648 struct adapter *adapter = ifp->if_softc;
4651 if (ifp->if_softc != arg) /* Not our event */
4654 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4657 IXGBE_CORE_LOCK(adapter);
4658 index = (vtag >> 5) & 0x7F;
4660 adapter->shadow_vfta[index] |= (1 << bit);
4661 ++adapter->num_vlans;
4662 ixgbe_init_locked(adapter);
4663 IXGBE_CORE_UNLOCK(adapter);
4667 ** This routine is run via an vlan
4668 ** unconfig EVENT, remove our entry
4669 ** in the soft vfta.
4672 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4674 struct adapter *adapter = ifp->if_softc;
4677 if (ifp->if_softc != arg)
4680 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4683 IXGBE_CORE_LOCK(adapter);
4684 index = (vtag >> 5) & 0x7F;
4686 adapter->shadow_vfta[index] &= ~(1 << bit);
4687 --adapter->num_vlans;
4688 /* Re-init to load the changes */
4689 ixgbe_init_locked(adapter);
4690 IXGBE_CORE_UNLOCK(adapter);
4694 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4696 struct ifnet *ifp = adapter->ifp;
4697 struct ixgbe_hw *hw = &adapter->hw;
4698 struct rx_ring *rxr;
4702 ** We get here thru init_locked, meaning
4703 ** a soft reset, this has already cleared
4704 ** the VFTA and other state, so if there
4705 ** have been no vlan's registered do nothing.
4707 if (adapter->num_vlans == 0)
4711 ** A soft reset zero's out the VFTA, so
4712 ** we need to repopulate it now.
4714 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4715 if (adapter->shadow_vfta[i] != 0)
4716 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4717 adapter->shadow_vfta[i]);
4719 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4720 /* Enable the Filter Table if enabled */
4721 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4722 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4723 ctrl |= IXGBE_VLNCTRL_VFE;
4725 if (hw->mac.type == ixgbe_mac_82598EB)
4726 ctrl |= IXGBE_VLNCTRL_VME;
4727 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4729 /* Setup the queues for vlans */
4730 for (int i = 0; i < adapter->num_queues; i++) {
4731 rxr = &adapter->rx_rings[i];
4732 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4733 if (hw->mac.type != ixgbe_mac_82598EB) {
4734 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4735 ctrl |= IXGBE_RXDCTL_VME;
4736 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4738 rxr->vtag_strip = TRUE;
4743 ixgbe_enable_intr(struct adapter *adapter)
4745 struct ixgbe_hw *hw = &adapter->hw;
4746 struct ix_queue *que = adapter->queues;
4747 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4750 /* Enable Fan Failure detection */
4751 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4752 mask |= IXGBE_EIMS_GPI_SDP1;
4754 mask |= IXGBE_EIMS_ECC;
4755 mask |= IXGBE_EIMS_GPI_SDP0;
4756 mask |= IXGBE_EIMS_GPI_SDP1;
4757 mask |= IXGBE_EIMS_GPI_SDP2;
4759 mask |= IXGBE_EIMS_FLOW_DIR;
4763 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4765 /* With RSS we use auto clear */
4766 if (adapter->msix_mem) {
4767 mask = IXGBE_EIMS_ENABLE_MASK;
4768 /* Don't autoclear Link */
4769 mask &= ~IXGBE_EIMS_OTHER;
4770 mask &= ~IXGBE_EIMS_LSC;
4771 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4775 ** Now enable all queues, this is done separately to
4776 ** allow for handling the extended (beyond 32) MSIX
4777 ** vectors that can be used by 82599
4779 for (int i = 0; i < adapter->num_queues; i++, que++)
4780 ixgbe_enable_queue(adapter, que->msix);
4782 IXGBE_WRITE_FLUSH(hw);
4788 ixgbe_disable_intr(struct adapter *adapter)
4790 if (adapter->msix_mem)
4791 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4792 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4793 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4795 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4796 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4797 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4799 IXGBE_WRITE_FLUSH(&adapter->hw);
4804 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4808 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4815 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4817 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4824 ** Setup the correct IVAR register for a particular MSIX interrupt
4825 ** (yes this is all very magic and confusing :)
4826 ** - entry is the register array entry
4827 ** - vector is the MSIX vector for this queue
4828 ** - type is RX/TX/MISC
4831 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4833 struct ixgbe_hw *hw = &adapter->hw;
4836 vector |= IXGBE_IVAR_ALLOC_VAL;
4838 switch (hw->mac.type) {
4840 case ixgbe_mac_82598EB:
4842 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4844 entry += (type * 64);
4845 index = (entry >> 2) & 0x1F;
4846 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4847 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4848 ivar |= (vector << (8 * (entry & 0x3)));
4849 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4852 case ixgbe_mac_82599EB:
4853 case ixgbe_mac_X540:
4854 if (type == -1) { /* MISC IVAR */
4855 index = (entry & 1) * 8;
4856 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4857 ivar &= ~(0xFF << index);
4858 ivar |= (vector << index);
4859 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4860 } else { /* RX/TX IVARS */
4861 index = (16 * (entry & 1)) + (8 * type);
4862 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4863 ivar &= ~(0xFF << index);
4864 ivar |= (vector << index);
4865 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4874 ixgbe_configure_ivars(struct adapter *adapter)
4876 struct ix_queue *que = adapter->queues;
4878 for (int i = 0; i < adapter->num_queues; i++, que++) {
4879 /* First the RX queue entry */
4880 ixgbe_set_ivar(adapter, i, que->msix, 0);
4881 /* ... and the TX */
4882 ixgbe_set_ivar(adapter, i, que->msix, 1);
4883 /* Set an Initial EITR value */
4884 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4887 /* For the Link interrupt */
4888 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4892 ** ixgbe_sfp_probe - called in the local timer to
4893 ** determine if a port had optics inserted.
4895 static bool ixgbe_sfp_probe(struct adapter *adapter)
4897 struct ixgbe_hw *hw = &adapter->hw;
4898 device_t dev = adapter->dev;
4899 bool result = FALSE;
4901 if ((hw->phy.type == ixgbe_phy_nl) &&
4902 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4903 s32 ret = hw->phy.ops.identify_sfp(hw);
4906 ret = hw->phy.ops.reset(hw);
4907 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4908 device_printf(dev,"Unsupported SFP+ module detected!");
4909 kprintf(" Reload driver with supported module.\n");
4910 adapter->sfp_probe = FALSE;
4913 device_printf(dev,"SFP+ module detected!\n");
4914 /* We now have supported optics */
4915 adapter->sfp_probe = FALSE;
4916 /* Set the optics type so system reports correctly */
4917 ixgbe_setup_optics(adapter);
4925 ** Tasklet for handling SFP module interrupts
4928 ixgbe_handle_mod(void *context, int pending)
4930 struct adapter *adapter = context;
4931 struct ixgbe_hw *hw = &adapter->hw;
4932 device_t dev = adapter->dev;
4935 err = hw->phy.ops.identify_sfp(hw);
4936 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4938 "Unsupported SFP+ module type was detected.\n");
4941 err = hw->mac.ops.setup_sfp(hw);
4942 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4944 "Setup failure - unsupported SFP+ module type.\n");
4947 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4953 ** Tasklet for handling MSF (multispeed fiber) interrupts
4956 ixgbe_handle_msf(void *context, int pending)
4958 struct adapter *adapter = context;
4959 struct ixgbe_hw *hw = &adapter->hw;
4963 autoneg = hw->phy.autoneg_advertised;
4964 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4965 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4966 if (hw->mac.ops.setup_link)
4967 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4973 ** Tasklet for reinitializing the Flow Director filter table
4976 ixgbe_reinit_fdir(void *context, int pending)
4978 struct adapter *adapter = context;
4979 struct ifnet *ifp = adapter->ifp;
4981 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4983 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4984 adapter->fdir_reinit = 0;
4985 /* re-enable flow director interrupts */
4986 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4987 /* Restart the interface */
4988 ifp->if_drv_flags |= IFF_DRV_RUNNING;
4993 /**********************************************************************
4995 * Update the board statistics counters.
4997 **********************************************************************/
4999 ixgbe_update_stats_counters(struct adapter *adapter)
5001 struct ifnet *ifp = adapter->ifp;
5002 struct ixgbe_hw *hw = &adapter->hw;
5003 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5004 u64 total_missed_rx = 0;
5006 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5007 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5008 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5009 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5011 for (int i = 0; i < 8; i++) {
5013 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5014 /* missed_rx tallies misses for the gprc workaround */
5016 /* global total per queue */
5017 adapter->stats.mpc[i] += mp;
5018 /* Running comprehensive total for stats display */
5019 total_missed_rx += adapter->stats.mpc[i];
5020 if (hw->mac.type == ixgbe_mac_82598EB)
5021 adapter->stats.rnbc[i] +=
5022 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5023 adapter->stats.pxontxc[i] +=
5024 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5025 adapter->stats.pxonrxc[i] +=
5026 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5027 adapter->stats.pxofftxc[i] +=
5028 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5029 adapter->stats.pxoffrxc[i] +=
5030 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5031 adapter->stats.pxon2offc[i] +=
5032 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5034 for (int i = 0; i < 16; i++) {
5035 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5036 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5037 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5038 adapter->stats.qbrc[i] +=
5039 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5040 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5041 adapter->stats.qbtc[i] +=
5042 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5043 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5045 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5046 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5047 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5049 /* Hardware workaround, gprc counts missed packets */
5050 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5051 adapter->stats.gprc -= missed_rx;
5053 if (hw->mac.type != ixgbe_mac_82598EB) {
5054 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5055 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5056 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5057 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5058 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5059 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5060 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5061 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5063 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5064 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5065 /* 82598 only has a counter in the high register */
5066 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5067 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5068 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5072 * Workaround: mprc hardware is incorrectly counting
5073 * broadcasts, so for now we subtract those.
5075 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5076 adapter->stats.bprc += bprc;
5077 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5078 if (hw->mac.type == ixgbe_mac_82598EB)
5079 adapter->stats.mprc -= bprc;
5081 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5082 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5083 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5084 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5085 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5086 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5088 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5089 adapter->stats.lxontxc += lxon;
5090 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5091 adapter->stats.lxofftxc += lxoff;
5092 total = lxon + lxoff;
5094 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5095 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5096 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5097 adapter->stats.gptc -= total;
5098 adapter->stats.mptc -= total;
5099 adapter->stats.ptc64 -= total;
5100 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5102 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5103 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5104 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5105 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5106 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5107 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5108 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5109 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5110 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5111 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5112 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5113 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5114 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5115 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5116 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5117 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5118 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5119 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5120 /* Only read FCOE on 82599 */
5121 if (hw->mac.type != ixgbe_mac_82598EB) {
5122 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5123 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5124 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5125 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5126 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5129 /* Fill out the OS statistics structure */
5130 ifp->if_ipackets = adapter->stats.gprc;
5131 ifp->if_opackets = adapter->stats.gptc;
5132 ifp->if_ibytes = adapter->stats.gorc;
5133 ifp->if_obytes = adapter->stats.gotc;
5134 ifp->if_imcasts = adapter->stats.mprc;
5135 ifp->if_collisions = 0;
5138 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5139 adapter->stats.rlec;
5142 /** ixgbe_sysctl_tdh_handler - Handler function
5143 * Retrieves the TDH value from the hardware
5146 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5150 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5153 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5154 error = sysctl_handle_int(oidp, &val, 0, req);
5155 if (error || !req->newptr)
5160 /** ixgbe_sysctl_tdt_handler - Handler function
5161 * Retrieves the TDT value from the hardware
5164 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5168 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5171 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5172 error = sysctl_handle_int(oidp, &val, 0, req);
5173 if (error || !req->newptr)
5178 /** ixgbe_sysctl_rdh_handler - Handler function
5179 * Retrieves the RDH value from the hardware
5182 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5186 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5189 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5190 error = sysctl_handle_int(oidp, &val, 0, req);
5191 if (error || !req->newptr)
5196 /** ixgbe_sysctl_rdt_handler - Handler function
5197 * Retrieves the RDT value from the hardware
5200 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5204 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5207 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5208 error = sysctl_handle_int(oidp, &val, 0, req);
5209 if (error || !req->newptr)
5215 * Add sysctl variables, one per statistic, to the system.
5218 ixgbe_add_hw_stats(struct adapter *adapter)
5220 struct tx_ring *txr = adapter->tx_rings;
5221 struct rx_ring *rxr = adapter->rx_rings;
5223 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5224 struct sysctl_oid *tree = adapter->sysctl_tree;
5225 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5226 struct ixgbe_hw_stats *stats = &adapter->stats;
5228 struct sysctl_oid *stat_node, *queue_node;
5229 struct sysctl_oid_list *stat_list, *queue_list;
5231 #define QUEUE_NAME_LEN 32
5232 char namebuf[QUEUE_NAME_LEN];
5234 /* Driver Statistics */
5235 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5236 CTLFLAG_RD, &adapter->dropped_pkts,
5237 "Driver dropped packets");
5238 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5239 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5240 "m_defrag() failed");
5241 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5242 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5243 "Driver tx dma failure in xmit");
5244 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5245 CTLFLAG_RD, &adapter->watchdog_events,
5246 "Watchdog timeouts");
5247 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5248 CTLFLAG_RD, &adapter->tso_tx,
5250 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5251 CTLFLAG_RD, &adapter->link_irq,
5252 "Link MSIX IRQ Handled");
5254 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5255 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5256 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5257 CTLFLAG_RD, NULL, "Queue Name");
5258 queue_list = SYSCTL_CHILDREN(queue_node);
5259 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5260 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5261 "irqs on this queue");
5262 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5263 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5264 ixgbe_sysctl_tdh_handler, "IU",
5265 "Transmit Descriptor Head");
5266 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5267 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5268 ixgbe_sysctl_tdt_handler, "IU",
5269 "Transmit Descriptor Tail");
5270 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5271 CTLFLAG_RD, &txr->no_desc_avail, 0,
5272 "Queue No Descriptor Available");
5273 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5274 CTLFLAG_RD, &txr->total_packets, 0,
5275 "Queue Packets Transmitted");
5278 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5279 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5280 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5281 CTLFLAG_RD, NULL, "Queue Name");
5282 queue_list = SYSCTL_CHILDREN(queue_node);
5285 struct lro_ctrl *lro = &rxr->lro;
5288 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5289 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5290 CTLFLAG_RD, NULL, "Queue Name");
5291 queue_list = SYSCTL_CHILDREN(queue_node);
5293 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5294 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5295 ixgbe_sysctl_rdh_handler, "IU",
5296 "Receive Descriptor Head");
5297 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5298 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5299 ixgbe_sysctl_rdt_handler, "IU",
5300 "Receive Descriptor Tail");
5301 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5302 CTLFLAG_RD, &rxr->rx_packets, 0,
5303 "Queue Packets Received");
5304 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5305 CTLFLAG_RD, &rxr->rx_bytes, 0,
5306 "Queue Bytes Received");
5308 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5309 CTLFLAG_RD, &lro->lro_queued, 0,
5311 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5312 CTLFLAG_RD, &lro->lro_flushed, 0,
5317 /* MAC stats get the own sub node */
5319 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5320 CTLFLAG_RD, NULL, "MAC Statistics");
5321 stat_list = SYSCTL_CHILDREN(stat_node);
5323 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5324 CTLFLAG_RD, &stats->crcerrs, 0,
5326 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5327 CTLFLAG_RD, &stats->illerrc, 0,
5328 "Illegal Byte Errors");
5329 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5330 CTLFLAG_RD, &stats->errbc, 0,
5332 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5333 CTLFLAG_RD, &stats->mspdc, 0,
5334 "MAC Short Packets Discarded");
5335 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5336 CTLFLAG_RD, &stats->mlfc, 0,
5337 "MAC Local Faults");
5338 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5339 CTLFLAG_RD, &stats->mrfc, 0,
5340 "MAC Remote Faults");
5341 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5342 CTLFLAG_RD, &stats->rlec, 0,
5343 "Receive Length Errors");
5344 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5345 CTLFLAG_RD, &stats->lxontxc, 0,
5346 "Link XON Transmitted");
5347 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5348 CTLFLAG_RD, &stats->lxonrxc, 0,
5349 "Link XON Received");
5350 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5351 CTLFLAG_RD, &stats->lxofftxc, 0,
5352 "Link XOFF Transmitted");
5353 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5354 CTLFLAG_RD, &stats->lxoffrxc, 0,
5355 "Link XOFF Received");
5357 /* Packet Reception Stats */
5358 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5359 CTLFLAG_RD, &stats->tor, 0,
5360 "Total Octets Received");
5361 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5362 CTLFLAG_RD, &stats->gorc, 0,
5363 "Good Octets Received");
5364 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5365 CTLFLAG_RD, &stats->tpr, 0,
5366 "Total Packets Received");
5367 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5368 CTLFLAG_RD, &stats->gprc, 0,
5369 "Good Packets Received");
5370 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5371 CTLFLAG_RD, &stats->mprc, 0,
5372 "Multicast Packets Received");
5373 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5374 CTLFLAG_RD, &stats->bprc, 0,
5375 "Broadcast Packets Received");
5376 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5377 CTLFLAG_RD, &stats->prc64, 0,
5378 "64 byte frames received ");
5379 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5380 CTLFLAG_RD, &stats->prc127, 0,
5381 "65-127 byte frames received");
5382 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5383 CTLFLAG_RD, &stats->prc255, 0,
5384 "128-255 byte frames received");
5385 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5386 CTLFLAG_RD, &stats->prc511, 0,
5387 "256-511 byte frames received");
5388 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5389 CTLFLAG_RD, &stats->prc1023, 0,
5390 "512-1023 byte frames received");
5391 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5392 CTLFLAG_RD, &stats->prc1522, 0,
5393 "1023-1522 byte frames received");
5394 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5395 CTLFLAG_RD, &stats->ruc, 0,
5396 "Receive Undersized");
5397 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5398 CTLFLAG_RD, &stats->rfc, 0,
5399 "Fragmented Packets Received ");
5400 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5401 CTLFLAG_RD, &stats->roc, 0,
5402 "Oversized Packets Received");
5403 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5404 CTLFLAG_RD, &stats->rjc, 0,
5406 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5407 CTLFLAG_RD, &stats->mngprc, 0,
5408 "Management Packets Received");
5409 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5410 CTLFLAG_RD, &stats->mngptc, 0,
5411 "Management Packets Dropped");
5412 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5413 CTLFLAG_RD, &stats->xec, 0,
5416 /* Packet Transmission Stats */
5417 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5418 CTLFLAG_RD, &stats->gotc, 0,
5419 "Good Octets Transmitted");
5420 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5421 CTLFLAG_RD, &stats->tpt, 0,
5422 "Total Packets Transmitted");
5423 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5424 CTLFLAG_RD, &stats->gptc, 0,
5425 "Good Packets Transmitted");
5426 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5427 CTLFLAG_RD, &stats->bptc, 0,
5428 "Broadcast Packets Transmitted");
5429 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5430 CTLFLAG_RD, &stats->mptc, 0,
5431 "Multicast Packets Transmitted");
5432 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5433 CTLFLAG_RD, &stats->mngptc, 0,
5434 "Management Packets Transmitted");
5435 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5436 CTLFLAG_RD, &stats->ptc64, 0,
5437 "64 byte frames transmitted ");
5438 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5439 CTLFLAG_RD, &stats->ptc127, 0,
5440 "65-127 byte frames transmitted");
5441 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5442 CTLFLAG_RD, &stats->ptc255, 0,
5443 "128-255 byte frames transmitted");
5444 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5445 CTLFLAG_RD, &stats->ptc511, 0,
5446 "256-511 byte frames transmitted");
5447 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5448 CTLFLAG_RD, &stats->ptc1023, 0,
5449 "512-1023 byte frames transmitted");
5450 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5451 CTLFLAG_RD, &stats->ptc1522, 0,
5452 "1024-1522 byte frames transmitted");
5455 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5456 CTLFLAG_RD, &stats->fccrc, 0,
5458 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5459 CTLFLAG_RD, &stats->fclast, 0,
5461 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5462 CTLFLAG_RD, &stats->fcoerpdc, 0,
5463 "FCoE Packets Dropped");
5464 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5465 CTLFLAG_RD, &stats->fcoeprc, 0,
5466 "FCoE Packets Received");
5467 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5468 CTLFLAG_RD, &stats->fcoeptc, 0,
5469 "FCoE Packets Transmitted");
5470 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5471 CTLFLAG_RD, &stats->fcoedwrc, 0,
5472 "FCoE DWords Received");
5473 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5474 CTLFLAG_RD, &stats->fcoedwtc, 0,
5475 "FCoE DWords Transmitted");
5479 ** Set flow control using sysctl:
5480 ** Flow control values:
5487 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5490 struct adapter *adapter = (struct adapter *) arg1;
5493 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5494 if ((error) || (req->newptr == NULL))
5497 /* Don't bother if it's not changed */
5498 if (adapter->fc == last)
5501 switch (adapter->fc) {
5502 case ixgbe_fc_rx_pause:
5503 case ixgbe_fc_tx_pause:
5505 adapter->hw.fc.requested_mode = adapter->fc;
5509 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5511 /* Don't autoneg if forcing a value */
5512 adapter->hw.fc.disable_fc_autoneg = TRUE;
5513 ixgbe_fc_enable(&adapter->hw);
5518 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5519 const char *description, int *limit, int value)
5522 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5523 SYSCTL_CHILDREN(adapter->sysctl_tree),
5524 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5528 ** Control link advertise speed:
5529 ** 1 - advertise only 1G
5530 ** 2 - advertise 100Mb
5531 ** 3 - advertise normal
5534 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5537 struct adapter *adapter;
5539 struct ixgbe_hw *hw;
5540 ixgbe_link_speed speed, last;
5542 adapter = (struct adapter *) arg1;
5545 last = adapter->advertise;
5547 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5548 if ((error) || (adapter->advertise == -1))
5551 if (adapter->advertise == last) /* no change */
5554 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5555 (hw->phy.multispeed_fiber)))
5558 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5559 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5563 if (adapter->advertise == 1)
5564 speed = IXGBE_LINK_SPEED_1GB_FULL;
5565 else if (adapter->advertise == 2)
5566 speed = IXGBE_LINK_SPEED_100_FULL;
5567 else if (adapter->advertise == 3)
5568 speed = IXGBE_LINK_SPEED_1GB_FULL |
5569 IXGBE_LINK_SPEED_10GB_FULL;
5570 else /* bogus value */
5573 hw->mac.autotry_restart = TRUE;
5574 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5580 ** Thermal Shutdown Trigger
5581 ** - cause a Thermal Overtemp IRQ
5584 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5586 int error, fire = 0;
5587 struct adapter *adapter = (struct adapter *) arg1;
5588 struct ixgbe_hw *hw = &adapter->hw;
5591 if (hw->mac.type != ixgbe_mac_X540)
5594 error = sysctl_handle_int(oidp, &fire, 0, req);
5595 if ((error) || (req->newptr == NULL))
5599 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5600 reg |= IXGBE_EICR_TS;
5601 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5608 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5612 /* convert rate in intr/s to hw representation */
5614 eitr = 1000000 / rate;
5615 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5618 /* Don't disable it */
5619 eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5620 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5621 /* Don't allow it to be too large */
5622 eitr = IXGBE_EITR_INTVL_MASK;
5625 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5629 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5631 struct adapter *sc = (void *)arg1;
5632 struct ifnet *ifp = sc->ifp;
5633 int error, intr_rate, running;
5634 struct ix_queue *que = sc->queues;
5636 intr_rate = sc->intr_rate;
5637 error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5638 if (error || req->newptr == NULL)
5643 ifnet_serialize_all(ifp);
5645 sc->intr_rate = intr_rate;
5646 running = ifp->if_flags & IFF_RUNNING;
5648 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5650 if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5651 for (int i = 0; i < sc->num_queues; i++, que++)
5652 ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5656 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5658 ifnet_deserialize_all(ifp);
5663 /* rearrange mbuf chain to get contiguous bytes */
5665 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5667 int hoff, iphlen, thoff;
5671 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5673 iphlen = m->m_pkthdr.csum_iphlen;
5674 thoff = m->m_pkthdr.csum_thlen;
5675 hoff = m->m_pkthdr.csum_lhlen;
5677 KASSERT(iphlen > 0, ("invalid ip hlen"));
5678 KASSERT(thoff > 0, ("invalid tcp hlen"));
5679 KASSERT(hoff > 0, ("invalid ether hlen"));
5681 if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5682 m = m_pullup(m, hoff + iphlen + thoff);