1 /******************************************************************************
3 Copyright (c) 2001-2012, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
36 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 static void ixgbe_start(struct ifnet *, struct ifaltq_subque *);
106 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111 static void ixgbe_qflush(struct ifnet *);
113 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void ixgbe_init(void *);
115 static void ixgbe_init_locked(struct adapter *);
116 static void ixgbe_stop(void *);
117 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int ixgbe_media_change(struct ifnet *);
119 static void ixgbe_identify_hardware(struct adapter *);
120 static int ixgbe_allocate_pci_resources(struct adapter *);
121 static int ixgbe_allocate_msix(struct adapter *);
122 static int ixgbe_allocate_legacy(struct adapter *);
123 static int ixgbe_allocate_queues(struct adapter *);
124 static int ixgbe_setup_msix(struct adapter *);
125 static void ixgbe_free_pci_resources(struct adapter *);
126 static void ixgbe_local_timer(void *);
127 static int ixgbe_setup_interface(device_t, struct adapter *);
128 static void ixgbe_config_link(struct adapter *);
130 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
131 static int ixgbe_setup_transmit_structures(struct adapter *);
132 static void ixgbe_setup_transmit_ring(struct tx_ring *);
133 static void ixgbe_initialize_transmit_units(struct adapter *);
134 static void ixgbe_free_transmit_structures(struct adapter *);
135 static void ixgbe_free_transmit_buffers(struct tx_ring *);
137 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
138 static int ixgbe_setup_receive_structures(struct adapter *);
139 static int ixgbe_setup_receive_ring(struct rx_ring *);
140 static void ixgbe_initialize_receive_units(struct adapter *);
141 static void ixgbe_free_receive_structures(struct adapter *);
142 static void ixgbe_free_receive_buffers(struct rx_ring *);
144 static void ixgbe_setup_hw_rsc(struct rx_ring *);
147 static void ixgbe_enable_intr(struct adapter *);
148 static void ixgbe_disable_intr(struct adapter *);
149 static void ixgbe_update_stats_counters(struct adapter *);
150 static void ixgbe_txeof(struct tx_ring *);
151 static void ixgbe_rxeof(struct ix_queue *, int);
152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void ixgbe_set_promisc(struct adapter *);
154 static void ixgbe_set_multi(struct adapter *);
155 static void ixgbe_update_link_status(struct adapter *);
156 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
162 struct ixgbe_dma_alloc *, int);
163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
165 const char *, int *, int);
166 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
167 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
168 static int ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
169 static void ixgbe_add_sysctl(struct adapter *);
170 static void ixgbe_set_eitr(struct adapter *, int, int);
171 static int ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
172 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void ixgbe_configure_ivars(struct adapter *);
174 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
176 static void ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
180 static void ixgbe_add_hw_stats(struct adapter *adapter);
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
186 /* Support for pluggable optic modules */
187 static bool ixgbe_sfp_probe(struct adapter *);
188 static void ixgbe_setup_optics(struct adapter *);
190 /* Legacy (single vector interrupt handler */
191 static void ixgbe_legacy_irq(void *);
193 /* The MSI/X Interrupt handlers */
194 static void ixgbe_msix_que(void *);
195 static void ixgbe_msix_link(void *);
197 /* Deferred interrupt tasklets */
198 static void ixgbe_handle_msf(void *, int);
199 static void ixgbe_handle_mod(void *, int);
202 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
203 static void ixgbe_reinit_fdir(void *, int);
206 /*********************************************************************
207 * FreeBSD Device Interface Entry Points
208 *********************************************************************/
210 static device_method_t ixgbe_methods[] = {
211 /* Device interface */
212 DEVMETHOD(device_probe, ixgbe_probe),
213 DEVMETHOD(device_attach, ixgbe_attach),
214 DEVMETHOD(device_detach, ixgbe_detach),
215 DEVMETHOD(device_shutdown, ixgbe_shutdown),
219 static driver_t ixgbe_driver = {
220 "ix", ixgbe_methods, sizeof(struct adapter),
223 devclass_t ixgbe_devclass;
224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230 ** TUNEABLE PARAMETERS:
233 /* How many packets rxeof tries to clean at a time */
234 static int ixgbe_rx_process_limit = 128;
235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
238 ** Smart speed setting, default to on
239 ** this only works as a compile option
240 ** right now as its during attach, set
241 ** this to 'ixgbe_smart_speed_off' to
244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
246 static int ixgbe_msi_enable = 1;
247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
250 * MSIX should be the default for best performance,
251 * but this allows it to be forced off for testing.
253 static int ixgbe_enable_msix = 1;
254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
257 * Header split: this causes the hardware to DMA
258 * the header into a separate mbuf from the payload,
259 * it can be a performance win in some workloads, but
260 * in others it actually hurts, its off by default.
262 static int ixgbe_header_split = FALSE;
263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
266 * Number of Queues, can be set to 0,
267 * it then autoconfigures based on the
268 * number of cpus with a max of 8. This
269 * can be overriden manually here.
271 static int ixgbe_num_queues = 0;
272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
275 ** Number of TX descriptors per ring,
276 ** setting higher than RX as this seems
277 ** the better performing choice.
279 static int ixgbe_txd = PERFORM_TXD;
280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
282 /* Number of RX descriptors per ring */
283 static int ixgbe_rxd = PERFORM_RXD;
284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
286 /* Keep running tab on them for sanity check */
287 static int ixgbe_total_ports;
291 ** For Flow Director: this is the
292 ** number of TX packets we sample
293 ** for the filter pool, this means
294 ** every 20th packet will be probed.
296 ** This feature can be disabled by
297 ** setting this to 0.
299 static int atr_sample_rate = 20;
301 ** Flow Director actually 'steals'
302 ** part of the packet buffer as its
303 ** filter pool, this variable controls
305 ** 0 = 64K, 1 = 128K, 2 = 256K
307 static int fdir_pballoc = 1;
312 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
313 * be a reference on how to implement netmap support in a driver.
314 * Additional comments are in ixgbe_netmap.h .
316 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
317 * that extend the standard driver.
319 #include <dev/netmap/ixgbe_netmap.h>
320 #endif /* DEV_NETMAP */
322 /*********************************************************************
323 * Device identification routine
325 * ixgbe_probe determines if the driver should be loaded on
326 * adapter based on PCI vendor/device id of the adapter.
328 * return BUS_PROBE_DEFAULT on success, positive on failure
329 *********************************************************************/
332 ixgbe_probe(device_t dev)
334 ixgbe_vendor_info_t *ent;
336 u16 pci_vendor_id = 0;
337 u16 pci_device_id = 0;
338 u16 pci_subvendor_id = 0;
339 u16 pci_subdevice_id = 0;
340 char adapter_name[256];
342 INIT_DEBUGOUT("ixgbe_probe: begin");
344 pci_vendor_id = pci_get_vendor(dev);
345 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
348 pci_device_id = pci_get_device(dev);
349 pci_subvendor_id = pci_get_subvendor(dev);
350 pci_subdevice_id = pci_get_subdevice(dev);
352 ent = ixgbe_vendor_info_array;
353 while (ent->vendor_id != 0) {
354 if ((pci_vendor_id == ent->vendor_id) &&
355 (pci_device_id == ent->device_id) &&
357 ((pci_subvendor_id == ent->subvendor_id) ||
358 (ent->subvendor_id == 0)) &&
360 ((pci_subdevice_id == ent->subdevice_id) ||
361 (ent->subdevice_id == 0))) {
362 ksprintf(adapter_name, "%s, Version - %s",
363 ixgbe_strings[ent->index],
364 ixgbe_driver_version);
365 device_set_desc_copy(dev, adapter_name);
367 return (BUS_PROBE_DEFAULT);
374 /*********************************************************************
375 * Device initialization routine
377 * The attach entry point is called when the driver is being loaded.
378 * This routine identifies the type of hardware, allocates all resources
379 * and initializes the hardware.
381 * return 0 on success, positive on failure
382 *********************************************************************/
385 ixgbe_attach(device_t dev)
387 struct adapter *adapter;
393 INIT_DEBUGOUT("ixgbe_attach: begin");
395 if (resource_disabled("ixgbe", device_get_unit(dev))) {
396 device_printf(dev, "Disabled by device hint\n");
400 /* Allocate, clear, and link in our adapter structure */
401 adapter = device_get_softc(dev);
402 adapter->dev = adapter->osdep.dev = dev;
406 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
408 /* Set up the timer callout */
409 callout_init_mp(&adapter->timer);
411 /* Determine hardware revision */
412 ixgbe_identify_hardware(adapter);
414 /* Enable bus mastering */
415 pci_enable_busmaster(dev);
417 /* Do base PCI setup - map BAR0 */
418 if (ixgbe_allocate_pci_resources(adapter)) {
419 device_printf(dev, "Allocation of PCI resources failed\n");
424 /* Do descriptor calc and sanity checks */
425 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
426 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
427 device_printf(dev, "TXD config issue, using default!\n");
428 adapter->num_tx_desc = DEFAULT_TXD;
430 adapter->num_tx_desc = ixgbe_txd;
433 ** With many RX rings it is easy to exceed the
434 ** system mbuf allocation. Tuning nmbclusters
435 ** can alleviate this.
437 if (nmbclusters > 0 ) {
439 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
440 if (s > nmbclusters) {
441 device_printf(dev, "RX Descriptors exceed "
442 "system mbuf max, using default instead!\n");
443 ixgbe_rxd = DEFAULT_RXD;
447 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
448 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
449 device_printf(dev, "RXD config issue, using default!\n");
450 adapter->num_rx_desc = DEFAULT_RXD;
452 adapter->num_rx_desc = ixgbe_rxd;
454 /* Allocate our TX/RX Queues */
455 if (ixgbe_allocate_queues(adapter)) {
460 /* Allocate multicast array memory. */
461 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
462 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
463 if (adapter->mta == NULL) {
464 device_printf(dev, "Can not allocate multicast setup array\n");
469 /* Initialize the shared code */
470 error = ixgbe_init_shared_code(hw);
471 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
473 ** No optics in this port, set up
474 ** so the timer routine will probe
475 ** for later insertion.
477 adapter->sfp_probe = TRUE;
479 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
480 device_printf(dev,"Unsupported SFP+ module detected!\n");
484 device_printf(dev,"Unable to initialize the shared code\n");
489 /* Make sure we have a good EEPROM before we read from it */
490 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
491 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
496 error = ixgbe_init_hw(hw);
498 case IXGBE_ERR_EEPROM_VERSION:
499 device_printf(dev, "This device is a pre-production adapter/"
500 "LOM. Please be aware there may be issues associated "
501 "with your hardware.\n If you are experiencing problems "
502 "please contact your Intel or hardware representative "
503 "who provided you with this hardware.\n");
505 case IXGBE_ERR_SFP_NOT_SUPPORTED:
506 device_printf(dev,"Unsupported SFP+ Module\n");
508 device_printf(dev,"Hardware Initialization Failure\n");
510 case IXGBE_ERR_SFP_NOT_PRESENT:
511 device_printf(dev,"No SFP+ Module found\n");
517 /* Detect and set physical type */
518 ixgbe_setup_optics(adapter);
520 if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
521 adapter->intr_type = PCI_INTR_TYPE_MSIX;
522 error = ixgbe_allocate_msix(adapter);
524 error = ixgbe_allocate_legacy(adapter);
529 /* Setup OS specific network interface */
530 if (ixgbe_setup_interface(dev, adapter) != 0)
533 /* Add sysctl tree */
534 ixgbe_add_sysctl(adapter);
536 /* Initialize statistics */
537 ixgbe_update_stats_counters(adapter);
539 /* Register for VLAN events */
540 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
541 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
542 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
543 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
545 /* Print PCIE bus type/speed/width info */
546 ixgbe_get_bus_info(hw);
547 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
548 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
549 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
550 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
551 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
552 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
555 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
556 (hw->bus.speed == ixgbe_bus_speed_2500)) {
557 device_printf(dev, "PCI-Express bandwidth available"
558 " for this card\n is not sufficient for"
559 " optimal performance.\n");
560 device_printf(dev, "For optimal performance a x8 "
561 "PCIE, or x4 PCIE 2 slot is required.\n");
564 /* let hardware know driver is loaded */
565 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
566 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
567 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
569 ixgbe_add_hw_stats(adapter);
572 ixgbe_netmap_attach(adapter);
573 #endif /* DEV_NETMAP */
574 INIT_DEBUGOUT("ixgbe_attach: end");
577 ixgbe_free_transmit_structures(adapter);
578 ixgbe_free_receive_structures(adapter);
580 if (adapter->ifp != NULL)
581 if_free(adapter->ifp);
582 ixgbe_free_pci_resources(adapter);
583 kfree(adapter->mta, M_DEVBUF);
588 /*********************************************************************
589 * Device removal routine
591 * The detach entry point is called when the driver is being removed.
592 * This routine stops the adapter and deallocates all the resources
593 * that were allocated for driver operation.
595 * return 0 on success, positive on failure
596 *********************************************************************/
599 ixgbe_detach(device_t dev)
601 struct adapter *adapter = device_get_softc(dev);
604 INIT_DEBUGOUT("ixgbe_detach: begin");
606 /* Make sure VLANS are not using driver */
607 if (adapter->ifp->if_vlantrunks != NULL) {
608 device_printf(dev,"Vlan in use, detach first\n");
612 IXGBE_CORE_LOCK(adapter);
614 IXGBE_CORE_UNLOCK(adapter);
616 /* Drain the Link queue */
618 taskqueue_drain(adapter->tq, &adapter->mod_task);
619 taskqueue_drain(adapter->tq, &adapter->msf_task);
621 taskqueue_drain(adapter->tq, &adapter->fdir_task);
623 taskqueue_free(adapter->tq);
626 /* let hardware know driver is unloading */
627 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
628 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
629 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
631 /* Unregister VLAN events */
632 if (adapter->vlan_attach != NULL)
633 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
634 if (adapter->vlan_detach != NULL)
635 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
637 ether_ifdetach(adapter->ifp);
638 callout_stop(&adapter->timer);
640 netmap_detach(adapter->ifp);
641 #endif /* DEV_NETMAP */
642 ixgbe_free_pci_resources(adapter);
643 bus_generic_detach(dev);
644 if_free(adapter->ifp);
646 ixgbe_free_transmit_structures(adapter);
647 ixgbe_free_receive_structures(adapter);
648 kfree(adapter->mta, M_DEVBUF);
649 sysctl_ctx_free(&adapter->sysctl_ctx);
651 IXGBE_CORE_LOCK_DESTROY(adapter);
655 /*********************************************************************
657 * Shutdown entry point
659 **********************************************************************/
662 ixgbe_shutdown(device_t dev)
664 struct adapter *adapter = device_get_softc(dev);
665 IXGBE_CORE_LOCK(adapter);
667 IXGBE_CORE_UNLOCK(adapter);
672 /*********************************************************************
673 * Transmit entry point
675 * ixgbe_start is called by the stack to initiate a transmit.
676 * The driver will remain in this routine as long as there are
677 * packets to transmit and transmit resources are available.
678 * In case resources are not available stack is notified and
679 * the packet is requeued.
680 **********************************************************************/
683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
686 struct adapter *adapter = txr->adapter;
688 IXGBE_TX_LOCK_ASSERT(txr);
690 if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
693 if (!adapter->link_active) {
694 ifq_purge(&ifp->if_snd);
698 while (!ifq_is_empty(&ifp->if_snd)) {
699 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
700 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
704 m_head = ifq_dequeue(&ifp->if_snd, NULL);
708 if (ixgbe_xmit(txr, &m_head)) {
709 #if 0 /* XXX: prepend to an ALTQ queue ? */
711 IF_PREPEND(&ifp->if_snd, m_head);
713 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
714 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
717 /* Send a copy of the frame to the BPF listener */
718 ETHER_BPF_MTAP(ifp, m_head);
720 /* Set watchdog on */
721 txr->watchdog_time = ticks;
722 txr->queue_status = IXGBE_QUEUE_WORKING;
729 * Legacy TX start - called by the stack, this
730 * always uses the first tx ring, and should
731 * not be used with multiqueue tx enabled.
734 ixgbe_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
736 struct adapter *adapter = ifp->if_softc;
737 struct tx_ring *txr = adapter->tx_rings;
739 ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
741 if (ifp->if_flags & IFF_RUNNING) {
743 ixgbe_start_locked(txr, ifp);
744 IXGBE_TX_UNLOCK(txr);
749 #if 0 /* __FreeBSD_version >= 800000 */
751 ** Multiqueue Transmit driver
755 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
757 struct adapter *adapter = ifp->if_softc;
758 struct ix_queue *que;
762 /* Which queue to use */
763 if ((m->m_flags & M_FLOWID) != 0)
764 i = m->m_pkthdr.flowid % adapter->num_queues;
766 i = curcpu % adapter->num_queues;
768 txr = &adapter->tx_rings[i];
769 que = &adapter->queues[i];
771 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
772 IXGBE_TX_TRYLOCK(txr)) {
773 err = ixgbe_mq_start_locked(ifp, txr, m);
774 IXGBE_TX_UNLOCK(txr);
776 err = drbr_enqueue(ifp, txr->br, m);
777 taskqueue_enqueue(que->tq, &que->que_task);
784 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
786 struct adapter *adapter = txr->adapter;
788 int enqueued, err = 0;
790 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
791 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
792 adapter->link_active == 0) {
794 err = drbr_enqueue(ifp, txr->br, m);
800 next = drbr_dequeue(ifp, txr->br);
801 } else if (drbr_needs_enqueue(ifp, txr->br)) {
802 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
804 next = drbr_dequeue(ifp, txr->br);
808 /* Process the queue */
809 while (next != NULL) {
810 if ((err = ixgbe_xmit(txr, &next)) != 0) {
812 err = drbr_enqueue(ifp, txr->br, next);
816 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
817 /* Send a copy of the frame to the BPF listener */
818 ETHER_BPF_MTAP(ifp, next);
819 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
821 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
823 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
824 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
827 next = drbr_dequeue(ifp, txr->br);
831 /* Set watchdog on */
832 txr->queue_status |= IXGBE_QUEUE_WORKING;
833 txr->watchdog_time = ticks;
836 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
843 ** Flush all ring buffers
846 ixgbe_qflush(struct ifnet *ifp)
848 struct adapter *adapter = ifp->if_softc;
849 struct tx_ring *txr = adapter->tx_rings;
852 for (int i = 0; i < adapter->num_queues; i++, txr++) {
854 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
856 IXGBE_TX_UNLOCK(txr);
860 #endif /* __FreeBSD_version >= 800000 */
862 /*********************************************************************
865 * ixgbe_ioctl is called when the user wants to configure the
868 * return 0 on success, positive on failure
869 **********************************************************************/
872 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
874 struct adapter *adapter = ifp->if_softc;
875 struct ifreq *ifr = (struct ifreq *) data;
876 #if defined(INET) || defined(INET6)
877 struct ifaddr *ifa = (struct ifaddr *)data;
878 bool avoid_reset = FALSE;
886 if (ifa->ifa_addr->sa_family == AF_INET)
890 if (ifa->ifa_addr->sa_family == AF_INET6)
893 #if defined(INET) || defined(INET6)
895 ** Calling init results in link renegotiation,
896 ** so we avoid doing it when possible.
899 ifp->if_flags |= IFF_UP;
900 if (!(ifp->if_flags & IFF_RUNNING))
902 if (!(ifp->if_flags & IFF_NOARP))
903 arp_ifinit(ifp, ifa);
905 error = ether_ioctl(ifp, command, data);
909 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
910 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
913 IXGBE_CORE_LOCK(adapter);
914 ifp->if_mtu = ifr->ifr_mtu;
915 adapter->max_frame_size =
916 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
917 ixgbe_init_locked(adapter);
918 IXGBE_CORE_UNLOCK(adapter);
922 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
923 IXGBE_CORE_LOCK(adapter);
924 if (ifp->if_flags & IFF_UP) {
925 if ((ifp->if_flags & IFF_RUNNING)) {
926 if ((ifp->if_flags ^ adapter->if_flags) &
927 (IFF_PROMISC | IFF_ALLMULTI)) {
928 ixgbe_set_promisc(adapter);
931 ixgbe_init_locked(adapter);
933 if (ifp->if_flags & IFF_RUNNING)
935 adapter->if_flags = ifp->if_flags;
936 IXGBE_CORE_UNLOCK(adapter);
940 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
941 if (ifp->if_flags & IFF_RUNNING) {
942 IXGBE_CORE_LOCK(adapter);
943 ixgbe_disable_intr(adapter);
944 ixgbe_set_multi(adapter);
945 ixgbe_enable_intr(adapter);
946 IXGBE_CORE_UNLOCK(adapter);
951 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
952 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
956 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
957 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
958 if (mask & IFCAP_HWCSUM)
959 ifp->if_capenable ^= IFCAP_HWCSUM;
960 if (mask & IFCAP_TSO4)
961 ifp->if_capenable ^= IFCAP_TSO4;
962 if (mask & IFCAP_TSO6)
963 ifp->if_capenable ^= IFCAP_TSO6;
965 if (mask & IFCAP_LRO)
966 ifp->if_capenable ^= IFCAP_LRO;
968 if (mask & IFCAP_VLAN_HWTAGGING)
969 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
970 if (mask & IFCAP_VLAN_HWFILTER)
971 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
973 if (mask & IFCAP_VLAN_HWTSO)
974 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
976 if (ifp->if_flags & IFF_RUNNING) {
977 IXGBE_CORE_LOCK(adapter);
978 ixgbe_init_locked(adapter);
979 IXGBE_CORE_UNLOCK(adapter);
982 VLAN_CAPABILITIES(ifp);
988 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
989 error = ether_ioctl(ifp, command, data);
996 /*********************************************************************
999 * This routine is used in two ways. It is used by the stack as
1000 * init entry point in network interface structure. It is also used
1001 * by the driver as a hw/sw initialization routine to get to a
1004 * return 0 on success, positive on failure
1005 **********************************************************************/
1006 #define IXGBE_MHADD_MFS_SHIFT 16
1009 ixgbe_init_locked(struct adapter *adapter)
1011 struct ifnet *ifp = adapter->ifp;
1012 device_t dev = adapter->dev;
1013 struct ixgbe_hw *hw = &adapter->hw;
1014 u32 k, txdctl, mhadd, gpie;
1017 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1018 INIT_DEBUGOUT("ixgbe_init: begin");
1019 hw->adapter_stopped = FALSE;
1020 ixgbe_stop_adapter(hw);
1021 callout_stop(&adapter->timer);
1023 /* reprogram the RAR[0] in case user changed it. */
1024 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1026 /* Get the latest mac address, User can use a LAA */
1027 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1028 IXGBE_ETH_LENGTH_OF_ADDRESS);
1029 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1030 hw->addr_ctrl.rar_used_count = 1;
1032 /* Set the various hardware offload abilities */
1033 ifp->if_hwassist = 0;
1034 if (ifp->if_capenable & IFCAP_TSO)
1035 ifp->if_hwassist |= CSUM_TSO;
1036 if (ifp->if_capenable & IFCAP_TXCSUM) {
1037 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1039 if (hw->mac.type != ixgbe_mac_82598EB)
1040 ifp->if_hwassist |= CSUM_SCTP;
1044 /* Prepare transmit descriptors and buffers */
1045 if (ixgbe_setup_transmit_structures(adapter)) {
1046 device_printf(dev,"Could not setup transmit structures\n");
1047 ixgbe_stop(adapter);
1052 ixgbe_initialize_transmit_units(adapter);
1054 /* Setup Multicast table */
1055 ixgbe_set_multi(adapter);
1058 ** Determine the correct mbuf pool
1059 ** for doing jumbo/headersplit
1062 if (adapter->max_frame_size <= 2048)
1063 adapter->rx_mbuf_sz = MCLBYTES;
1064 else if (adapter->max_frame_size <= 4096)
1065 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1066 else if (adapter->max_frame_size <= 9216)
1067 adapter->rx_mbuf_sz = MJUM9BYTES;
1069 adapter->rx_mbuf_sz = MJUM16BYTES;
1071 adapter->rx_mbuf_sz = MCLBYTES;
1074 /* Prepare receive descriptors and buffers */
1075 if (ixgbe_setup_receive_structures(adapter)) {
1076 device_printf(dev,"Could not setup receive structures\n");
1077 ixgbe_stop(adapter);
1081 /* Configure RX settings */
1082 ixgbe_initialize_receive_units(adapter);
1084 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1086 /* Enable Fan Failure Interrupt */
1087 gpie |= IXGBE_SDP1_GPIEN;
1089 /* Add for Module detection */
1090 if (hw->mac.type == ixgbe_mac_82599EB)
1091 gpie |= IXGBE_SDP2_GPIEN;
1093 /* Thermal Failure Detection */
1094 if (hw->mac.type == ixgbe_mac_X540)
1095 gpie |= IXGBE_SDP0_GPIEN;
1097 if (adapter->msix > 1) {
1098 /* Enable Enhanced MSIX mode */
1099 gpie |= IXGBE_GPIE_MSIX_MODE;
1100 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1103 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1106 if (ifp->if_mtu > ETHERMTU) {
1107 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1108 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1109 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1110 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1113 /* Now enable all the queues */
1115 for (int i = 0; i < adapter->num_queues; i++) {
1116 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1117 txdctl |= IXGBE_TXDCTL_ENABLE;
1118 /* Set WTHRESH to 8, burst writeback */
1119 txdctl |= (8 << 16);
1121 * When the internal queue falls below PTHRESH (32),
1122 * start prefetching as long as there are at least
1123 * HTHRESH (1) buffers ready. The values are taken
1124 * from the Intel linux driver 3.8.21.
1125 * Prefetching enables tx line rate even with 1 queue.
1127 txdctl |= (32 << 0) | (1 << 8);
1128 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1131 for (int i = 0; i < adapter->num_queues; i++) {
1132 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1133 if (hw->mac.type == ixgbe_mac_82598EB) {
1139 rxdctl &= ~0x3FFFFF;
1142 rxdctl |= IXGBE_RXDCTL_ENABLE;
1143 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1144 for (k = 0; k < 10; k++) {
1145 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1146 IXGBE_RXDCTL_ENABLE)
1154 * In netmap mode, we must preserve the buffers made
1155 * available to userspace before the if_init()
1156 * (this is true by default on the TX side, because
1157 * init makes all buffers available to userspace).
1159 * netmap_reset() and the device specific routines
1160 * (e.g. ixgbe_setup_receive_rings()) map these
1161 * buffers at the end of the NIC ring, so here we
1162 * must set the RDT (tail) register to make sure
1163 * they are not overwritten.
1165 * In this driver the NIC ring starts at RDH = 0,
1166 * RDT points to the last slot available for reception (?),
1167 * so RDT = num_rx_desc - 1 means the whole ring is available.
1169 if (ifp->if_capenable & IFCAP_NETMAP) {
1170 struct netmap_adapter *na = NA(adapter->ifp);
1171 struct netmap_kring *kring = &na->rx_rings[i];
1172 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1174 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1176 #endif /* DEV_NETMAP */
1177 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1180 /* Set up VLAN support and filter */
1181 ixgbe_setup_vlan_hw_support(adapter);
1183 /* Enable Receive engine */
1184 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1185 if (hw->mac.type == ixgbe_mac_82598EB)
1186 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1187 rxctrl |= IXGBE_RXCTRL_RXEN;
1188 ixgbe_enable_rx_dma(hw, rxctrl);
1190 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1192 /* Set up MSI/X routing */
1193 if (ixgbe_enable_msix) {
1194 ixgbe_configure_ivars(adapter);
1195 /* Set up auto-mask */
1196 if (hw->mac.type == ixgbe_mac_82598EB)
1197 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1199 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1200 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1202 } else { /* Simple settings for Legacy/MSI */
1203 ixgbe_set_ivar(adapter, 0, 0, 0);
1204 ixgbe_set_ivar(adapter, 0, 0, 1);
1205 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1209 /* Init Flow director */
1210 if (hw->mac.type != ixgbe_mac_82598EB) {
1211 u32 hdrm = 32 << fdir_pballoc;
1213 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1214 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1219 ** Check on any SFP devices that
1220 ** need to be kick-started
1222 if (hw->phy.type == ixgbe_phy_none) {
1223 int err = hw->phy.ops.identify(hw);
1224 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1226 "Unsupported SFP+ module type was detected.\n");
1231 /* Set moderation on the Link interrupt */
1232 ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1234 /* Config/Enable Link */
1235 ixgbe_config_link(adapter);
1237 /* Hardware Packet Buffer & Flow Control setup */
1239 u32 rxpb, frame, size, tmp;
1241 frame = adapter->max_frame_size;
1243 /* Calculate High Water */
1244 if (hw->mac.type == ixgbe_mac_X540)
1245 tmp = IXGBE_DV_X540(frame, frame);
1247 tmp = IXGBE_DV(frame, frame);
1248 size = IXGBE_BT2KB(tmp);
1249 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1250 hw->fc.high_water[0] = rxpb - size;
1252 /* Now calculate Low Water */
1253 if (hw->mac.type == ixgbe_mac_X540)
1254 tmp = IXGBE_LOW_DV_X540(frame);
1256 tmp = IXGBE_LOW_DV(frame);
1257 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1259 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1260 hw->fc.pause_time = IXGBE_FC_PAUSE;
1261 hw->fc.send_xon = TRUE;
1263 /* Initialize the FC settings */
1266 /* And now turn on interrupts */
1267 ixgbe_enable_intr(adapter);
1269 /* Now inform the stack we're ready */
1270 ifp->if_flags |= IFF_RUNNING;
1271 ifq_clr_oactive(&ifp->if_snd);
1277 ixgbe_init(void *arg)
1279 struct adapter *adapter = arg;
1281 IXGBE_CORE_LOCK(adapter);
1282 ixgbe_init_locked(adapter);
1283 IXGBE_CORE_UNLOCK(adapter);
1290 ** MSIX Interrupt Handlers and Tasklets
1295 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1297 struct ixgbe_hw *hw = &adapter->hw;
1298 u64 queue = (u64)(1 << vector);
1301 if (hw->mac.type == ixgbe_mac_82598EB) {
1302 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1303 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1305 mask = (queue & 0xFFFFFFFF);
1307 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1308 mask = (queue >> 32);
1310 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1315 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1317 struct ixgbe_hw *hw = &adapter->hw;
1318 u64 queue = (u64)(1 << vector);
1321 if (hw->mac.type == ixgbe_mac_82598EB) {
1322 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1323 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1325 mask = (queue & 0xFFFFFFFF);
1327 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1328 mask = (queue >> 32);
1330 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1335 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1339 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1340 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1341 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1343 mask = (queues & 0xFFFFFFFF);
1344 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1345 mask = (queues >> 32);
1346 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1350 /*********************************************************************
1352 * Legacy Interrupt Service routine
1354 **********************************************************************/
1357 ixgbe_legacy_irq(void *arg)
1359 struct ix_queue *que = arg;
1360 struct adapter *adapter = que->adapter;
1361 struct ixgbe_hw *hw = &adapter->hw;
1362 struct tx_ring *txr = adapter->tx_rings;
1366 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1369 if (reg_eicr == 0) {
1370 ixgbe_enable_intr(adapter);
1374 ixgbe_rxeof(que, adapter->rx_process_limit);
1378 if (!ifq_is_empty(&adapter->ifp->if_snd))
1379 ixgbe_start_locked(txr, adapter->ifp);
1380 IXGBE_TX_UNLOCK(txr);
1382 /* Check for fan failure */
1383 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1384 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1385 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1386 "REPLACE IMMEDIATELY!!\n");
1387 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1390 /* Link status change */
1391 if (reg_eicr & IXGBE_EICR_LSC) {
1392 ixgbe_check_link(&adapter->hw,
1393 &adapter->link_speed, &adapter->link_up, 0);
1394 ixgbe_update_link_status(adapter);
1397 ixgbe_enable_intr(adapter);
1401 /*********************************************************************
1403 * MSIX Queue Interrupt Service routine
1405 **********************************************************************/
1407 ixgbe_msix_que(void *arg)
1409 struct ix_queue *que = arg;
1410 struct adapter *adapter = que->adapter;
1411 struct tx_ring *txr = que->txr;
1413 ixgbe_disable_queue(adapter, que->msix);
1416 ixgbe_rxeof(que, adapter->rx_process_limit);
1420 if (!ifq_is_empty(&adapter->ifp->if_snd))
1421 ixgbe_start_locked(txr, adapter->ifp);
1422 IXGBE_TX_UNLOCK(txr);
1424 /* Reenable this interrupt */
1425 ixgbe_enable_queue(adapter, que->msix);
1430 ixgbe_msix_link(void *arg)
1432 struct adapter *adapter = arg;
1433 struct ixgbe_hw *hw = &adapter->hw;
1436 ++adapter->link_irq;
1438 /* First get the cause */
1439 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1440 /* Clear interrupt with write */
1441 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1443 /* Link status change */
1444 if (reg_eicr & IXGBE_EICR_LSC) {
1445 ixgbe_check_link(&adapter->hw,
1446 &adapter->link_speed, &adapter->link_up, 0);
1447 ixgbe_update_link_status(adapter);
1450 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1452 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1453 /* This is probably overkill :) */
1454 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1456 /* Disable the interrupt */
1457 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1458 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1461 if (reg_eicr & IXGBE_EICR_ECC) {
1462 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1463 "Please Reboot!!\n");
1464 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1467 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1468 /* Clear the interrupt */
1469 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1470 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1471 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1472 /* Clear the interrupt */
1473 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1474 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1478 /* Check for fan failure */
1479 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1480 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1481 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1482 "REPLACE IMMEDIATELY!!\n");
1483 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1486 /* Check for over temp condition */
1487 if ((hw->mac.type == ixgbe_mac_X540) &&
1488 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1489 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1490 "PHY IS SHUT DOWN!!\n");
1491 device_printf(adapter->dev, "System shutdown required\n");
1492 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1495 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1499 /*********************************************************************
1501 * Media Ioctl callback
1503 * This routine is called whenever the user queries the status of
1504 * the interface using ifconfig.
1506 **********************************************************************/
1508 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1510 struct adapter *adapter = ifp->if_softc;
1512 ASSERT_IFNET_SERIALIZED_ALL(ifp);
1514 INIT_DEBUGOUT("ixgbe_media_status: begin");
1515 ixgbe_update_link_status(adapter);
1517 ifmr->ifm_status = IFM_AVALID;
1518 ifmr->ifm_active = IFM_ETHER;
1520 if (!adapter->link_active)
1523 ifmr->ifm_status |= IFM_ACTIVE;
1525 switch (adapter->link_speed) {
1526 case IXGBE_LINK_SPEED_100_FULL:
1527 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1529 case IXGBE_LINK_SPEED_1GB_FULL:
1530 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1532 case IXGBE_LINK_SPEED_10GB_FULL:
1533 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1540 /*********************************************************************
1542 * Media Ioctl callback
1544 * This routine is called when the user changes speed/duplex using
1545 * media/mediopt option with ifconfig.
1547 **********************************************************************/
1549 ixgbe_media_change(struct ifnet * ifp)
1551 struct adapter *adapter = ifp->if_softc;
1552 struct ifmedia *ifm = &adapter->media;
1554 INIT_DEBUGOUT("ixgbe_media_change: begin");
1556 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1559 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1561 adapter->hw.phy.autoneg_advertised =
1562 IXGBE_LINK_SPEED_100_FULL |
1563 IXGBE_LINK_SPEED_1GB_FULL |
1564 IXGBE_LINK_SPEED_10GB_FULL;
1567 device_printf(adapter->dev, "Only auto media type\n");
1574 /*********************************************************************
1576 * This routine maps the mbufs to tx descriptors, allowing the
1577 * TX engine to transmit the packets.
1578 * - return 0 on success, positive on failure
1580 **********************************************************************/
1583 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1585 struct adapter *adapter = txr->adapter;
1586 u32 olinfo_status = 0, cmd_type_len;
1588 int i, j, error, nsegs, maxsegs;
1589 int first, last = 0;
1590 struct mbuf *m_head;
1591 bus_dma_segment_t segs[adapter->num_segs];
1593 struct ixgbe_tx_buf *txbuf;
1594 union ixgbe_adv_tx_desc *txd = NULL;
1598 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1599 error = ixgbe_tso_pullup(txr, m_headp);
1605 /* Basic descriptor defines */
1606 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1607 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1609 if (m_head->m_flags & M_VLANTAG)
1610 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1613 * Important to capture the first descriptor
1614 * used because it will contain the index of
1615 * the one we tell the hardware to report back
1617 first = txr->next_avail_desc;
1618 txbuf = &txr->tx_buffers[first];
1622 * Map the packet for DMA.
1624 maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1625 if (maxsegs > adapter->num_segs)
1626 maxsegs = adapter->num_segs;
1628 error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1629 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1631 if (error == ENOBUFS)
1632 adapter->mbuf_defrag_failed++;
1634 adapter->no_tx_dma_setup++;
1641 /* Make certain there are enough descriptors */
1642 if (nsegs > txr->tx_avail - 2) {
1643 txr->no_desc_avail++;
1650 ** Set up the appropriate offload context
1651 ** this becomes the first descriptor of
1654 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1655 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1656 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1657 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1658 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1662 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1663 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1665 #ifdef IXGBE_IEEE1588
1666 /* This is changing soon to an mtag detection */
1667 if (we detect this mbuf has a TSTAMP mtag)
1668 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1672 /* Do the flow director magic */
1673 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1675 if (txr->atr_count >= atr_sample_rate) {
1676 ixgbe_atr(txr, m_head);
1681 /* Record payload length */
1683 olinfo_status |= m_head->m_pkthdr.len <<
1684 IXGBE_ADVTXD_PAYLEN_SHIFT;
1686 i = txr->next_avail_desc;
1687 for (j = 0; j < nsegs; j++) {
1691 txbuf = &txr->tx_buffers[i];
1692 txd = &txr->tx_base[i];
1693 seglen = segs[j].ds_len;
1694 segaddr = htole64(segs[j].ds_addr);
1696 txd->read.buffer_addr = segaddr;
1697 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1698 cmd_type_len |seglen);
1699 txd->read.olinfo_status = htole32(olinfo_status);
1700 last = i; /* descriptor that will get completion IRQ */
1702 if (++i == adapter->num_tx_desc)
1705 txbuf->m_head = NULL;
1706 txbuf->eop_index = -1;
1709 txd->read.cmd_type_len |=
1710 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1711 txr->tx_avail -= nsegs;
1712 txr->next_avail_desc = i;
1714 txbuf->m_head = m_head;
1715 /* Swap the dma map between the first and last descriptor */
1716 txr->tx_buffers[first].map = txbuf->map;
1718 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1720 /* Set the index of the descriptor that will be marked done */
1721 txbuf = &txr->tx_buffers[first];
1722 txbuf->eop_index = last;
1724 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1725 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1727 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1728 * hardware that this frame is available to transmit.
1730 ++txr->total_packets;
1731 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1736 bus_dmamap_unload(txr->txtag, txbuf->map);
1742 ixgbe_set_promisc(struct adapter *adapter)
1745 struct ifnet *ifp = adapter->ifp;
1747 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1748 reg_rctl &= (~IXGBE_FCTRL_UPE);
1749 reg_rctl &= (~IXGBE_FCTRL_MPE);
1750 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1752 if (ifp->if_flags & IFF_PROMISC) {
1753 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1754 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1755 } else if (ifp->if_flags & IFF_ALLMULTI) {
1756 reg_rctl |= IXGBE_FCTRL_MPE;
1757 reg_rctl &= ~IXGBE_FCTRL_UPE;
1758 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1764 /*********************************************************************
1767 * This routine is called whenever multicast address list is updated.
1769 **********************************************************************/
1770 #define IXGBE_RAR_ENTRIES 16
1773 ixgbe_set_multi(struct adapter *adapter)
1778 struct ifmultiaddr *ifma;
1780 struct ifnet *ifp = adapter->ifp;
1782 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1785 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1786 MAX_NUM_MULTICAST_ADDRESSES);
1788 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1789 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1790 if (ifp->if_flags & IFF_PROMISC)
1791 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1792 else if (ifp->if_flags & IFF_ALLMULTI) {
1793 fctrl |= IXGBE_FCTRL_MPE;
1794 fctrl &= ~IXGBE_FCTRL_UPE;
1796 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1798 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1800 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1801 if (ifma->ifma_addr->sa_family != AF_LINK)
1803 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1804 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1805 IXGBE_ETH_LENGTH_OF_ADDRESS);
1810 ixgbe_update_mc_addr_list(&adapter->hw,
1811 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1817 * This is an iterator function now needed by the multicast
1818 * shared code. It simply feeds the shared code routine the
1819 * addresses in the array of ixgbe_set_multi() one by one.
1822 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1824 u8 *addr = *update_ptr;
1828 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1829 *update_ptr = newptr;
1834 /*********************************************************************
1837 * This routine checks for link status,updates statistics,
1838 * and runs the watchdog check.
1840 **********************************************************************/
1843 ixgbe_local_timer(void *arg)
1845 struct adapter *adapter = arg;
1846 device_t dev = adapter->dev;
1847 struct ifnet *ifp = adapter->ifp;
1848 struct ix_queue *que = adapter->queues;
1849 struct tx_ring *txr = adapter->tx_rings;
1850 int hung, busy, paused;
1852 IXGBE_CORE_LOCK(adapter);
1853 hung = busy = paused = 0;
1855 /* Check for pluggable optics */
1856 if (adapter->sfp_probe)
1857 if (!ixgbe_sfp_probe(adapter))
1858 goto out; /* Nothing to do */
1860 ixgbe_update_link_status(adapter);
1861 ixgbe_update_stats_counters(adapter);
1864 * If the interface has been paused
1865 * then don't do the watchdog check
1867 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1871 ** Check the TX queues status
1872 ** - central locked handling of OACTIVE
1873 ** - watchdog only if all queues show hung
1875 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1876 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1879 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1882 /* Only truely watchdog if all queues show hung */
1883 if (hung == adapter->num_queues)
1885 /* Only turn off the stack flow when ALL are depleted */
1886 if (busy == adapter->num_queues)
1887 ifq_set_oactive(&ifp->if_snd);
1888 else if (ifq_is_oactive(&ifp->if_snd) && (busy < adapter->num_queues))
1889 ifq_clr_oactive(&ifp->if_snd);
1892 ixgbe_rearm_queues(adapter, adapter->que_mask);
1893 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1894 IXGBE_CORE_UNLOCK(adapter);
1898 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1899 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1900 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1901 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1902 device_printf(dev,"TX(%d) desc avail = %d,"
1903 "Next TX to Clean = %d\n",
1904 txr->me, txr->tx_avail, txr->next_to_clean);
1905 adapter->ifp->if_flags &= ~IFF_RUNNING;
1906 adapter->watchdog_events++;
1907 ixgbe_init_locked(adapter);
1909 IXGBE_CORE_UNLOCK(adapter);
1913 ** Note: this routine updates the OS on the link state
1914 ** the real check of the hardware only happens with
1915 ** a link interrupt.
1918 ixgbe_update_link_status(struct adapter *adapter)
1920 struct ifnet *ifp = adapter->ifp;
1921 struct tx_ring *txr = adapter->tx_rings;
1922 device_t dev = adapter->dev;
1925 if (adapter->link_up){
1926 if (adapter->link_active == FALSE) {
1928 device_printf(dev,"Link is up %d Gbps %s \n",
1929 ((adapter->link_speed == 128)? 10:1),
1931 adapter->link_active = TRUE;
1932 /* Update any Flow Control changes */
1933 ixgbe_fc_enable(&adapter->hw);
1934 ifp->if_link_state = LINK_STATE_UP;
1935 if_link_state_change(ifp);
1937 } else { /* Link down */
1938 if (adapter->link_active == TRUE) {
1940 device_printf(dev,"Link is Down\n");
1941 ifp->if_link_state = LINK_STATE_DOWN;
1942 if_link_state_change(ifp);
1943 adapter->link_active = FALSE;
1944 for (int i = 0; i < adapter->num_queues;
1946 txr->queue_status = IXGBE_QUEUE_IDLE;
1954 /*********************************************************************
1956 * This routine disables all traffic on the adapter by issuing a
1957 * global reset on the MAC and deallocates TX/RX buffers.
1959 **********************************************************************/
1962 ixgbe_stop(void *arg)
1965 struct adapter *adapter = arg;
1966 struct ixgbe_hw *hw = &adapter->hw;
1969 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1971 INIT_DEBUGOUT("ixgbe_stop: begin\n");
1972 ixgbe_disable_intr(adapter);
1973 callout_stop(&adapter->timer);
1975 /* Let the stack know...*/
1976 ifp->if_flags &= ~IFF_RUNNING;
1977 ifq_clr_oactive(&ifp->if_snd);
1980 hw->adapter_stopped = FALSE;
1981 ixgbe_stop_adapter(hw);
1982 /* Turn off the laser */
1983 if (hw->phy.multispeed_fiber)
1984 ixgbe_disable_tx_laser(hw);
1986 /* reprogram the RAR[0] in case user changed it. */
1987 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1993 /*********************************************************************
1995 * Determine hardware revision.
1997 **********************************************************************/
1999 ixgbe_identify_hardware(struct adapter *adapter)
2001 device_t dev = adapter->dev;
2002 struct ixgbe_hw *hw = &adapter->hw;
2004 /* Save off the information about this board */
2005 hw->vendor_id = pci_get_vendor(dev);
2006 hw->device_id = pci_get_device(dev);
2007 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2008 hw->subsystem_vendor_id =
2009 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2010 hw->subsystem_device_id =
2011 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2013 /* We need this here to set the num_segs below */
2014 ixgbe_set_mac_type(hw);
2016 /* Pick up the 82599 and VF settings */
2017 if (hw->mac.type != ixgbe_mac_82598EB) {
2018 hw->phy.smart_speed = ixgbe_smart_speed;
2019 adapter->num_segs = IXGBE_82599_SCATTER;
2021 adapter->num_segs = IXGBE_82598_SCATTER;
2026 /*********************************************************************
2028 * Determine optic type
2030 **********************************************************************/
2032 ixgbe_setup_optics(struct adapter *adapter)
2034 struct ixgbe_hw *hw = &adapter->hw;
2037 layer = ixgbe_get_supported_physical_layer(hw);
2039 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2040 adapter->optics = IFM_10G_T;
2044 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2045 adapter->optics = IFM_1000_T;
2049 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2050 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2051 adapter->optics = IFM_10G_LR;
2055 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2056 adapter->optics = IFM_10G_SR;
2060 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2061 adapter->optics = IFM_10G_TWINAX;
2065 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2066 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2067 adapter->optics = IFM_10G_CX4;
2071 /* If we get here just set the default */
2072 adapter->optics = IFM_ETHER | IFM_AUTO;
2076 /*********************************************************************
2078 * Setup the Legacy or MSI Interrupt handler
2080 **********************************************************************/
2082 ixgbe_allocate_legacy(struct adapter *adapter)
2084 device_t dev = adapter->dev;
2085 struct ix_queue *que = adapter->queues;
2087 unsigned int intr_flags;
2090 if (adapter->msix == 1)
2093 /* Try allocating a MSI interrupt first */
2094 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2097 /* We allocate a single interrupt resource */
2098 adapter->res = bus_alloc_resource_any(dev,
2099 SYS_RES_IRQ, &rid, intr_flags);
2100 if (adapter->res == NULL) {
2101 device_printf(dev, "Unable to allocate bus resource: "
2106 /* Tasklets for Link, SFP and Multispeed Fiber */
2107 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2108 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2110 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2113 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2114 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2115 device_printf(dev, "Failed to register fast interrupt "
2116 "handler: %d\n", error);
2117 taskqueue_free(adapter->tq);
2121 /* For simplicity in the handlers */
2122 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2128 /*********************************************************************
2130 * Setup MSIX Interrupt resources and handlers
2132 **********************************************************************/
2134 ixgbe_allocate_msix(struct adapter *adapter)
2136 device_t dev = adapter->dev;
2137 struct ix_queue *que = adapter->queues;
2138 int error, rid, vector = 0;
2141 error = pci_setup_msix(dev);
2143 device_printf(dev, "MSI-X setup failed\n");
2147 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2151 ** Bind the msix vector, and thus the
2152 ** ring to the corresponding cpu.
2154 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2156 device_printf(dev, "pci_alloc_msix_vector failed\n");
2160 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2161 RF_SHAREABLE | RF_ACTIVE);
2162 if (que->res == NULL) {
2163 device_printf(dev,"Unable to allocate"
2164 " bus resource: que interrupt [%d]\n", vector);
2167 /* Set the handler function */
2168 ksnprintf(desc, sizeof(desc), "%s que %d",
2169 device_get_nameunit(dev), i);
2170 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2171 ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2174 device_printf(dev, "Failed to register QUE handler");
2178 adapter->que_mask |= (u64)(1 << que->msix);
2181 /* and Link, bind vector to cpu #0 */
2183 error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2185 device_printf(dev, "pci_alloc_msix_vector failed\n");
2188 adapter->res = bus_alloc_resource_any(dev,
2189 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2190 if (!adapter->res) {
2191 device_printf(dev,"Unable to allocate"
2192 " bus resource: Link interrupt [%d]\n", rid);
2195 /* Set the link handler function */
2196 error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2197 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2200 adapter->res = NULL;
2201 device_printf(dev, "Failed to register LINK handler");
2204 pci_enable_msix(dev);
2206 adapter->linkvec = vector;
2207 /* Tasklets for Link, SFP and Multispeed Fiber */
2208 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2209 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2211 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2218 * Setup Either MSI/X or MSI
2221 ixgbe_setup_msix(struct adapter *adapter)
2223 device_t dev = adapter->dev;
2224 int rid, want, queues, msgs;
2226 /* Override by tuneable */
2227 if (ixgbe_enable_msix == 0)
2230 /* First try MSI/X */
2231 rid = PCIR_BAR(MSIX_82598_BAR);
2232 adapter->msix_mem = bus_alloc_resource_any(dev,
2233 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2234 if (!adapter->msix_mem) {
2235 rid += 4; /* 82599 maps in higher BAR */
2236 adapter->msix_mem = bus_alloc_resource_any(dev,
2237 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2239 if (!adapter->msix_mem) {
2240 /* May not be enabled */
2241 device_printf(adapter->dev,
2242 "Unable to map MSIX table \n");
2246 msgs = pci_msix_count(dev);
2247 if (msgs == 0) { /* system has msix disabled */
2248 bus_release_resource(dev, SYS_RES_MEMORY,
2249 rid, adapter->msix_mem);
2250 adapter->msix_mem = NULL;
2254 /* Figure out a reasonable auto config value */
2255 queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2257 if (ixgbe_num_queues != 0)
2258 queues = ixgbe_num_queues;
2259 /* Set max queues to 8 when autoconfiguring */
2260 else if ((ixgbe_num_queues == 0) && (queues > 8))
2264 ** Want one vector (RX/TX pair) per queue
2265 ** plus an additional for Link.
2271 device_printf(adapter->dev,
2272 "MSIX Configuration Problem, "
2273 "%d vectors but %d queues wanted!\n",
2275 return (0); /* Will go to Legacy setup */
2278 device_printf(adapter->dev,
2279 "Using MSIX interrupts with %d vectors\n", msgs);
2280 adapter->num_queues = queues;
2284 msgs = pci_msi_count(dev);
2290 ixgbe_allocate_pci_resources(struct adapter *adapter)
2293 device_t dev = adapter->dev;
2296 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2299 if (!(adapter->pci_mem)) {
2300 device_printf(dev,"Unable to allocate bus resource: memory\n");
2304 adapter->osdep.mem_bus_space_tag =
2305 rman_get_bustag(adapter->pci_mem);
2306 adapter->osdep.mem_bus_space_handle =
2307 rman_get_bushandle(adapter->pci_mem);
2308 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2310 /* Legacy defaults */
2311 adapter->num_queues = 1;
2312 adapter->hw.back = &adapter->osdep;
2315 ** Now setup MSI or MSI/X, should
2316 ** return us the number of supported
2317 ** vectors. (Will be 1 for MSI)
2319 adapter->msix = ixgbe_setup_msix(adapter);
2324 ixgbe_free_pci_resources(struct adapter * adapter)
2326 struct ix_queue *que = adapter->queues;
2327 device_t dev = adapter->dev;
2330 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2331 memrid = PCIR_BAR(MSIX_82598_BAR);
2333 memrid = PCIR_BAR(MSIX_82599_BAR);
2336 ** There is a slight possibility of a failure mode
2337 ** in attach that will result in entering this function
2338 ** before interrupt resources have been initialized, and
2339 ** in that case we do not want to execute the loops below
2340 ** We can detect this reliably by the state of the adapter
2343 if (adapter->res == NULL)
2347 ** Release all msix queue resources:
2349 for (int i = 0; i < adapter->num_queues; i++, que++) {
2350 rid = que->msix + 1;
2351 if (que->tag != NULL) {
2352 bus_teardown_intr(dev, que->res, que->tag);
2355 if (que->res != NULL)
2356 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2360 /* Clean the Legacy or Link interrupt last */
2361 if (adapter->linkvec) /* we are doing MSIX */
2362 rid = adapter->linkvec + 1;
2364 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2366 if (adapter->tag != NULL) {
2367 bus_teardown_intr(dev, adapter->res, adapter->tag);
2368 adapter->tag = NULL;
2370 if (adapter->res != NULL)
2371 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2372 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2373 pci_release_msi(adapter->dev);
2377 pci_release_msi(dev);
2379 if (adapter->msix_mem != NULL)
2380 bus_release_resource(dev, SYS_RES_MEMORY,
2381 memrid, adapter->msix_mem);
2383 if (adapter->pci_mem != NULL)
2384 bus_release_resource(dev, SYS_RES_MEMORY,
2385 PCIR_BAR(0), adapter->pci_mem);
2390 /*********************************************************************
2392 * Setup networking device structure and register an interface.
2394 **********************************************************************/
2396 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2398 struct ixgbe_hw *hw = &adapter->hw;
2401 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2403 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2405 device_printf(dev, "can not allocate ifnet structure\n");
2408 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2409 ifp->if_baudrate = 1000000000;
2410 ifp->if_init = ixgbe_init;
2411 ifp->if_softc = adapter;
2412 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2413 ifp->if_ioctl = ixgbe_ioctl;
2414 ifp->if_start = ixgbe_start;
2415 #if 0 /* __FreeBSD_version >= 800000 */
2416 ifp->if_transmit = ixgbe_mq_start;
2417 ifp->if_qflush = ixgbe_qflush;
2419 ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 2);
2421 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2423 adapter->max_frame_size =
2424 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2427 * Tell the upper layer(s) we support long frames.
2429 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2431 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2432 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2433 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2438 ifp->if_capenable = ifp->if_capabilities;
2440 /* Don't enable LRO by default */
2442 ifp->if_capabilities |= IFCAP_LRO;
2446 ** Don't turn this on by default, if vlans are
2447 ** created on another pseudo device (eg. lagg)
2448 ** then vlan events are not passed thru, breaking
2449 ** operation, but with HW FILTER off it works. If
2450 ** using vlans directly on the ixgbe driver you can
2451 ** enable this and get full hardware tag filtering.
2453 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2456 * Specify the media types supported by this adapter and register
2457 * callbacks to update media and link information
2459 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2460 ixgbe_media_status);
2461 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2462 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2463 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2464 ifmedia_add(&adapter->media,
2465 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2466 ifmedia_add(&adapter->media,
2467 IFM_ETHER | IFM_1000_T, 0, NULL);
2469 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2470 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2476 ixgbe_config_link(struct adapter *adapter)
2478 struct ixgbe_hw *hw = &adapter->hw;
2479 u32 autoneg, err = 0;
2480 bool sfp, negotiate;
2482 sfp = ixgbe_is_sfp(hw);
2485 if (hw->phy.multispeed_fiber) {
2486 hw->mac.ops.setup_sfp(hw);
2487 ixgbe_enable_tx_laser(hw);
2488 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2490 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2492 if (hw->mac.ops.check_link)
2493 err = ixgbe_check_link(hw, &autoneg,
2494 &adapter->link_up, FALSE);
2497 autoneg = hw->phy.autoneg_advertised;
2498 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2499 err = hw->mac.ops.get_link_capabilities(hw,
2500 &autoneg, &negotiate);
2503 if (hw->mac.ops.setup_link)
2504 err = hw->mac.ops.setup_link(hw, autoneg,
2505 negotiate, adapter->link_up);
2511 /********************************************************************
2512 * Manage DMA'able memory.
2513 *******************************************************************/
2515 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2519 *(bus_addr_t *) arg = segs->ds_addr;
2524 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2525 struct ixgbe_dma_alloc *dma, int mapflags)
2527 device_t dev = adapter->dev;
2530 r = bus_dma_tag_create(NULL, /* parent */
2531 DBA_ALIGN, 0, /* alignment, bounds */
2532 BUS_SPACE_MAXADDR, /* lowaddr */
2533 BUS_SPACE_MAXADDR, /* highaddr */
2534 NULL, NULL, /* filter, filterarg */
2537 size, /* maxsegsize */
2538 BUS_DMA_ALLOCNOW, /* flags */
2541 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2545 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2546 BUS_DMA_NOWAIT, &dma->dma_map);
2548 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2552 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2556 mapflags | BUS_DMA_NOWAIT);
2558 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2562 dma->dma_size = size;
2565 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2567 bus_dma_tag_destroy(dma->dma_tag);
2569 dma->dma_map = NULL;
2570 dma->dma_tag = NULL;
2575 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2577 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2578 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2579 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2580 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2581 bus_dma_tag_destroy(dma->dma_tag);
2585 /*********************************************************************
2587 * Allocate memory for the transmit and receive rings, and then
2588 * the descriptors associated with each, called only once at attach.
2590 **********************************************************************/
2592 ixgbe_allocate_queues(struct adapter *adapter)
2594 device_t dev = adapter->dev;
2595 struct ix_queue *que;
2596 struct tx_ring *txr;
2597 struct rx_ring *rxr;
2598 int rsize, tsize, error = IXGBE_SUCCESS;
2599 int txconf = 0, rxconf = 0;
2601 /* First allocate the top level queue structs */
2602 if (!(adapter->queues =
2603 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2604 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2605 device_printf(dev, "Unable to allocate queue memory\n");
2610 /* First allocate the TX ring struct memory */
2611 if (!(adapter->tx_rings =
2612 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2613 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2614 device_printf(dev, "Unable to allocate TX ring memory\n");
2619 /* Next allocate the RX */
2620 if (!(adapter->rx_rings =
2621 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2622 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2623 device_printf(dev, "Unable to allocate RX ring memory\n");
2628 /* For the ring itself */
2629 tsize = roundup2(adapter->num_tx_desc *
2630 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2633 * Now set up the TX queues, txconf is needed to handle the
2634 * possibility that things fail midcourse and we need to
2635 * undo memory gracefully
2637 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2638 /* Set up some basics */
2639 txr = &adapter->tx_rings[i];
2640 txr->adapter = adapter;
2643 /* Initialize the TX side lock */
2644 IXGBE_TX_LOCK_INIT(txr);
2646 if (ixgbe_dma_malloc(adapter, tsize,
2647 &txr->txdma, BUS_DMA_NOWAIT)) {
2649 "Unable to allocate TX Descriptor memory\n");
2653 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2654 bzero((void *)txr->tx_base, tsize);
2656 /* Now allocate transmit buffers for the ring */
2657 if (ixgbe_allocate_transmit_buffers(txr)) {
2659 "Critical Failure setting up transmit buffers\n");
2663 #if 0 /* __FreeBSD_version >= 800000 */
2664 /* Allocate a buf ring */
2665 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2666 M_WAITOK, &txr->tx_mtx);
2667 if (txr->br == NULL) {
2669 "Critical Failure setting up buf ring\n");
2677 * Next the RX queues...
2679 rsize = roundup2(adapter->num_rx_desc *
2680 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2681 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2682 rxr = &adapter->rx_rings[i];
2683 /* Set up some basics */
2684 rxr->adapter = adapter;
2687 /* Initialize the RX side lock */
2688 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2689 device_get_nameunit(dev), rxr->me);
2690 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2692 if (ixgbe_dma_malloc(adapter, rsize,
2693 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2695 "Unable to allocate RxDescriptor memory\n");
2699 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2700 bzero((void *)rxr->rx_base, rsize);
2702 /* Allocate receive buffers for the ring*/
2703 if (ixgbe_allocate_receive_buffers(rxr)) {
2705 "Critical Failure setting up receive buffers\n");
2712 ** Finally set up the queue holding structs
2714 for (int i = 0; i < adapter->num_queues; i++) {
2715 que = &adapter->queues[i];
2716 que->adapter = adapter;
2717 que->txr = &adapter->tx_rings[i];
2718 que->rxr = &adapter->rx_rings[i];
2724 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2725 ixgbe_dma_free(adapter, &rxr->rxdma);
2727 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2728 ixgbe_dma_free(adapter, &txr->txdma);
2729 kfree(adapter->rx_rings, M_DEVBUF);
2731 kfree(adapter->tx_rings, M_DEVBUF);
2733 kfree(adapter->queues, M_DEVBUF);
2738 /*********************************************************************
2740 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2741 * the information needed to transmit a packet on the wire. This is
2742 * called only once at attach, setup is done every reset.
2744 **********************************************************************/
2746 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2748 struct adapter *adapter = txr->adapter;
2749 device_t dev = adapter->dev;
2750 struct ixgbe_tx_buf *txbuf;
2754 * Setup DMA descriptor areas.
2756 if ((error = bus_dma_tag_create(
2758 1, 0, /* alignment, bounds */
2759 BUS_SPACE_MAXADDR, /* lowaddr */
2760 BUS_SPACE_MAXADDR, /* highaddr */
2761 NULL, NULL, /* filter, filterarg */
2762 IXGBE_TSO_SIZE, /* maxsize */
2763 adapter->num_segs, /* nsegments */
2764 PAGE_SIZE, /* maxsegsize */
2767 device_printf(dev,"Unable to allocate TX DMA tag\n");
2771 if (!(txr->tx_buffers =
2772 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2773 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2774 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2779 /* Create the descriptor buffer dma maps */
2780 txbuf = txr->tx_buffers;
2781 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2782 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2784 device_printf(dev, "Unable to create TX DMA map\n");
2791 /* We free all, it handles case where we are in the middle */
2792 ixgbe_free_transmit_structures(adapter);
2796 /*********************************************************************
2798 * Initialize a transmit ring.
2800 **********************************************************************/
2802 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2804 struct adapter *adapter = txr->adapter;
2805 struct ixgbe_tx_buf *txbuf;
2808 struct netmap_adapter *na = NA(adapter->ifp);
2809 struct netmap_slot *slot;
2810 #endif /* DEV_NETMAP */
2812 /* Clear the old ring contents */
2816 * (under lock): if in netmap mode, do some consistency
2817 * checks and set slot to entry 0 of the netmap ring.
2819 slot = netmap_reset(na, NR_TX, txr->me, 0);
2820 #endif /* DEV_NETMAP */
2821 bzero((void *)txr->tx_base,
2822 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2824 txr->next_avail_desc = 0;
2825 txr->next_to_clean = 0;
2827 /* Free any existing tx buffers. */
2828 txbuf = txr->tx_buffers;
2829 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2830 if (txbuf->m_head != NULL) {
2831 bus_dmamap_sync(txr->txtag, txbuf->map,
2832 BUS_DMASYNC_POSTWRITE);
2833 bus_dmamap_unload(txr->txtag, txbuf->map);
2834 m_freem(txbuf->m_head);
2835 txbuf->m_head = NULL;
2839 * In netmap mode, set the map for the packet buffer.
2840 * NOTE: Some drivers (not this one) also need to set
2841 * the physical buffer address in the NIC ring.
2842 * Slots in the netmap ring (indexed by "si") are
2843 * kring->nkr_hwofs positions "ahead" wrt the
2844 * corresponding slot in the NIC ring. In some drivers
2845 * (not here) nkr_hwofs can be negative. Function
2846 * netmap_idx_n2k() handles wraparounds properly.
2849 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2850 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2852 #endif /* DEV_NETMAP */
2853 /* Clear the EOP index */
2854 txbuf->eop_index = -1;
2858 /* Set the rate at which we sample packets */
2859 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2860 txr->atr_sample = atr_sample_rate;
2863 /* Set number of descriptors available */
2864 txr->tx_avail = adapter->num_tx_desc;
2866 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2867 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2868 IXGBE_TX_UNLOCK(txr);
2871 /*********************************************************************
2873 * Initialize all transmit rings.
2875 **********************************************************************/
2877 ixgbe_setup_transmit_structures(struct adapter *adapter)
2879 struct tx_ring *txr = adapter->tx_rings;
2881 for (int i = 0; i < adapter->num_queues; i++, txr++)
2882 ixgbe_setup_transmit_ring(txr);
2887 /*********************************************************************
2889 * Enable transmit unit.
2891 **********************************************************************/
2893 ixgbe_initialize_transmit_units(struct adapter *adapter)
2895 struct tx_ring *txr = adapter->tx_rings;
2896 struct ixgbe_hw *hw = &adapter->hw;
2898 /* Setup the Base and Length of the Tx Descriptor Ring */
2900 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2901 u64 tdba = txr->txdma.dma_paddr;
2904 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2905 (tdba & 0x00000000ffffffffULL));
2906 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2907 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2908 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2910 /* Setup the HW Tx Head and Tail descriptor pointers */
2911 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2912 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2914 /* Setup Transmit Descriptor Cmd Settings */
2915 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2916 txr->queue_status = IXGBE_QUEUE_IDLE;
2918 /* Disable Head Writeback */
2919 switch (hw->mac.type) {
2920 case ixgbe_mac_82598EB:
2921 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2923 case ixgbe_mac_82599EB:
2924 case ixgbe_mac_X540:
2926 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2929 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2930 switch (hw->mac.type) {
2931 case ixgbe_mac_82598EB:
2932 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2934 case ixgbe_mac_82599EB:
2935 case ixgbe_mac_X540:
2937 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2943 if (hw->mac.type != ixgbe_mac_82598EB) {
2944 u32 dmatxctl, rttdcs;
2945 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2946 dmatxctl |= IXGBE_DMATXCTL_TE;
2947 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2948 /* Disable arbiter to set MTQC */
2949 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2950 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2951 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2952 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2953 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2954 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2960 /*********************************************************************
2962 * Free all transmit rings.
2964 **********************************************************************/
2966 ixgbe_free_transmit_structures(struct adapter *adapter)
2968 struct tx_ring *txr = adapter->tx_rings;
2970 for (int i = 0; i < adapter->num_queues; i++, txr++) {
2972 ixgbe_free_transmit_buffers(txr);
2973 ixgbe_dma_free(adapter, &txr->txdma);
2974 IXGBE_TX_UNLOCK(txr);
2975 IXGBE_TX_LOCK_DESTROY(txr);
2977 kfree(adapter->tx_rings, M_DEVBUF);
2980 /*********************************************************************
2982 * Free transmit ring related data structures.
2984 **********************************************************************/
2986 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2988 struct adapter *adapter = txr->adapter;
2989 struct ixgbe_tx_buf *tx_buffer;
2992 INIT_DEBUGOUT("free_transmit_ring: begin");
2994 if (txr->tx_buffers == NULL)
2997 tx_buffer = txr->tx_buffers;
2998 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2999 if (tx_buffer->m_head != NULL) {
3000 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3001 BUS_DMASYNC_POSTWRITE);
3002 bus_dmamap_unload(txr->txtag,
3004 m_freem(tx_buffer->m_head);
3005 tx_buffer->m_head = NULL;
3006 if (tx_buffer->map != NULL) {
3007 bus_dmamap_destroy(txr->txtag,
3009 tx_buffer->map = NULL;
3011 } else if (tx_buffer->map != NULL) {
3012 bus_dmamap_unload(txr->txtag,
3014 bus_dmamap_destroy(txr->txtag,
3016 tx_buffer->map = NULL;
3019 #if 0 /* __FreeBSD_version >= 800000 */
3020 if (txr->br != NULL)
3021 buf_ring_free(txr->br, M_DEVBUF);
3023 if (txr->tx_buffers != NULL) {
3024 kfree(txr->tx_buffers, M_DEVBUF);
3025 txr->tx_buffers = NULL;
3027 if (txr->txtag != NULL) {
3028 bus_dma_tag_destroy(txr->txtag);
3034 /*********************************************************************
3036 * Advanced Context Descriptor setup for VLAN or CSUM
3038 **********************************************************************/
3041 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3043 struct adapter *adapter = txr->adapter;
3044 struct ixgbe_adv_tx_context_desc *TXD;
3045 struct ixgbe_tx_buf *tx_buffer;
3046 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3047 struct ether_vlan_header *eh;
3049 struct ip6_hdr *ip6;
3050 int ehdrlen, ip_hlen = 0;
3053 bool offload = TRUE;
3054 int ctxd = txr->next_avail_desc;
3058 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3061 tx_buffer = &txr->tx_buffers[ctxd];
3062 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3065 ** In advanced descriptors the vlan tag must
3066 ** be placed into the descriptor itself.
3068 if (mp->m_flags & M_VLANTAG) {
3069 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3070 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3071 } else if (offload == FALSE)
3075 * Determine where frame payload starts.
3076 * Jump over vlan headers if already present,
3077 * helpful for QinQ too.
3079 eh = mtod(mp, struct ether_vlan_header *);
3080 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3081 etype = ntohs(eh->evl_proto);
3082 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3084 etype = ntohs(eh->evl_encap_proto);
3085 ehdrlen = ETHER_HDR_LEN;
3088 /* Set the ether header length */
3089 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3093 ip = (struct ip *)(mp->m_data + ehdrlen);
3094 ip_hlen = ip->ip_hl << 2;
3096 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3098 case ETHERTYPE_IPV6:
3099 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3100 ip_hlen = sizeof(struct ip6_hdr);
3101 /* XXX-BZ this will go badly in case of ext hdrs. */
3102 ipproto = ip6->ip6_nxt;
3103 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3110 vlan_macip_lens |= ip_hlen;
3111 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3115 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3116 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3120 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3121 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3126 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3127 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3135 /* Now copy bits into descriptor */
3136 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3137 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3138 TXD->seqnum_seed = htole32(0);
3139 TXD->mss_l4len_idx = htole32(0);
3141 tx_buffer->m_head = NULL;
3142 tx_buffer->eop_index = -1;
3144 /* We've consumed the first desc, adjust counters */
3145 if (++ctxd == adapter->num_tx_desc)
3147 txr->next_avail_desc = ctxd;
3153 /**********************************************************************
3155 * Setup work for hardware segmentation offload (TSO) on
3156 * adapters using advanced tx descriptors
3158 **********************************************************************/
3160 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3163 struct adapter *adapter = txr->adapter;
3164 struct ixgbe_adv_tx_context_desc *TXD;
3165 struct ixgbe_tx_buf *tx_buffer;
3166 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3167 u16 vtag = 0, eh_type;
3168 u32 mss_l4len_idx = 0, len;
3169 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3170 struct ether_vlan_header *eh;
3171 #if 0 /* IPv6 TSO */
3173 struct ip6_hdr *ip6;
3183 * Determine where frame payload starts.
3184 * Jump over vlan headers if already present
3186 eh = mtod(mp, struct ether_vlan_header *);
3187 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3188 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3189 eh_type = eh->evl_proto;
3191 ehdrlen = ETHER_HDR_LEN;
3192 eh_type = eh->evl_encap_proto;
3195 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3196 len = ehdrlen + sizeof(struct tcphdr);
3197 switch (ntohs(eh_type)) {
3198 #if 0 /* IPv6 TSO */
3200 case ETHERTYPE_IPV6:
3201 if (mp->m_len < len + sizeof(struct ip6_hdr))
3203 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3204 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3205 if (ip6->ip6_nxt != IPPROTO_TCP)
3207 ip_hlen = sizeof(struct ip6_hdr);
3208 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3209 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3210 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3216 if (mp->m_len < len + sizeof(struct ip))
3218 ip = (struct ip *)(mp->m_data + ehdrlen);
3219 if (ip->ip_p != IPPROTO_TCP)
3222 ip_hlen = ip->ip_hl << 2;
3223 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3224 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3225 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3226 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3227 /* Tell transmit desc to also do IPv4 checksum. */
3228 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3232 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3233 __func__, ntohs(eh_type));
3237 ctxd = txr->next_avail_desc;
3238 tx_buffer = &txr->tx_buffers[ctxd];
3239 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3241 tcp_hlen = th->th_off << 2;
3243 /* This is used in the transmit desc in encap */
3244 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3246 /* VLAN MACLEN IPLEN */
3247 if (mp->m_flags & M_VLANTAG) {
3248 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3249 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3252 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3253 vlan_macip_lens |= ip_hlen;
3254 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3256 /* ADV DTYPE TUCMD */
3257 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3258 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3259 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3262 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3263 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3264 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3266 TXD->seqnum_seed = htole32(0);
3267 tx_buffer->m_head = NULL;
3268 tx_buffer->eop_index = -1;
3270 if (++ctxd == adapter->num_tx_desc)
3274 txr->next_avail_desc = ctxd;
3280 ** This routine parses packet headers so that Flow
3281 ** Director can make a hashed filter table entry
3282 ** allowing traffic flows to be identified and kept
3283 ** on the same cpu. This would be a performance
3284 ** hit, but we only do it at IXGBE_FDIR_RATE of
3288 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3290 struct adapter *adapter = txr->adapter;
3291 struct ix_queue *que;
3295 struct ether_vlan_header *eh;
3296 union ixgbe_atr_hash_dword input = {.dword = 0};
3297 union ixgbe_atr_hash_dword common = {.dword = 0};
3298 int ehdrlen, ip_hlen;
3301 eh = mtod(mp, struct ether_vlan_header *);
3302 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3303 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3304 etype = eh->evl_proto;
3306 ehdrlen = ETHER_HDR_LEN;
3307 etype = eh->evl_encap_proto;
3310 /* Only handling IPv4 */
3311 if (etype != htons(ETHERTYPE_IP))
3314 ip = (struct ip *)(mp->m_data + ehdrlen);
3315 ip_hlen = ip->ip_hl << 2;
3317 /* check if we're UDP or TCP */
3320 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3321 /* src and dst are inverted */
3322 common.port.dst ^= th->th_sport;
3323 common.port.src ^= th->th_dport;
3324 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3327 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3328 /* src and dst are inverted */
3329 common.port.dst ^= uh->uh_sport;
3330 common.port.src ^= uh->uh_dport;
3331 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3337 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3338 if (mp->m_pkthdr.ether_vtag)
3339 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3341 common.flex_bytes ^= etype;
3342 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3344 que = &adapter->queues[txr->me];
3346 ** This assumes the Rx queue and Tx
3347 ** queue are bound to the same CPU
3349 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3350 input, common, que->msix);
3352 #endif /* IXGBE_FDIR */
3354 /**********************************************************************
3356 * Examine each tx_buffer in the used queue. If the hardware is done
3357 * processing the packet then free associated resources. The
3358 * tx_buffer is put back on the free queue.
3360 **********************************************************************/
3362 ixgbe_txeof(struct tx_ring *txr)
3364 struct adapter *adapter = txr->adapter;
3365 struct ifnet *ifp = adapter->ifp;
3366 u32 first, last, done, processed;
3367 struct ixgbe_tx_buf *tx_buffer;
3368 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3370 IXGBE_TX_LOCK_ASSERT(txr);
3373 if (ifp->if_capenable & IFCAP_NETMAP) {
3374 struct netmap_adapter *na = NA(ifp);
3375 struct netmap_kring *kring = &na->tx_rings[txr->me];
3377 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3379 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3380 BUS_DMASYNC_POSTREAD);
3382 * In netmap mode, all the work is done in the context
3383 * of the client thread. Interrupt handlers only wake up
3384 * clients, which may be sleeping on individual rings
3385 * or on a global resource for all rings.
3386 * To implement tx interrupt mitigation, we wake up the client
3387 * thread roughly every half ring, even if the NIC interrupts
3388 * more frequently. This is implemented as follows:
3389 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3390 * the slot that should wake up the thread (nkr_num_slots
3391 * means the user thread should not be woken up);
3392 * - the driver ignores tx interrupts unless netmap_mitigate=0
3393 * or the slot has the DD bit set.
3395 * When the driver has separate locks, we need to
3396 * release and re-acquire txlock to avoid deadlocks.
3397 * XXX see if we can find a better way.
3399 if (!netmap_mitigate ||
3400 (kring->nr_kflags < kring->nkr_num_slots &&
3401 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3402 kring->nr_kflags = kring->nkr_num_slots;
3403 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3404 IXGBE_TX_UNLOCK(txr);
3405 IXGBE_CORE_LOCK(adapter);
3406 selwakeuppri(&na->tx_si, PI_NET);
3407 IXGBE_CORE_UNLOCK(adapter);
3412 #endif /* DEV_NETMAP */
3414 if (txr->tx_avail == adapter->num_tx_desc) {
3415 txr->queue_status = IXGBE_QUEUE_IDLE;
3420 first = txr->next_to_clean;
3421 tx_buffer = &txr->tx_buffers[first];
3422 /* For cleanup we just use legacy struct */
3423 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3424 last = tx_buffer->eop_index;
3427 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3430 ** Get the index of the first descriptor
3431 ** BEYOND the EOP and call that 'done'.
3432 ** I do this so the comparison in the
3433 ** inner while loop below can be simple
3435 if (++last == adapter->num_tx_desc) last = 0;
3438 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3439 BUS_DMASYNC_POSTREAD);
3441 ** Only the EOP descriptor of a packet now has the DD
3442 ** bit set, this is what we look for...
3444 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3445 /* We clean the range of the packet */
3446 while (first != done) {
3447 tx_desc->upper.data = 0;
3448 tx_desc->lower.data = 0;
3449 tx_desc->buffer_addr = 0;
3453 if (tx_buffer->m_head) {
3455 tx_buffer->m_head->m_pkthdr.len;
3456 bus_dmamap_sync(txr->txtag,
3458 BUS_DMASYNC_POSTWRITE);
3459 bus_dmamap_unload(txr->txtag,
3461 m_freem(tx_buffer->m_head);
3462 tx_buffer->m_head = NULL;
3463 tx_buffer->map = NULL;
3465 tx_buffer->eop_index = -1;
3466 txr->watchdog_time = ticks;
3468 if (++first == adapter->num_tx_desc)
3471 tx_buffer = &txr->tx_buffers[first];
3473 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3476 IFNET_STAT_INC(ifp, opackets, 1);
3477 /* See if there is more work now */
3478 last = tx_buffer->eop_index;
3481 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3482 /* Get next done point */
3483 if (++last == adapter->num_tx_desc) last = 0;
3488 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3489 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3491 txr->next_to_clean = first;
3494 ** Watchdog calculation, we know there's
3495 ** work outstanding or the first return
3496 ** would have been taken, so none processed
3497 ** for too long indicates a hang.
3499 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3500 txr->queue_status = IXGBE_QUEUE_HUNG;
3502 /* With a minimum free clear the depleted state bit. */
3503 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3504 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3506 if (txr->tx_avail == adapter->num_tx_desc) {
3507 txr->queue_status = IXGBE_QUEUE_IDLE;
3511 /*********************************************************************
3513 * Refresh mbuf buffers for RX descriptor rings
3514 * - now keeps its own state so discards due to resource
3515 * exhaustion are unnecessary, if an mbuf cannot be obtained
3516 * it just returns, keeping its placeholder, thus it can simply
3517 * be recalled to try again.
3519 **********************************************************************/
3521 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3523 struct adapter *adapter = rxr->adapter;
3524 bus_dma_segment_t hseg[1];
3525 bus_dma_segment_t pseg[1];
3526 struct ixgbe_rx_buf *rxbuf;
3527 struct mbuf *mh, *mp;
3528 int i, j, nsegs, error;
3529 bool refreshed = FALSE;
3531 i = j = rxr->next_to_refresh;
3532 /* Control the loop with one beyond */
3533 if (++j == adapter->num_rx_desc)
3536 while (j != limit) {
3537 rxbuf = &rxr->rx_buffers[i];
3538 if (rxr->hdr_split == FALSE)
3541 if (rxbuf->m_head == NULL) {
3542 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3548 mh->m_pkthdr.len = mh->m_len = MHLEN;
3550 mh->m_flags |= M_PKTHDR;
3551 /* Get the memory mapping */
3552 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3553 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3555 kprintf("Refresh mbufs: hdr dmamap load"
3556 " failure - %d\n", error);
3558 rxbuf->m_head = NULL;
3562 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3563 BUS_DMASYNC_PREREAD);
3564 rxr->rx_base[i].read.hdr_addr =
3565 htole64(hseg[0].ds_addr);
3568 if (rxbuf->m_pack == NULL) {
3569 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3570 M_PKTHDR, adapter->rx_mbuf_sz);
3576 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3577 /* Get the memory mapping */
3578 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3579 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3581 kprintf("Refresh mbufs: payload dmamap load"
3582 " failure - %d\n", error);
3584 rxbuf->m_pack = NULL;
3588 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3589 BUS_DMASYNC_PREREAD);
3590 rxr->rx_base[i].read.pkt_addr =
3591 htole64(pseg[0].ds_addr);
3594 /* Next is precalculated */
3596 rxr->next_to_refresh = i;
3597 if (++j == adapter->num_rx_desc)
3601 if (refreshed) /* Update hardware tail index */
3602 IXGBE_WRITE_REG(&adapter->hw,
3603 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3607 /*********************************************************************
3609 * Allocate memory for rx_buffer structures. Since we use one
3610 * rx_buffer per received packet, the maximum number of rx_buffer's
3611 * that we'll need is equal to the number of receive descriptors
3612 * that we've allocated.
3614 **********************************************************************/
3616 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3618 struct adapter *adapter = rxr->adapter;
3619 device_t dev = adapter->dev;
3620 struct ixgbe_rx_buf *rxbuf;
3621 int i, bsize, error;
3623 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3624 if (!(rxr->rx_buffers =
3625 (struct ixgbe_rx_buf *) kmalloc(bsize,
3626 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3627 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3632 if ((error = bus_dma_tag_create(NULL, /* parent */
3633 1, 0, /* alignment, bounds */
3634 BUS_SPACE_MAXADDR, /* lowaddr */
3635 BUS_SPACE_MAXADDR, /* highaddr */
3636 NULL, NULL, /* filter, filterarg */
3637 MSIZE, /* maxsize */
3639 MSIZE, /* maxsegsize */
3642 device_printf(dev, "Unable to create RX DMA tag\n");
3646 if ((error = bus_dma_tag_create(NULL, /* parent */
3647 1, 0, /* alignment, bounds */
3648 BUS_SPACE_MAXADDR, /* lowaddr */
3649 BUS_SPACE_MAXADDR, /* highaddr */
3650 NULL, NULL, /* filter, filterarg */
3651 MJUM16BYTES, /* maxsize */
3653 MJUM16BYTES, /* maxsegsize */
3656 device_printf(dev, "Unable to create RX DMA tag\n");
3660 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3661 rxbuf = &rxr->rx_buffers[i];
3662 error = bus_dmamap_create(rxr->htag,
3663 BUS_DMA_NOWAIT, &rxbuf->hmap);
3665 device_printf(dev, "Unable to create RX head map\n");
3668 error = bus_dmamap_create(rxr->ptag,
3669 BUS_DMA_NOWAIT, &rxbuf->pmap);
3671 device_printf(dev, "Unable to create RX pkt map\n");
3679 /* Frees all, but can handle partial completion */
3680 ixgbe_free_receive_structures(adapter);
3685 ** Used to detect a descriptor that has
3686 ** been merged by Hardware RSC.
3689 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3691 return (le32toh(rx->wb.lower.lo_dword.data) &
3692 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3695 /*********************************************************************
3697 * Initialize Hardware RSC (LRO) feature on 82599
3698 * for an RX ring, this is toggled by the LRO capability
3699 * even though it is transparent to the stack.
3701 **********************************************************************/
3704 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3706 struct adapter *adapter = rxr->adapter;
3707 struct ixgbe_hw *hw = &adapter->hw;
3708 u32 rscctrl, rdrxctl;
3710 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3711 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3712 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3713 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3714 #endif /* DEV_NETMAP */
3715 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3716 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3717 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3719 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3720 rscctrl |= IXGBE_RSCCTL_RSCEN;
3722 ** Limit the total number of descriptors that
3723 ** can be combined, so it does not exceed 64K
3725 if (adapter->rx_mbuf_sz == MCLBYTES)
3726 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3727 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3728 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3729 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3730 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3731 else /* Using 16K cluster */
3732 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3734 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3736 /* Enable TCP header recognition */
3737 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3738 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3739 IXGBE_PSRTYPE_TCPHDR));
3741 /* Disable RSC for ACK packets */
3742 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3743 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3750 ixgbe_free_receive_ring(struct rx_ring *rxr)
3752 struct adapter *adapter;
3753 struct ixgbe_rx_buf *rxbuf;
3756 adapter = rxr->adapter;
3757 for (i = 0; i < adapter->num_rx_desc; i++) {
3758 rxbuf = &rxr->rx_buffers[i];
3759 if (rxbuf->m_head != NULL) {
3760 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3761 BUS_DMASYNC_POSTREAD);
3762 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3763 rxbuf->m_head->m_flags |= M_PKTHDR;
3764 m_freem(rxbuf->m_head);
3766 if (rxbuf->m_pack != NULL) {
3767 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3768 BUS_DMASYNC_POSTREAD);
3769 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3770 rxbuf->m_pack->m_flags |= M_PKTHDR;
3771 m_freem(rxbuf->m_pack);
3773 rxbuf->m_head = NULL;
3774 rxbuf->m_pack = NULL;
3779 /*********************************************************************
3781 * Initialize a receive ring and its buffers.
3783 **********************************************************************/
3785 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3787 struct adapter *adapter;
3792 struct ixgbe_rx_buf *rxbuf;
3793 bus_dma_segment_t pseg[1], hseg[1];
3795 struct lro_ctrl *lro = &rxr->lro;
3797 int rsize, nsegs, error = 0;
3799 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3800 struct netmap_slot *slot;
3801 #endif /* DEV_NETMAP */
3803 adapter = rxr->adapter;
3809 /* Clear the ring contents */
3812 /* same as in ixgbe_setup_transmit_ring() */
3813 slot = netmap_reset(na, NR_RX, rxr->me, 0);
3814 #endif /* DEV_NETMAP */
3815 rsize = roundup2(adapter->num_rx_desc *
3816 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3817 bzero((void *)rxr->rx_base, rsize);
3819 /* Free current RX buffer structs and their mbufs */
3820 ixgbe_free_receive_ring(rxr);
3822 /* Configure header split? */
3823 if (ixgbe_header_split)
3824 rxr->hdr_split = TRUE;
3826 /* Now replenish the mbufs */
3827 for (int j = 0; j != adapter->num_rx_desc; ++j) {
3828 struct mbuf *mh, *mp;
3830 rxbuf = &rxr->rx_buffers[j];
3833 * In netmap mode, fill the map and set the buffer
3834 * address in the NIC ring, considering the offset
3835 * between the netmap and NIC rings (see comment in
3836 * ixgbe_setup_transmit_ring() ). No need to allocate
3837 * an mbuf, so end the block with a continue;
3840 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3844 addr = PNMB(slot + sj, &paddr);
3845 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3846 /* Update descriptor */
3847 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3850 #endif /* DEV_NETMAP */
3852 ** Don't allocate mbufs if not
3853 ** doing header split, its wasteful
3855 if (rxr->hdr_split == FALSE)
3858 /* First the header */
3859 rxbuf->m_head = m_gethdr(MB_DONTWAIT, MT_DATA);
3860 if (rxbuf->m_head == NULL) {
3864 m_adj(rxbuf->m_head, ETHER_ALIGN);
3866 mh->m_len = mh->m_pkthdr.len = MHLEN;
3867 mh->m_flags |= M_PKTHDR;
3868 /* Get the memory mapping */
3869 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3870 rxbuf->hmap, rxbuf->m_head, hseg, 1,
3871 &nsegs, BUS_DMA_NOWAIT);
3873 if (error != 0) /* Nothing elegant to do here */
3875 bus_dmamap_sync(rxr->htag,
3876 rxbuf->hmap, BUS_DMASYNC_PREREAD);
3877 /* Update descriptor */
3878 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3881 /* Now the payload cluster */
3882 rxbuf->m_pack = m_getjcl(MB_DONTWAIT, MT_DATA,
3883 M_PKTHDR, adapter->rx_mbuf_sz);
3884 if (rxbuf->m_pack == NULL) {
3889 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3890 /* Get the memory mapping */
3891 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3892 rxbuf->pmap, mp, hseg, 1,
3893 &nsegs, BUS_DMA_NOWAIT);
3896 bus_dmamap_sync(rxr->ptag,
3897 rxbuf->pmap, BUS_DMASYNC_PREREAD);
3898 /* Update descriptor */
3899 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3903 /* Setup our descriptor indices */
3904 rxr->next_to_check = 0;
3905 rxr->next_to_refresh = 0;
3906 rxr->lro_enabled = FALSE;
3907 rxr->rx_split_packets = 0;
3909 rxr->discard = FALSE;
3910 rxr->vtag_strip = FALSE;
3912 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3913 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3916 ** Now set up the LRO interface:
3917 ** 82598 uses software LRO, the
3918 ** 82599 and X540 use a hardware assist.
3921 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3922 (ifp->if_capenable & IFCAP_RXCSUM) &&
3923 (ifp->if_capenable & IFCAP_LRO))
3924 ixgbe_setup_hw_rsc(rxr);
3925 else if (ifp->if_capenable & IFCAP_LRO) {
3926 int err = tcp_lro_init(lro);
3928 device_printf(dev, "LRO Initialization failed!\n");
3931 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3932 rxr->lro_enabled = TRUE;
3933 lro->ifp = adapter->ifp;
3937 IXGBE_RX_UNLOCK(rxr);
3941 ixgbe_free_receive_ring(rxr);
3942 IXGBE_RX_UNLOCK(rxr);
3946 /*********************************************************************
3948 * Initialize all receive rings.
3950 **********************************************************************/
3952 ixgbe_setup_receive_structures(struct adapter *adapter)
3954 struct rx_ring *rxr = adapter->rx_rings;
3957 for (j = 0; j < adapter->num_queues; j++, rxr++)
3958 if (ixgbe_setup_receive_ring(rxr))
3964 * Free RX buffers allocated so far, we will only handle
3965 * the rings that completed, the failing case will have
3966 * cleaned up for itself. 'j' failed, so its the terminus.
3968 for (int i = 0; i < j; ++i) {
3969 rxr = &adapter->rx_rings[i];
3970 ixgbe_free_receive_ring(rxr);
3976 /*********************************************************************
3978 * Setup receive registers and features.
3980 **********************************************************************/
3981 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3983 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3986 ixgbe_initialize_receive_units(struct adapter *adapter)
3988 struct rx_ring *rxr = adapter->rx_rings;
3989 struct ixgbe_hw *hw = &adapter->hw;
3990 struct ifnet *ifp = adapter->ifp;
3991 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
3992 u32 reta, mrqc = 0, hlreg, random[10];
3996 * Make sure receives are disabled while
3997 * setting up the descriptor ring
3999 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4000 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4001 rxctrl & ~IXGBE_RXCTRL_RXEN);
4003 /* Enable broadcasts */
4004 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4005 fctrl |= IXGBE_FCTRL_BAM;
4006 fctrl |= IXGBE_FCTRL_DPF;
4007 fctrl |= IXGBE_FCTRL_PMCF;
4008 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4010 /* Set for Jumbo Frames? */
4011 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4012 if (ifp->if_mtu > ETHERMTU)
4013 hlreg |= IXGBE_HLREG0_JUMBOEN;
4015 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4017 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4018 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4019 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4021 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4022 #endif /* DEV_NETMAP */
4023 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4025 bufsz = (adapter->rx_mbuf_sz +
4026 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4028 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4029 u64 rdba = rxr->rxdma.dma_paddr;
4031 /* Setup the Base and Length of the Rx Descriptor Ring */
4032 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4033 (rdba & 0x00000000ffffffffULL));
4034 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4035 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4036 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4038 /* Set up the SRRCTL register */
4039 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4040 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4041 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4043 if (rxr->hdr_split) {
4044 /* Use a standard mbuf for the header */
4045 srrctl |= ((IXGBE_RX_HDR <<
4046 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4047 & IXGBE_SRRCTL_BSIZEHDR_MASK);
4048 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4050 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4051 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4053 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4054 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4055 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4058 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4059 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4060 IXGBE_PSRTYPE_UDPHDR |
4061 IXGBE_PSRTYPE_IPV4HDR |
4062 IXGBE_PSRTYPE_IPV6HDR;
4063 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4066 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4069 if (adapter->num_queues > 1) {
4073 /* set up random bits */
4074 karc4rand(&random, sizeof(random));
4076 /* Set up the redirection table */
4077 for (i = 0, j = 0; i < 128; i++, j++) {
4078 if (j == adapter->num_queues) j = 0;
4079 reta = (reta << 8) | (j * 0x11);
4081 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4084 /* Now fill our hash function seeds */
4085 for (int i = 0; i < 10; i++)
4086 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4088 /* Perform hash on these packet types */
4089 mrqc = IXGBE_MRQC_RSSEN
4090 | IXGBE_MRQC_RSS_FIELD_IPV4
4091 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4092 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4093 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4094 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4095 | IXGBE_MRQC_RSS_FIELD_IPV6
4096 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4097 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4098 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4099 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4101 /* RSS and RX IPP Checksum are mutually exclusive */
4102 rxcsum |= IXGBE_RXCSUM_PCSD;
4105 if (ifp->if_capenable & IFCAP_RXCSUM)
4106 rxcsum |= IXGBE_RXCSUM_PCSD;
4108 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4109 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4111 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4116 /*********************************************************************
4118 * Free all receive rings.
4120 **********************************************************************/
4122 ixgbe_free_receive_structures(struct adapter *adapter)
4124 struct rx_ring *rxr = adapter->rx_rings;
4126 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4128 struct lro_ctrl *lro = &rxr->lro;
4130 ixgbe_free_receive_buffers(rxr);
4131 /* Free LRO memory */
4135 /* Free the ring memory as well */
4136 ixgbe_dma_free(adapter, &rxr->rxdma);
4139 kfree(adapter->rx_rings, M_DEVBUF);
4143 /*********************************************************************
4145 * Free receive ring data structures
4147 **********************************************************************/
4149 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4151 struct adapter *adapter = rxr->adapter;
4152 struct ixgbe_rx_buf *rxbuf;
4154 INIT_DEBUGOUT("free_receive_structures: begin");
4156 /* Cleanup any existing buffers */
4157 if (rxr->rx_buffers != NULL) {
4158 for (int i = 0; i < adapter->num_rx_desc; i++) {
4159 rxbuf = &rxr->rx_buffers[i];
4160 if (rxbuf->m_head != NULL) {
4161 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4162 BUS_DMASYNC_POSTREAD);
4163 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4164 rxbuf->m_head->m_flags |= M_PKTHDR;
4165 m_freem(rxbuf->m_head);
4167 if (rxbuf->m_pack != NULL) {
4168 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4169 BUS_DMASYNC_POSTREAD);
4170 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4171 rxbuf->m_pack->m_flags |= M_PKTHDR;
4172 m_freem(rxbuf->m_pack);
4174 rxbuf->m_head = NULL;
4175 rxbuf->m_pack = NULL;
4176 if (rxbuf->hmap != NULL) {
4177 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4180 if (rxbuf->pmap != NULL) {
4181 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4185 if (rxr->rx_buffers != NULL) {
4186 kfree(rxr->rx_buffers, M_DEVBUF);
4187 rxr->rx_buffers = NULL;
4191 if (rxr->htag != NULL) {
4192 bus_dma_tag_destroy(rxr->htag);
4195 if (rxr->ptag != NULL) {
4196 bus_dma_tag_destroy(rxr->ptag);
4203 static __inline void
4204 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4208 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4209 * should be computed by hardware. Also it should not have VLAN tag in
4210 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4213 if (rxr->lro_enabled &&
4214 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4215 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4216 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4217 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4218 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4219 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4220 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4221 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4223 * Send to the stack if:
4224 ** - LRO not enabled, or
4225 ** - no LRO resources, or
4226 ** - lro enqueue fails
4228 if (rxr->lro.lro_cnt != 0)
4229 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4233 IXGBE_RX_UNLOCK(rxr);
4234 (*ifp->if_input)(ifp, m);
4238 static __inline void
4239 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4241 struct ixgbe_rx_buf *rbuf;
4243 rbuf = &rxr->rx_buffers[i];
4245 if (rbuf->fmp != NULL) {/* Partial chain ? */
4246 rbuf->fmp->m_flags |= M_PKTHDR;
4252 ** With advanced descriptors the writeback
4253 ** clobbers the buffer addrs, so its easier
4254 ** to just free the existing mbufs and take
4255 ** the normal refresh path to get new buffers
4259 m_free(rbuf->m_head);
4260 rbuf->m_head = NULL;
4264 m_free(rbuf->m_pack);
4265 rbuf->m_pack = NULL;
4272 ixgbe_add_sysctl(struct adapter *adapter)
4274 sysctl_ctx_init(&adapter->sysctl_ctx);
4275 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4276 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4277 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
4278 if (adapter->sysctl_tree == NULL) {
4279 device_printf(adapter->dev, "can't add sysctl node\n");
4282 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4283 SYSCTL_CHILDREN(adapter->sysctl_tree),
4284 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
4285 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
4287 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4288 SYSCTL_CHILDREN(adapter->sysctl_tree),
4289 OID_AUTO, "intr_rate", CTLTYPE_INT | CTLFLAG_RW,
4290 adapter, 0, ixgbe_sysctl_intr_rate, "I", "interrupt rate");
4293 ** Allow a kind of speed control by forcing the autoneg
4294 ** advertised speed list to only a certain value, this
4295 ** supports 1G on 82599 devices, and 100Mb on x540.
4297 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4298 SYSCTL_CHILDREN(adapter->sysctl_tree),
4299 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
4300 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
4302 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4303 SYSCTL_CHILDREN(adapter->sysctl_tree),
4304 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
4305 0, ixgbe_set_thermal_test, "I", "Thermal Test");
4307 /* Sysctl for limiting the amount of work done in the taskqueue */
4308 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
4309 "max number of rx packets to process", &adapter->rx_process_limit,
4310 ixgbe_rx_process_limit);
4313 /*********************************************************************
4315 * This routine executes in interrupt context. It replenishes
4316 * the mbufs in the descriptor and sends data which has been
4317 * dma'ed into host memory to upper layer.
4319 * We loop at most count times if count is > 0, or until done if
4322 * Return TRUE for more work, FALSE for all clean.
4323 *********************************************************************/
4325 ixgbe_rxeof(struct ix_queue *que, int count)
4327 struct adapter *adapter = que->adapter;
4328 struct rx_ring *rxr = que->rxr;
4329 struct ifnet *ifp = adapter->ifp;
4331 struct lro_ctrl *lro = &rxr->lro;
4332 struct lro_entry *queued;
4334 int i, nextp, processed = 0;
4336 union ixgbe_adv_rx_desc *cur;
4337 struct ixgbe_rx_buf *rbuf, *nbuf;
4342 if (ifp->if_capenable & IFCAP_NETMAP) {
4344 * Same as the txeof routine: only wakeup clients on intr.
4345 * NKR_PENDINTR in nr_kflags is used to implement interrupt
4346 * mitigation (ixgbe_rxsync() will not look for new packets
4347 * unless NKR_PENDINTR is set).
4349 struct netmap_adapter *na = NA(ifp);
4351 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4352 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4353 IXGBE_RX_UNLOCK(rxr);
4354 IXGBE_CORE_LOCK(adapter);
4355 selwakeuppri(&na->rx_si, PI_NET);
4356 IXGBE_CORE_UNLOCK(adapter);
4359 #endif /* DEV_NETMAP */
4360 for (i = rxr->next_to_check; count != 0;) {
4361 struct mbuf *sendmp, *mh, *mp;
4363 u16 hlen, plen, hdr;
4367 /* Sync the ring. */
4368 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4369 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4371 cur = &rxr->rx_base[i];
4372 staterr = le32toh(cur->wb.upper.status_error);
4374 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4376 if ((ifp->if_flags & IFF_RUNNING) == 0)
4383 cur->wb.upper.status_error = 0;
4384 rbuf = &rxr->rx_buffers[i];
4388 plen = le16toh(cur->wb.upper.length);
4389 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4390 IXGBE_RXDADV_PKTTYPE_MASK;
4391 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4392 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4394 /* Process vlan info */
4395 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4396 vtag = le16toh(cur->wb.upper.vlan);
4398 /* Make sure bad packets are discarded */
4399 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4401 IFNET_STAT_INC(ifp, ierrors, 1);
4402 rxr->rx_discarded++;
4404 rxr->discard = FALSE;
4406 rxr->discard = TRUE;
4407 ixgbe_rx_discard(rxr, i);
4412 ** On 82599 which supports a hardware
4413 ** LRO (called HW RSC), packets need
4414 ** not be fragmented across sequential
4415 ** descriptors, rather the next descriptor
4416 ** is indicated in bits of the descriptor.
4417 ** This also means that we might proceses
4418 ** more than one packet at a time, something
4419 ** that has never been true before, it
4420 ** required eliminating global chain pointers
4421 ** in favor of what we are doing here. -jfv
4425 ** Figure out the next descriptor
4428 if (rxr->hw_rsc == TRUE) {
4429 rsc = ixgbe_rsc_count(cur);
4430 rxr->rsc_num += (rsc - 1);
4432 if (rsc) { /* Get hardware index */
4434 IXGBE_RXDADV_NEXTP_MASK) >>
4435 IXGBE_RXDADV_NEXTP_SHIFT);
4436 } else { /* Just sequential */
4438 if (nextp == adapter->num_rx_desc)
4441 nbuf = &rxr->rx_buffers[nextp];
4445 ** The header mbuf is ONLY used when header
4446 ** split is enabled, otherwise we get normal
4447 ** behavior, ie, both header and payload
4448 ** are DMA'd into the payload buffer.
4450 ** Rather than using the fmp/lmp global pointers
4451 ** we now keep the head of a packet chain in the
4452 ** buffer struct and pass this along from one
4453 ** descriptor to the next, until we get EOP.
4455 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4456 /* This must be an initial descriptor */
4457 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4458 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4459 if (hlen > IXGBE_RX_HDR)
4460 hlen = IXGBE_RX_HDR;
4462 mh->m_flags |= M_PKTHDR;
4464 mh->m_pkthdr.len = mh->m_len;
4465 /* Null buf pointer so it is refreshed */
4466 rbuf->m_head = NULL;
4468 ** Check the payload length, this
4469 ** could be zero if its a small
4475 mp->m_flags &= ~M_PKTHDR;
4477 mh->m_pkthdr.len += mp->m_len;
4478 /* Null buf pointer so it is refreshed */
4479 rbuf->m_pack = NULL;
4480 rxr->rx_split_packets++;
4483 ** Now create the forward
4484 ** chain so when complete
4488 /* stash the chain head */
4490 /* Make forward chain */
4492 mp->m_next = nbuf->m_pack;
4494 mh->m_next = nbuf->m_pack;
4496 /* Singlet, prepare to send */
4498 /* If hardware handled vtag */
4500 sendmp->m_pkthdr.ether_vlantag = vtag;
4501 sendmp->m_flags |= M_VLANTAG;
4506 ** Either no header split, or a
4507 ** secondary piece of a fragmented
4512 ** See if there is a stored head
4513 ** that determines what we are
4516 rbuf->m_pack = rbuf->fmp = NULL;
4518 if (sendmp != NULL) { /* secondary frag */
4519 mp->m_flags &= ~M_PKTHDR;
4520 sendmp->m_pkthdr.len += mp->m_len;
4522 /* first desc of a non-ps chain */
4524 sendmp->m_flags |= M_PKTHDR;
4525 sendmp->m_pkthdr.len = mp->m_len;
4526 if (staterr & IXGBE_RXD_STAT_VP) {
4527 sendmp->m_pkthdr.ether_vlantag = vtag;
4528 sendmp->m_flags |= M_VLANTAG;
4531 /* Pass the head pointer on */
4535 mp->m_next = nbuf->m_pack;
4539 /* Sending this frame? */
4541 sendmp->m_pkthdr.rcvif = ifp;
4542 IFNET_STAT_INC(ifp, ipackets, 1);
4544 /* capture data for AIM */
4545 rxr->bytes += sendmp->m_pkthdr.len;
4546 rxr->rx_bytes += sendmp->m_pkthdr.len;
4547 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4548 ixgbe_rx_checksum(staterr, sendmp, ptype);
4549 #if 0 /* __FreeBSD_version >= 800000 */
4550 sendmp->m_pkthdr.flowid = que->msix;
4551 sendmp->m_flags |= M_FLOWID;
4555 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4556 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4558 /* Advance our pointers to the next descriptor. */
4559 if (++i == adapter->num_rx_desc)
4562 /* Now send to the stack or do LRO */
4563 if (sendmp != NULL) {
4564 rxr->next_to_check = i;
4565 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4566 i = rxr->next_to_check;
4569 /* Every 8 descriptors we go to refresh mbufs */
4570 if (processed == 8) {
4571 ixgbe_refresh_mbufs(rxr, i);
4576 /* Refresh any remaining buf structs */
4577 if (ixgbe_rx_unrefreshed(rxr))
4578 ixgbe_refresh_mbufs(rxr, i);
4580 rxr->next_to_check = i;
4583 * Flush any outstanding LRO work
4586 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4587 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4588 tcp_lro_flush(lro, queued);
4592 IXGBE_RX_UNLOCK(rxr);
4595 ** We still have cleaning to do?
4596 ** Schedule another interrupt if so.
4598 if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4599 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4604 /*********************************************************************
4606 * Verify that the hardware indicated that the checksum is valid.
4607 * Inform the stack about the status of checksum so that stack
4608 * doesn't spend time verifying the checksum.
4610 *********************************************************************/
4612 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4614 u16 status = (u16) staterr;
4615 u8 errors = (u8) (staterr >> 24);
4618 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4619 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4622 if (status & IXGBE_RXD_STAT_IPCS) {
4623 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4624 /* IP Checksum Good */
4625 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4626 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4629 mp->m_pkthdr.csum_flags = 0;
4631 if (status & IXGBE_RXD_STAT_L4CS) {
4632 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4635 type = CSUM_SCTP_VALID;
4637 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4638 mp->m_pkthdr.csum_flags |= type;
4640 mp->m_pkthdr.csum_data = htons(0xffff);
4648 ** This routine is run via an vlan config EVENT,
4649 ** it enables us to use the HW Filter table since
4650 ** we can get the vlan id. This just creates the
4651 ** entry in the soft version of the VFTA, init will
4652 ** repopulate the real table.
4655 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4657 struct adapter *adapter = ifp->if_softc;
4660 if (ifp->if_softc != arg) /* Not our event */
4663 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4666 IXGBE_CORE_LOCK(adapter);
4667 index = (vtag >> 5) & 0x7F;
4669 adapter->shadow_vfta[index] |= (1 << bit);
4670 ++adapter->num_vlans;
4671 ixgbe_init_locked(adapter);
4672 IXGBE_CORE_UNLOCK(adapter);
4676 ** This routine is run via an vlan
4677 ** unconfig EVENT, remove our entry
4678 ** in the soft vfta.
4681 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4683 struct adapter *adapter = ifp->if_softc;
4686 if (ifp->if_softc != arg)
4689 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4692 IXGBE_CORE_LOCK(adapter);
4693 index = (vtag >> 5) & 0x7F;
4695 adapter->shadow_vfta[index] &= ~(1 << bit);
4696 --adapter->num_vlans;
4697 /* Re-init to load the changes */
4698 ixgbe_init_locked(adapter);
4699 IXGBE_CORE_UNLOCK(adapter);
4703 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4705 struct ifnet *ifp = adapter->ifp;
4706 struct ixgbe_hw *hw = &adapter->hw;
4707 struct rx_ring *rxr;
4711 ** We get here thru init_locked, meaning
4712 ** a soft reset, this has already cleared
4713 ** the VFTA and other state, so if there
4714 ** have been no vlan's registered do nothing.
4716 if (adapter->num_vlans == 0)
4720 ** A soft reset zero's out the VFTA, so
4721 ** we need to repopulate it now.
4723 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4724 if (adapter->shadow_vfta[i] != 0)
4725 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4726 adapter->shadow_vfta[i]);
4728 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4729 /* Enable the Filter Table if enabled */
4730 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4731 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4732 ctrl |= IXGBE_VLNCTRL_VFE;
4734 if (hw->mac.type == ixgbe_mac_82598EB)
4735 ctrl |= IXGBE_VLNCTRL_VME;
4736 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4738 /* Setup the queues for vlans */
4739 for (int i = 0; i < adapter->num_queues; i++) {
4740 rxr = &adapter->rx_rings[i];
4741 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4742 if (hw->mac.type != ixgbe_mac_82598EB) {
4743 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4744 ctrl |= IXGBE_RXDCTL_VME;
4745 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4747 rxr->vtag_strip = TRUE;
4752 ixgbe_enable_intr(struct adapter *adapter)
4754 struct ixgbe_hw *hw = &adapter->hw;
4755 struct ix_queue *que = adapter->queues;
4756 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4759 /* Enable Fan Failure detection */
4760 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4761 mask |= IXGBE_EIMS_GPI_SDP1;
4763 mask |= IXGBE_EIMS_ECC;
4764 mask |= IXGBE_EIMS_GPI_SDP0;
4765 mask |= IXGBE_EIMS_GPI_SDP1;
4766 mask |= IXGBE_EIMS_GPI_SDP2;
4768 mask |= IXGBE_EIMS_FLOW_DIR;
4772 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4774 /* With RSS we use auto clear */
4775 if (adapter->msix_mem) {
4776 mask = IXGBE_EIMS_ENABLE_MASK;
4777 /* Don't autoclear Link */
4778 mask &= ~IXGBE_EIMS_OTHER;
4779 mask &= ~IXGBE_EIMS_LSC;
4780 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4784 ** Now enable all queues, this is done separately to
4785 ** allow for handling the extended (beyond 32) MSIX
4786 ** vectors that can be used by 82599
4788 for (int i = 0; i < adapter->num_queues; i++, que++)
4789 ixgbe_enable_queue(adapter, que->msix);
4791 IXGBE_WRITE_FLUSH(hw);
4797 ixgbe_disable_intr(struct adapter *adapter)
4799 if (adapter->msix_mem)
4800 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4801 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4802 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4804 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4805 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4806 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4808 IXGBE_WRITE_FLUSH(&adapter->hw);
4813 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4817 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4824 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4826 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4833 ** Setup the correct IVAR register for a particular MSIX interrupt
4834 ** (yes this is all very magic and confusing :)
4835 ** - entry is the register array entry
4836 ** - vector is the MSIX vector for this queue
4837 ** - type is RX/TX/MISC
4840 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4842 struct ixgbe_hw *hw = &adapter->hw;
4845 vector |= IXGBE_IVAR_ALLOC_VAL;
4847 switch (hw->mac.type) {
4849 case ixgbe_mac_82598EB:
4851 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4853 entry += (type * 64);
4854 index = (entry >> 2) & 0x1F;
4855 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4856 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4857 ivar |= (vector << (8 * (entry & 0x3)));
4858 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
4861 case ixgbe_mac_82599EB:
4862 case ixgbe_mac_X540:
4863 if (type == -1) { /* MISC IVAR */
4864 index = (entry & 1) * 8;
4865 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
4866 ivar &= ~(0xFF << index);
4867 ivar |= (vector << index);
4868 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
4869 } else { /* RX/TX IVARS */
4870 index = (16 * (entry & 1)) + (8 * type);
4871 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
4872 ivar &= ~(0xFF << index);
4873 ivar |= (vector << index);
4874 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
4883 ixgbe_configure_ivars(struct adapter *adapter)
4885 struct ix_queue *que = adapter->queues;
4887 for (int i = 0; i < adapter->num_queues; i++, que++) {
4888 /* First the RX queue entry */
4889 ixgbe_set_ivar(adapter, i, que->msix, 0);
4890 /* ... and the TX */
4891 ixgbe_set_ivar(adapter, i, que->msix, 1);
4892 /* Set an Initial EITR value */
4893 ixgbe_set_eitr(adapter, que->msix, IXGBE_INTR_RATE);
4896 /* For the Link interrupt */
4897 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
4901 ** ixgbe_sfp_probe - called in the local timer to
4902 ** determine if a port had optics inserted.
4904 static bool ixgbe_sfp_probe(struct adapter *adapter)
4906 struct ixgbe_hw *hw = &adapter->hw;
4907 device_t dev = adapter->dev;
4908 bool result = FALSE;
4910 if ((hw->phy.type == ixgbe_phy_nl) &&
4911 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
4912 s32 ret = hw->phy.ops.identify_sfp(hw);
4915 ret = hw->phy.ops.reset(hw);
4916 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4917 device_printf(dev,"Unsupported SFP+ module detected!");
4918 kprintf(" Reload driver with supported module.\n");
4919 adapter->sfp_probe = FALSE;
4922 device_printf(dev,"SFP+ module detected!\n");
4923 /* We now have supported optics */
4924 adapter->sfp_probe = FALSE;
4925 /* Set the optics type so system reports correctly */
4926 ixgbe_setup_optics(adapter);
4934 ** Tasklet for handling SFP module interrupts
4937 ixgbe_handle_mod(void *context, int pending)
4939 struct adapter *adapter = context;
4940 struct ixgbe_hw *hw = &adapter->hw;
4941 device_t dev = adapter->dev;
4944 err = hw->phy.ops.identify_sfp(hw);
4945 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4947 "Unsupported SFP+ module type was detected.\n");
4950 err = hw->mac.ops.setup_sfp(hw);
4951 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
4953 "Setup failure - unsupported SFP+ module type.\n");
4956 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
4962 ** Tasklet for handling MSF (multispeed fiber) interrupts
4965 ixgbe_handle_msf(void *context, int pending)
4967 struct adapter *adapter = context;
4968 struct ixgbe_hw *hw = &adapter->hw;
4972 autoneg = hw->phy.autoneg_advertised;
4973 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
4974 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
4975 if (hw->mac.ops.setup_link)
4976 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
4982 ** Tasklet for reinitializing the Flow Director filter table
4985 ixgbe_reinit_fdir(void *context, int pending)
4987 struct adapter *adapter = context;
4988 struct ifnet *ifp = adapter->ifp;
4990 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
4992 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
4993 adapter->fdir_reinit = 0;
4994 /* re-enable flow director interrupts */
4995 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
4996 /* Restart the interface */
4997 ifp->if_drv_flags |= IFF_DRV_RUNNING;
5002 /**********************************************************************
5004 * Update the board statistics counters.
5006 **********************************************************************/
5008 ixgbe_update_stats_counters(struct adapter *adapter)
5010 struct ifnet *ifp = adapter->ifp;
5011 struct ixgbe_hw *hw = &adapter->hw;
5012 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5013 u64 total_missed_rx = 0;
5015 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5016 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5017 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5018 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5020 for (int i = 0; i < 8; i++) {
5022 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5023 /* missed_rx tallies misses for the gprc workaround */
5025 /* global total per queue */
5026 adapter->stats.mpc[i] += mp;
5027 /* Running comprehensive total for stats display */
5028 total_missed_rx += adapter->stats.mpc[i];
5029 if (hw->mac.type == ixgbe_mac_82598EB)
5030 adapter->stats.rnbc[i] +=
5031 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5032 adapter->stats.pxontxc[i] +=
5033 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5034 adapter->stats.pxonrxc[i] +=
5035 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5036 adapter->stats.pxofftxc[i] +=
5037 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5038 adapter->stats.pxoffrxc[i] +=
5039 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5040 adapter->stats.pxon2offc[i] +=
5041 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5043 for (int i = 0; i < 16; i++) {
5044 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5045 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5046 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5047 adapter->stats.qbrc[i] +=
5048 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5049 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5050 adapter->stats.qbtc[i] +=
5051 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5052 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5054 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5055 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5056 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5058 /* Hardware workaround, gprc counts missed packets */
5059 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5060 adapter->stats.gprc -= missed_rx;
5062 if (hw->mac.type != ixgbe_mac_82598EB) {
5063 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5064 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5065 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5066 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5067 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5068 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5069 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5070 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5072 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5073 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5074 /* 82598 only has a counter in the high register */
5075 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5076 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5077 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5081 * Workaround: mprc hardware is incorrectly counting
5082 * broadcasts, so for now we subtract those.
5084 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5085 adapter->stats.bprc += bprc;
5086 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5087 if (hw->mac.type == ixgbe_mac_82598EB)
5088 adapter->stats.mprc -= bprc;
5090 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5091 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5092 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5093 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5094 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5095 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5097 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5098 adapter->stats.lxontxc += lxon;
5099 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5100 adapter->stats.lxofftxc += lxoff;
5101 total = lxon + lxoff;
5103 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5104 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5105 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5106 adapter->stats.gptc -= total;
5107 adapter->stats.mptc -= total;
5108 adapter->stats.ptc64 -= total;
5109 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5111 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5112 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5113 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5114 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5115 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5116 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5117 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5118 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5119 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5120 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5121 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5122 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5123 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5124 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5125 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5126 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5127 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5128 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5129 /* Only read FCOE on 82599 */
5130 if (hw->mac.type != ixgbe_mac_82598EB) {
5131 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5132 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5133 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5134 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5135 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5138 /* Fill out the OS statistics structure */
5139 IFNET_STAT_SET(ifp, ipackets, adapter->stats.gprc);
5140 IFNET_STAT_SET(ifp, opackets, adapter->stats.gptc);
5141 IFNET_STAT_SET(ifp, ibytes, adapter->stats.gorc);
5142 IFNET_STAT_SET(ifp, obytes, adapter->stats.gotc);
5143 IFNET_STAT_SET(ifp, imcasts, adapter->stats.mprc);
5146 IFNET_STAT_SET(ifp, ierrors, total_missed_rx + adapter->stats.crcerrs +
5147 adapter->stats.rlec);
5150 /** ixgbe_sysctl_tdh_handler - Handler function
5151 * Retrieves the TDH value from the hardware
5154 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5158 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5161 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5162 error = sysctl_handle_int(oidp, &val, 0, req);
5163 if (error || !req->newptr)
5168 /** ixgbe_sysctl_tdt_handler - Handler function
5169 * Retrieves the TDT value from the hardware
5172 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5176 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5179 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5180 error = sysctl_handle_int(oidp, &val, 0, req);
5181 if (error || !req->newptr)
5186 /** ixgbe_sysctl_rdh_handler - Handler function
5187 * Retrieves the RDH value from the hardware
5190 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5194 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5197 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5198 error = sysctl_handle_int(oidp, &val, 0, req);
5199 if (error || !req->newptr)
5204 /** ixgbe_sysctl_rdt_handler - Handler function
5205 * Retrieves the RDT value from the hardware
5208 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5212 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5215 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5216 error = sysctl_handle_int(oidp, &val, 0, req);
5217 if (error || !req->newptr)
5223 * Add sysctl variables, one per statistic, to the system.
5226 ixgbe_add_hw_stats(struct adapter *adapter)
5228 struct tx_ring *txr = adapter->tx_rings;
5229 struct rx_ring *rxr = adapter->rx_rings;
5231 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5232 struct sysctl_oid *tree = adapter->sysctl_tree;
5233 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5234 struct ixgbe_hw_stats *stats = &adapter->stats;
5236 struct sysctl_oid *stat_node, *queue_node;
5237 struct sysctl_oid_list *stat_list, *queue_list;
5239 #define QUEUE_NAME_LEN 32
5240 char namebuf[QUEUE_NAME_LEN];
5242 /* Driver Statistics */
5243 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5244 CTLFLAG_RD, &adapter->dropped_pkts,
5245 "Driver dropped packets");
5246 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5247 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5248 "m_defrag() failed");
5249 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5250 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5251 "Driver tx dma failure in xmit");
5252 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5253 CTLFLAG_RD, &adapter->watchdog_events,
5254 "Watchdog timeouts");
5255 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5256 CTLFLAG_RD, &adapter->tso_tx,
5258 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5259 CTLFLAG_RD, &adapter->link_irq,
5260 "Link MSIX IRQ Handled");
5262 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5263 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5264 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5265 CTLFLAG_RD, NULL, "Queue Name");
5266 queue_list = SYSCTL_CHILDREN(queue_node);
5267 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5268 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5269 "irqs on this queue");
5270 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5271 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5272 ixgbe_sysctl_tdh_handler, "IU",
5273 "Transmit Descriptor Head");
5274 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5275 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5276 ixgbe_sysctl_tdt_handler, "IU",
5277 "Transmit Descriptor Tail");
5278 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5279 CTLFLAG_RD, &txr->no_desc_avail, 0,
5280 "Queue No Descriptor Available");
5281 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5282 CTLFLAG_RD, &txr->total_packets, 0,
5283 "Queue Packets Transmitted");
5286 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5287 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5288 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5289 CTLFLAG_RD, NULL, "Queue Name");
5290 queue_list = SYSCTL_CHILDREN(queue_node);
5293 struct lro_ctrl *lro = &rxr->lro;
5296 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5297 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5298 CTLFLAG_RD, NULL, "Queue Name");
5299 queue_list = SYSCTL_CHILDREN(queue_node);
5301 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5302 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5303 ixgbe_sysctl_rdh_handler, "IU",
5304 "Receive Descriptor Head");
5305 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5306 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5307 ixgbe_sysctl_rdt_handler, "IU",
5308 "Receive Descriptor Tail");
5309 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5310 CTLFLAG_RD, &rxr->rx_packets, 0,
5311 "Queue Packets Received");
5312 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5313 CTLFLAG_RD, &rxr->rx_bytes, 0,
5314 "Queue Bytes Received");
5316 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5317 CTLFLAG_RD, &lro->lro_queued, 0,
5319 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5320 CTLFLAG_RD, &lro->lro_flushed, 0,
5325 /* MAC stats get the own sub node */
5327 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5328 CTLFLAG_RD, NULL, "MAC Statistics");
5329 stat_list = SYSCTL_CHILDREN(stat_node);
5331 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5332 CTLFLAG_RD, &stats->crcerrs, 0,
5334 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5335 CTLFLAG_RD, &stats->illerrc, 0,
5336 "Illegal Byte Errors");
5337 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5338 CTLFLAG_RD, &stats->errbc, 0,
5340 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5341 CTLFLAG_RD, &stats->mspdc, 0,
5342 "MAC Short Packets Discarded");
5343 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5344 CTLFLAG_RD, &stats->mlfc, 0,
5345 "MAC Local Faults");
5346 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5347 CTLFLAG_RD, &stats->mrfc, 0,
5348 "MAC Remote Faults");
5349 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5350 CTLFLAG_RD, &stats->rlec, 0,
5351 "Receive Length Errors");
5352 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5353 CTLFLAG_RD, &stats->lxontxc, 0,
5354 "Link XON Transmitted");
5355 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5356 CTLFLAG_RD, &stats->lxonrxc, 0,
5357 "Link XON Received");
5358 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5359 CTLFLAG_RD, &stats->lxofftxc, 0,
5360 "Link XOFF Transmitted");
5361 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5362 CTLFLAG_RD, &stats->lxoffrxc, 0,
5363 "Link XOFF Received");
5365 /* Packet Reception Stats */
5366 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5367 CTLFLAG_RD, &stats->tor, 0,
5368 "Total Octets Received");
5369 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5370 CTLFLAG_RD, &stats->gorc, 0,
5371 "Good Octets Received");
5372 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5373 CTLFLAG_RD, &stats->tpr, 0,
5374 "Total Packets Received");
5375 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5376 CTLFLAG_RD, &stats->gprc, 0,
5377 "Good Packets Received");
5378 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5379 CTLFLAG_RD, &stats->mprc, 0,
5380 "Multicast Packets Received");
5381 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5382 CTLFLAG_RD, &stats->bprc, 0,
5383 "Broadcast Packets Received");
5384 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5385 CTLFLAG_RD, &stats->prc64, 0,
5386 "64 byte frames received ");
5387 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5388 CTLFLAG_RD, &stats->prc127, 0,
5389 "65-127 byte frames received");
5390 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5391 CTLFLAG_RD, &stats->prc255, 0,
5392 "128-255 byte frames received");
5393 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5394 CTLFLAG_RD, &stats->prc511, 0,
5395 "256-511 byte frames received");
5396 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5397 CTLFLAG_RD, &stats->prc1023, 0,
5398 "512-1023 byte frames received");
5399 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5400 CTLFLAG_RD, &stats->prc1522, 0,
5401 "1023-1522 byte frames received");
5402 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5403 CTLFLAG_RD, &stats->ruc, 0,
5404 "Receive Undersized");
5405 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5406 CTLFLAG_RD, &stats->rfc, 0,
5407 "Fragmented Packets Received ");
5408 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5409 CTLFLAG_RD, &stats->roc, 0,
5410 "Oversized Packets Received");
5411 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5412 CTLFLAG_RD, &stats->rjc, 0,
5414 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5415 CTLFLAG_RD, &stats->mngprc, 0,
5416 "Management Packets Received");
5417 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5418 CTLFLAG_RD, &stats->mngptc, 0,
5419 "Management Packets Dropped");
5420 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5421 CTLFLAG_RD, &stats->xec, 0,
5424 /* Packet Transmission Stats */
5425 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5426 CTLFLAG_RD, &stats->gotc, 0,
5427 "Good Octets Transmitted");
5428 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5429 CTLFLAG_RD, &stats->tpt, 0,
5430 "Total Packets Transmitted");
5431 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5432 CTLFLAG_RD, &stats->gptc, 0,
5433 "Good Packets Transmitted");
5434 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5435 CTLFLAG_RD, &stats->bptc, 0,
5436 "Broadcast Packets Transmitted");
5437 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5438 CTLFLAG_RD, &stats->mptc, 0,
5439 "Multicast Packets Transmitted");
5440 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5441 CTLFLAG_RD, &stats->mngptc, 0,
5442 "Management Packets Transmitted");
5443 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5444 CTLFLAG_RD, &stats->ptc64, 0,
5445 "64 byte frames transmitted ");
5446 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5447 CTLFLAG_RD, &stats->ptc127, 0,
5448 "65-127 byte frames transmitted");
5449 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5450 CTLFLAG_RD, &stats->ptc255, 0,
5451 "128-255 byte frames transmitted");
5452 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5453 CTLFLAG_RD, &stats->ptc511, 0,
5454 "256-511 byte frames transmitted");
5455 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5456 CTLFLAG_RD, &stats->ptc1023, 0,
5457 "512-1023 byte frames transmitted");
5458 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5459 CTLFLAG_RD, &stats->ptc1522, 0,
5460 "1024-1522 byte frames transmitted");
5463 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5464 CTLFLAG_RD, &stats->fccrc, 0,
5466 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5467 CTLFLAG_RD, &stats->fclast, 0,
5469 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5470 CTLFLAG_RD, &stats->fcoerpdc, 0,
5471 "FCoE Packets Dropped");
5472 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5473 CTLFLAG_RD, &stats->fcoeprc, 0,
5474 "FCoE Packets Received");
5475 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5476 CTLFLAG_RD, &stats->fcoeptc, 0,
5477 "FCoE Packets Transmitted");
5478 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5479 CTLFLAG_RD, &stats->fcoedwrc, 0,
5480 "FCoE DWords Received");
5481 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5482 CTLFLAG_RD, &stats->fcoedwtc, 0,
5483 "FCoE DWords Transmitted");
5487 ** Set flow control using sysctl:
5488 ** Flow control values:
5495 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5498 struct adapter *adapter = (struct adapter *) arg1;
5501 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5502 if ((error) || (req->newptr == NULL))
5505 /* Don't bother if it's not changed */
5506 if (adapter->fc == last)
5509 switch (adapter->fc) {
5510 case ixgbe_fc_rx_pause:
5511 case ixgbe_fc_tx_pause:
5513 adapter->hw.fc.requested_mode = adapter->fc;
5517 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5519 /* Don't autoneg if forcing a value */
5520 adapter->hw.fc.disable_fc_autoneg = TRUE;
5521 ixgbe_fc_enable(&adapter->hw);
5526 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5527 const char *description, int *limit, int value)
5530 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5531 SYSCTL_CHILDREN(adapter->sysctl_tree),
5532 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5536 ** Control link advertise speed:
5537 ** 1 - advertise only 1G
5538 ** 2 - advertise 100Mb
5539 ** 3 - advertise normal
5542 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5545 struct adapter *adapter;
5547 struct ixgbe_hw *hw;
5548 ixgbe_link_speed speed, last;
5550 adapter = (struct adapter *) arg1;
5553 last = adapter->advertise;
5555 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5556 if ((error) || (adapter->advertise == -1))
5559 if (adapter->advertise == last) /* no change */
5562 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5563 (hw->phy.multispeed_fiber)))
5566 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5567 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5571 if (adapter->advertise == 1)
5572 speed = IXGBE_LINK_SPEED_1GB_FULL;
5573 else if (adapter->advertise == 2)
5574 speed = IXGBE_LINK_SPEED_100_FULL;
5575 else if (adapter->advertise == 3)
5576 speed = IXGBE_LINK_SPEED_1GB_FULL |
5577 IXGBE_LINK_SPEED_10GB_FULL;
5578 else /* bogus value */
5581 hw->mac.autotry_restart = TRUE;
5582 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5588 ** Thermal Shutdown Trigger
5589 ** - cause a Thermal Overtemp IRQ
5592 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5594 int error, fire = 0;
5595 struct adapter *adapter = (struct adapter *) arg1;
5596 struct ixgbe_hw *hw = &adapter->hw;
5599 if (hw->mac.type != ixgbe_mac_X540)
5602 error = sysctl_handle_int(oidp, &fire, 0, req);
5603 if ((error) || (req->newptr == NULL))
5607 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5608 reg |= IXGBE_EICR_TS;
5609 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5616 ixgbe_set_eitr(struct adapter *sc, int idx, int rate)
5620 /* convert rate in intr/s to hw representation */
5622 eitr = 1000000 / rate;
5623 eitr <<= IXGBE_EITR_INTVL_SHIFT;
5626 /* Don't disable it */
5627 eitr = 1 << IXGBE_EITR_INTVL_SHIFT;
5628 } else if (eitr > IXGBE_EITR_INTVL_MASK) {
5629 /* Don't allow it to be too large */
5630 eitr = IXGBE_EITR_INTVL_MASK;
5633 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(idx), eitr);
5637 ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS)
5639 struct adapter *sc = (void *)arg1;
5640 struct ifnet *ifp = sc->ifp;
5641 int error, intr_rate, running;
5642 struct ix_queue *que = sc->queues;
5644 intr_rate = sc->intr_rate;
5645 error = sysctl_handle_int(oidp, &intr_rate, 0, req);
5646 if (error || req->newptr == NULL)
5651 ifnet_serialize_all(ifp);
5653 sc->intr_rate = intr_rate;
5654 running = ifp->if_flags & IFF_RUNNING;
5656 ixgbe_set_eitr(sc, 0, sc->intr_rate);
5658 if (running && (sc->intr_type == PCI_INTR_TYPE_MSIX)) {
5659 for (int i = 0; i < sc->num_queues; i++, que++)
5660 ixgbe_set_eitr(sc, que->msix, sc->intr_rate);
5664 if_printf(ifp, "interrupt rate set to %d/sec\n", sc->intr_rate);
5666 ifnet_deserialize_all(ifp);
5671 /* rearrange mbuf chain to get contiguous bytes */
5673 ixgbe_tso_pullup(struct tx_ring *txr, struct mbuf **mp)
5675 int hoff, iphlen, thoff;
5679 KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5681 iphlen = m->m_pkthdr.csum_iphlen;
5682 thoff = m->m_pkthdr.csum_thlen;
5683 hoff = m->m_pkthdr.csum_lhlen;
5685 KASSERT(iphlen > 0, ("invalid ip hlen"));
5686 KASSERT(thoff > 0, ("invalid tcp hlen"));
5687 KASSERT(hoff > 0, ("invalid ether hlen"));
5689 if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5690 m = m_pullup(m, hoff + iphlen + thoff);