1 /******************************************************************************
3 Copyright (c) 2001-2012, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
36 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 static void ixgbe_start(struct ifnet *);
106 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111 static void ixgbe_qflush(struct ifnet *);
113 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void ixgbe_init(void *);
115 static void ixgbe_init_locked(struct adapter *);
116 static void ixgbe_stop(void *);
117 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int ixgbe_media_change(struct ifnet *);
119 static void ixgbe_identify_hardware(struct adapter *);
120 static int ixgbe_allocate_pci_resources(struct adapter *);
121 static int ixgbe_allocate_msix(struct adapter *);
122 static int ixgbe_allocate_legacy(struct adapter *);
123 static int ixgbe_allocate_queues(struct adapter *);
124 #if 0 /* HAVE_MSIX */
125 static int ixgbe_setup_msix(struct adapter *);
127 static void ixgbe_free_pci_resources(struct adapter *);
128 static void ixgbe_local_timer(void *);
129 static int ixgbe_setup_interface(device_t, struct adapter *);
130 static void ixgbe_config_link(struct adapter *);
132 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133 static int ixgbe_setup_transmit_structures(struct adapter *);
134 static void ixgbe_setup_transmit_ring(struct tx_ring *);
135 static void ixgbe_initialize_transmit_units(struct adapter *);
136 static void ixgbe_free_transmit_structures(struct adapter *);
137 static void ixgbe_free_transmit_buffers(struct tx_ring *);
139 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140 static int ixgbe_setup_receive_structures(struct adapter *);
141 static int ixgbe_setup_receive_ring(struct rx_ring *);
142 static void ixgbe_initialize_receive_units(struct adapter *);
143 static void ixgbe_free_receive_structures(struct adapter *);
144 static void ixgbe_free_receive_buffers(struct rx_ring *);
146 static void ixgbe_setup_hw_rsc(struct rx_ring *);
149 static void ixgbe_enable_intr(struct adapter *);
150 static void ixgbe_disable_intr(struct adapter *);
151 static void ixgbe_update_stats_counters(struct adapter *);
152 static bool ixgbe_txeof(struct tx_ring *);
153 static bool ixgbe_rxeof(struct ix_queue *, int);
154 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
155 static void ixgbe_set_promisc(struct adapter *);
156 static void ixgbe_set_multi(struct adapter *);
157 static void ixgbe_update_link_status(struct adapter *);
158 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
159 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
160 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
162 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
163 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
164 struct ixgbe_dma_alloc *, int);
165 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
166 static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
167 const char *, int *, int);
168 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
170 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
172 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void ixgbe_configure_ivars(struct adapter *);
174 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
176 static void ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
180 static void ixgbe_add_hw_stats(struct adapter *adapter);
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
186 /* Support for pluggable optic modules */
187 static bool ixgbe_sfp_probe(struct adapter *);
188 static void ixgbe_setup_optics(struct adapter *);
190 /* Legacy (single vector interrupt handler */
191 static void ixgbe_legacy_irq(void *);
193 /* The MSI/X Interrupt handlers */
194 static void ixgbe_msix_que(void *);
195 static void ixgbe_msix_link(void *);
197 /* Deferred interrupt tasklets */
198 static void ixgbe_handle_que(void *, int);
199 static void ixgbe_handle_link(void *, int);
200 static void ixgbe_handle_msf(void *, int);
201 static void ixgbe_handle_mod(void *, int);
204 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
205 static void ixgbe_reinit_fdir(void *, int);
208 /*********************************************************************
209 * FreeBSD Device Interface Entry Points
210 *********************************************************************/
212 static device_method_t ixgbe_methods[] = {
213 /* Device interface */
214 DEVMETHOD(device_probe, ixgbe_probe),
215 DEVMETHOD(device_attach, ixgbe_attach),
216 DEVMETHOD(device_detach, ixgbe_detach),
217 DEVMETHOD(device_shutdown, ixgbe_shutdown),
221 static driver_t ixgbe_driver = {
222 "ix", ixgbe_methods, sizeof(struct adapter),
225 devclass_t ixgbe_devclass;
226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
232 ** TUNEABLE PARAMETERS:
236 ** AIM: Adaptive Interrupt Moderation
237 ** which means that the interrupt rate
238 ** is varied over time based on the
239 ** traffic for that interrupt vector
241 static int ixgbe_enable_aim = TRUE;
242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
247 /* How many packets rxeof tries to clean at a time */
248 static int ixgbe_rx_process_limit = 128;
249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
252 ** Smart speed setting, default to on
253 ** this only works as a compile option
254 ** right now as its during attach, set
255 ** this to 'ixgbe_smart_speed_off' to
258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
260 static int ixgbe_msi_enable = 1;
261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
264 * MSIX should be the default for best performance,
265 * but this allows it to be forced off for testing.
267 static int ixgbe_enable_msix = 1;
268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
271 * Header split: this causes the hardware to DMA
272 * the header into a separate mbuf from the payload,
273 * it can be a performance win in some workloads, but
274 * in others it actually hurts, its off by default.
276 static int ixgbe_header_split = FALSE;
277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
280 * Number of Queues, can be set to 0,
281 * it then autoconfigures based on the
282 * number of cpus with a max of 8. This
283 * can be overriden manually here.
285 static int ixgbe_num_queues = 0;
286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
289 ** Number of TX descriptors per ring,
290 ** setting higher than RX as this seems
291 ** the better performing choice.
293 static int ixgbe_txd = PERFORM_TXD;
294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
296 /* Number of RX descriptors per ring */
297 static int ixgbe_rxd = PERFORM_RXD;
298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
300 /* Keep running tab on them for sanity check */
301 static int ixgbe_total_ports;
305 ** For Flow Director: this is the
306 ** number of TX packets we sample
307 ** for the filter pool, this means
308 ** every 20th packet will be probed.
310 ** This feature can be disabled by
311 ** setting this to 0.
313 static int atr_sample_rate = 20;
315 ** Flow Director actually 'steals'
316 ** part of the packet buffer as its
317 ** filter pool, this variable controls
319 ** 0 = 64K, 1 = 128K, 2 = 256K
321 static int fdir_pballoc = 1;
326 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
327 * be a reference on how to implement netmap support in a driver.
328 * Additional comments are in ixgbe_netmap.h .
330 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
331 * that extend the standard driver.
333 #include <dev/netmap/ixgbe_netmap.h>
334 #endif /* DEV_NETMAP */
336 /*********************************************************************
337 * Device identification routine
339 * ixgbe_probe determines if the driver should be loaded on
340 * adapter based on PCI vendor/device id of the adapter.
342 * return BUS_PROBE_DEFAULT on success, positive on failure
343 *********************************************************************/
346 ixgbe_probe(device_t dev)
348 ixgbe_vendor_info_t *ent;
350 u16 pci_vendor_id = 0;
351 u16 pci_device_id = 0;
352 u16 pci_subvendor_id = 0;
353 u16 pci_subdevice_id = 0;
354 char adapter_name[256];
356 INIT_DEBUGOUT("ixgbe_probe: begin");
358 pci_vendor_id = pci_get_vendor(dev);
359 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
362 pci_device_id = pci_get_device(dev);
363 pci_subvendor_id = pci_get_subvendor(dev);
364 pci_subdevice_id = pci_get_subdevice(dev);
366 ent = ixgbe_vendor_info_array;
367 while (ent->vendor_id != 0) {
368 if ((pci_vendor_id == ent->vendor_id) &&
369 (pci_device_id == ent->device_id) &&
371 ((pci_subvendor_id == ent->subvendor_id) ||
372 (ent->subvendor_id == 0)) &&
374 ((pci_subdevice_id == ent->subdevice_id) ||
375 (ent->subdevice_id == 0))) {
376 ksprintf(adapter_name, "%s, Version - %s",
377 ixgbe_strings[ent->index],
378 ixgbe_driver_version);
379 device_set_desc_copy(dev, adapter_name);
381 return (BUS_PROBE_DEFAULT);
388 /*********************************************************************
389 * Device initialization routine
391 * The attach entry point is called when the driver is being loaded.
392 * This routine identifies the type of hardware, allocates all resources
393 * and initializes the hardware.
395 * return 0 on success, positive on failure
396 *********************************************************************/
399 ixgbe_attach(device_t dev)
401 struct adapter *adapter;
407 INIT_DEBUGOUT("ixgbe_attach: begin");
409 if (resource_disabled("ixgbe", device_get_unit(dev))) {
410 device_printf(dev, "Disabled by device hint\n");
414 /* Allocate, clear, and link in our adapter structure */
415 adapter = device_get_softc(dev);
416 adapter->dev = adapter->osdep.dev = dev;
420 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
424 sysctl_ctx_init(&adapter->sysctl_ctx);
425 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
426 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
427 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
428 if (adapter->sysctl_tree == NULL) {
429 device_printf(adapter->dev, "can't add sysctl node\n");
432 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
433 SYSCTL_CHILDREN(adapter->sysctl_tree),
434 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
435 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
437 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
438 SYSCTL_CHILDREN(adapter->sysctl_tree),
439 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
440 &ixgbe_enable_aim, 1, "Interrupt Moderation");
443 ** Allow a kind of speed control by forcing the autoneg
444 ** advertised speed list to only a certain value, this
445 ** supports 1G on 82599 devices, and 100Mb on x540.
447 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
448 SYSCTL_CHILDREN(adapter->sysctl_tree),
449 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
450 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
452 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
453 SYSCTL_CHILDREN(adapter->sysctl_tree),
454 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
455 0, ixgbe_set_thermal_test, "I", "Thermal Test");
457 /* Set up the timer callout */
458 callout_init_mp(&adapter->timer);
460 /* Determine hardware revision */
461 ixgbe_identify_hardware(adapter);
463 /* Do base PCI setup - map BAR0 */
464 if (ixgbe_allocate_pci_resources(adapter)) {
465 device_printf(dev, "Allocation of PCI resources failed\n");
470 /* Do descriptor calc and sanity checks */
471 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
472 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
473 device_printf(dev, "TXD config issue, using default!\n");
474 adapter->num_tx_desc = DEFAULT_TXD;
476 adapter->num_tx_desc = ixgbe_txd;
479 ** With many RX rings it is easy to exceed the
480 ** system mbuf allocation. Tuning nmbclusters
481 ** can alleviate this.
483 if (nmbclusters > 0 ) {
485 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
486 if (s > nmbclusters) {
487 device_printf(dev, "RX Descriptors exceed "
488 "system mbuf max, using default instead!\n");
489 ixgbe_rxd = DEFAULT_RXD;
493 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
494 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
495 device_printf(dev, "RXD config issue, using default!\n");
496 adapter->num_rx_desc = DEFAULT_RXD;
498 adapter->num_rx_desc = ixgbe_rxd;
500 /* Allocate our TX/RX Queues */
501 if (ixgbe_allocate_queues(adapter)) {
506 /* Allocate multicast array memory. */
507 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
508 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
509 if (adapter->mta == NULL) {
510 device_printf(dev, "Can not allocate multicast setup array\n");
515 /* Initialize the shared code */
516 error = ixgbe_init_shared_code(hw);
517 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
519 ** No optics in this port, set up
520 ** so the timer routine will probe
521 ** for later insertion.
523 adapter->sfp_probe = TRUE;
525 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
526 device_printf(dev,"Unsupported SFP+ module detected!\n");
530 device_printf(dev,"Unable to initialize the shared code\n");
535 /* Make sure we have a good EEPROM before we read from it */
536 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
537 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
542 error = ixgbe_init_hw(hw);
544 case IXGBE_ERR_EEPROM_VERSION:
545 device_printf(dev, "This device is a pre-production adapter/"
546 "LOM. Please be aware there may be issues associated "
547 "with your hardware.\n If you are experiencing problems "
548 "please contact your Intel or hardware representative "
549 "who provided you with this hardware.\n");
551 case IXGBE_ERR_SFP_NOT_SUPPORTED:
552 device_printf(dev,"Unsupported SFP+ Module\n");
554 device_printf(dev,"Hardware Initialization Failure\n");
556 case IXGBE_ERR_SFP_NOT_PRESENT:
557 device_printf(dev,"No SFP+ Module found\n");
563 /* Detect and set physical type */
564 ixgbe_setup_optics(adapter);
566 if ((adapter->msix > 1) && (ixgbe_enable_msix))
567 error = ixgbe_allocate_msix(adapter);
569 error = ixgbe_allocate_legacy(adapter);
573 /* Setup OS specific network interface */
574 if (ixgbe_setup_interface(dev, adapter) != 0)
577 /* Sysctl for limiting the amount of work done in the taskqueue */
578 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
579 "max number of rx packets to process", &adapter->rx_process_limit,
580 ixgbe_rx_process_limit);
582 /* Initialize statistics */
583 ixgbe_update_stats_counters(adapter);
585 /* Register for VLAN events */
586 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
587 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
588 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
589 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
591 /* Print PCIE bus type/speed/width info */
592 ixgbe_get_bus_info(hw);
593 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
594 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
595 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
596 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
597 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
598 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
601 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
602 (hw->bus.speed == ixgbe_bus_speed_2500)) {
603 device_printf(dev, "PCI-Express bandwidth available"
604 " for this card\n is not sufficient for"
605 " optimal performance.\n");
606 device_printf(dev, "For optimal performance a x8 "
607 "PCIE, or x4 PCIE 2 slot is required.\n");
610 /* let hardware know driver is loaded */
611 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
612 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
613 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
615 ixgbe_add_hw_stats(adapter);
618 ixgbe_netmap_attach(adapter);
619 #endif /* DEV_NETMAP */
620 INIT_DEBUGOUT("ixgbe_attach: end");
623 ixgbe_free_transmit_structures(adapter);
624 ixgbe_free_receive_structures(adapter);
626 if (adapter->ifp != NULL)
627 if_free(adapter->ifp);
628 ixgbe_free_pci_resources(adapter);
629 kfree(adapter->mta, M_DEVBUF);
634 /*********************************************************************
635 * Device removal routine
637 * The detach entry point is called when the driver is being removed.
638 * This routine stops the adapter and deallocates all the resources
639 * that were allocated for driver operation.
641 * return 0 on success, positive on failure
642 *********************************************************************/
645 ixgbe_detach(device_t dev)
647 struct adapter *adapter = device_get_softc(dev);
648 struct ix_queue *que = adapter->queues;
651 INIT_DEBUGOUT("ixgbe_detach: begin");
653 /* Make sure VLANS are not using driver */
654 if (adapter->ifp->if_vlantrunks != NULL) {
655 device_printf(dev,"Vlan in use, detach first\n");
659 IXGBE_CORE_LOCK(adapter);
661 IXGBE_CORE_UNLOCK(adapter);
663 for (int i = 0; i < adapter->num_queues; i++, que++) {
665 taskqueue_drain(que->tq, &que->que_task);
666 taskqueue_free(que->tq);
670 /* Drain the Link queue */
672 taskqueue_drain(adapter->tq, &adapter->link_task);
673 taskqueue_drain(adapter->tq, &adapter->mod_task);
674 taskqueue_drain(adapter->tq, &adapter->msf_task);
676 taskqueue_drain(adapter->tq, &adapter->fdir_task);
678 taskqueue_free(adapter->tq);
681 /* let hardware know driver is unloading */
682 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
683 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
684 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
686 /* Unregister VLAN events */
687 if (adapter->vlan_attach != NULL)
688 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
689 if (adapter->vlan_detach != NULL)
690 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
692 ether_ifdetach(adapter->ifp);
693 callout_stop(&adapter->timer);
695 netmap_detach(adapter->ifp);
696 #endif /* DEV_NETMAP */
697 ixgbe_free_pci_resources(adapter);
698 bus_generic_detach(dev);
699 if_free(adapter->ifp);
701 ixgbe_free_transmit_structures(adapter);
702 ixgbe_free_receive_structures(adapter);
703 kfree(adapter->mta, M_DEVBUF);
704 sysctl_ctx_free(&adapter->sysctl_ctx);
706 IXGBE_CORE_LOCK_DESTROY(adapter);
710 /*********************************************************************
712 * Shutdown entry point
714 **********************************************************************/
717 ixgbe_shutdown(device_t dev)
719 struct adapter *adapter = device_get_softc(dev);
720 IXGBE_CORE_LOCK(adapter);
722 IXGBE_CORE_UNLOCK(adapter);
727 /*********************************************************************
728 * Transmit entry point
730 * ixgbe_start is called by the stack to initiate a transmit.
731 * The driver will remain in this routine as long as there are
732 * packets to transmit and transmit resources are available.
733 * In case resources are not available stack is notified and
734 * the packet is requeued.
735 **********************************************************************/
738 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
741 struct adapter *adapter = txr->adapter;
743 IXGBE_TX_LOCK_ASSERT(txr);
745 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
747 if (!adapter->link_active)
750 while (!ifq_is_empty(&ifp->if_snd)) {
751 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
752 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
756 m_head = ifq_dequeue(&ifp->if_snd, NULL);
760 if (ixgbe_xmit(txr, &m_head)) {
761 #if 0 /* XXX: prepend to an ALTQ queue ? */
763 IF_PREPEND(&ifp->if_snd, m_head);
765 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
766 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
769 /* Send a copy of the frame to the BPF listener */
770 ETHER_BPF_MTAP(ifp, m_head);
772 /* Set watchdog on */
773 txr->watchdog_time = ticks;
774 txr->queue_status = IXGBE_QUEUE_WORKING;
781 * Legacy TX start - called by the stack, this
782 * always uses the first tx ring, and should
783 * not be used with multiqueue tx enabled.
786 ixgbe_start(struct ifnet *ifp)
788 struct adapter *adapter = ifp->if_softc;
789 struct tx_ring *txr = adapter->tx_rings;
791 if (ifp->if_flags & IFF_RUNNING) {
793 ixgbe_start_locked(txr, ifp);
794 IXGBE_TX_UNLOCK(txr);
799 #if 0 /* __FreeBSD_version >= 800000 */
801 ** Multiqueue Transmit driver
805 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
807 struct adapter *adapter = ifp->if_softc;
808 struct ix_queue *que;
812 /* Which queue to use */
813 if ((m->m_flags & M_FLOWID) != 0)
814 i = m->m_pkthdr.flowid % adapter->num_queues;
816 i = curcpu % adapter->num_queues;
818 txr = &adapter->tx_rings[i];
819 que = &adapter->queues[i];
821 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
822 IXGBE_TX_TRYLOCK(txr)) {
823 err = ixgbe_mq_start_locked(ifp, txr, m);
824 IXGBE_TX_UNLOCK(txr);
826 err = drbr_enqueue(ifp, txr->br, m);
827 taskqueue_enqueue(que->tq, &que->que_task);
834 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
836 struct adapter *adapter = txr->adapter;
838 int enqueued, err = 0;
840 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
841 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
842 adapter->link_active == 0) {
844 err = drbr_enqueue(ifp, txr->br, m);
850 next = drbr_dequeue(ifp, txr->br);
851 } else if (drbr_needs_enqueue(ifp, txr->br)) {
852 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
854 next = drbr_dequeue(ifp, txr->br);
858 /* Process the queue */
859 while (next != NULL) {
860 if ((err = ixgbe_xmit(txr, &next)) != 0) {
862 err = drbr_enqueue(ifp, txr->br, next);
866 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
867 /* Send a copy of the frame to the BPF listener */
868 ETHER_BPF_MTAP(ifp, next);
869 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
871 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
873 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
874 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
877 next = drbr_dequeue(ifp, txr->br);
881 /* Set watchdog on */
882 txr->queue_status |= IXGBE_QUEUE_WORKING;
883 txr->watchdog_time = ticks;
886 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
893 ** Flush all ring buffers
896 ixgbe_qflush(struct ifnet *ifp)
898 struct adapter *adapter = ifp->if_softc;
899 struct tx_ring *txr = adapter->tx_rings;
902 for (int i = 0; i < adapter->num_queues; i++, txr++) {
904 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
906 IXGBE_TX_UNLOCK(txr);
910 #endif /* __FreeBSD_version >= 800000 */
912 /*********************************************************************
915 * ixgbe_ioctl is called when the user wants to configure the
918 * return 0 on success, positive on failure
919 **********************************************************************/
922 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
924 struct adapter *adapter = ifp->if_softc;
925 struct ifreq *ifr = (struct ifreq *) data;
926 #if defined(INET) || defined(INET6)
927 struct ifaddr *ifa = (struct ifaddr *)data;
928 bool avoid_reset = FALSE;
936 if (ifa->ifa_addr->sa_family == AF_INET)
940 if (ifa->ifa_addr->sa_family == AF_INET6)
943 #if defined(INET) || defined(INET6)
945 ** Calling init results in link renegotiation,
946 ** so we avoid doing it when possible.
949 ifp->if_flags |= IFF_UP;
950 if (!(ifp->if_flags & IFF_RUNNING))
952 if (!(ifp->if_flags & IFF_NOARP))
953 arp_ifinit(ifp, ifa);
955 error = ether_ioctl(ifp, command, data);
959 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
960 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
963 IXGBE_CORE_LOCK(adapter);
964 ifp->if_mtu = ifr->ifr_mtu;
965 adapter->max_frame_size =
966 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
967 ixgbe_init_locked(adapter);
968 IXGBE_CORE_UNLOCK(adapter);
972 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
973 IXGBE_CORE_LOCK(adapter);
974 if (ifp->if_flags & IFF_UP) {
975 if ((ifp->if_flags & IFF_RUNNING)) {
976 if ((ifp->if_flags ^ adapter->if_flags) &
977 (IFF_PROMISC | IFF_ALLMULTI)) {
978 ixgbe_set_promisc(adapter);
981 ixgbe_init_locked(adapter);
983 if (ifp->if_flags & IFF_RUNNING)
985 adapter->if_flags = ifp->if_flags;
986 IXGBE_CORE_UNLOCK(adapter);
990 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
991 if (ifp->if_flags & IFF_RUNNING) {
992 IXGBE_CORE_LOCK(adapter);
993 ixgbe_disable_intr(adapter);
994 ixgbe_set_multi(adapter);
995 ixgbe_enable_intr(adapter);
996 IXGBE_CORE_UNLOCK(adapter);
1001 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1002 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1006 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1007 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1008 if (mask & IFCAP_HWCSUM)
1009 ifp->if_capenable ^= IFCAP_HWCSUM;
1011 if (mask & IFCAP_TSO4)
1012 ifp->if_capenable ^= IFCAP_TSO4;
1013 if (mask & IFCAP_TSO6)
1014 ifp->if_capenable ^= IFCAP_TSO6;
1017 if (mask & IFCAP_LRO)
1018 ifp->if_capenable ^= IFCAP_LRO;
1020 if (mask & IFCAP_VLAN_HWTAGGING)
1021 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1022 if (mask & IFCAP_VLAN_HWFILTER)
1023 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1025 if (mask & IFCAP_VLAN_HWTSO)
1026 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1028 if (ifp->if_flags & IFF_RUNNING) {
1029 IXGBE_CORE_LOCK(adapter);
1030 ixgbe_init_locked(adapter);
1031 IXGBE_CORE_UNLOCK(adapter);
1034 VLAN_CAPABILITIES(ifp);
1040 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1041 error = ether_ioctl(ifp, command, data);
1048 /*********************************************************************
1051 * This routine is used in two ways. It is used by the stack as
1052 * init entry point in network interface structure. It is also used
1053 * by the driver as a hw/sw initialization routine to get to a
1056 * return 0 on success, positive on failure
1057 **********************************************************************/
1058 #define IXGBE_MHADD_MFS_SHIFT 16
1061 ixgbe_init_locked(struct adapter *adapter)
1063 struct ifnet *ifp = adapter->ifp;
1064 device_t dev = adapter->dev;
1065 struct ixgbe_hw *hw = &adapter->hw;
1066 u32 k, txdctl, mhadd, gpie;
1069 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1070 INIT_DEBUGOUT("ixgbe_init: begin");
1071 hw->adapter_stopped = FALSE;
1072 ixgbe_stop_adapter(hw);
1073 callout_stop(&adapter->timer);
1075 /* reprogram the RAR[0] in case user changed it. */
1076 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1078 /* Get the latest mac address, User can use a LAA */
1079 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1080 IXGBE_ETH_LENGTH_OF_ADDRESS);
1081 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1082 hw->addr_ctrl.rar_used_count = 1;
1084 /* Set the various hardware offload abilities */
1085 ifp->if_hwassist = 0;
1087 if (ifp->if_capenable & IFCAP_TSO)
1088 ifp->if_hwassist |= CSUM_TSO;
1090 if (ifp->if_capenable & IFCAP_TXCSUM) {
1091 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1093 if (hw->mac.type != ixgbe_mac_82598EB)
1094 ifp->if_hwassist |= CSUM_SCTP;
1098 /* Prepare transmit descriptors and buffers */
1099 if (ixgbe_setup_transmit_structures(adapter)) {
1100 device_printf(dev,"Could not setup transmit structures\n");
1101 ixgbe_stop(adapter);
1106 ixgbe_initialize_transmit_units(adapter);
1108 /* Setup Multicast table */
1109 ixgbe_set_multi(adapter);
1112 ** Determine the correct mbuf pool
1113 ** for doing jumbo/headersplit
1115 if (adapter->max_frame_size <= 2048)
1116 adapter->rx_mbuf_sz = MCLBYTES;
1117 else if (adapter->max_frame_size <= 4096)
1118 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1119 else if (adapter->max_frame_size <= 9216)
1120 adapter->rx_mbuf_sz = MJUM9BYTES;
1122 adapter->rx_mbuf_sz = MJUM16BYTES;
1124 /* Prepare receive descriptors and buffers */
1125 if (ixgbe_setup_receive_structures(adapter)) {
1126 device_printf(dev,"Could not setup receive structures\n");
1127 ixgbe_stop(adapter);
1131 /* Configure RX settings */
1132 ixgbe_initialize_receive_units(adapter);
1134 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1136 /* Enable Fan Failure Interrupt */
1137 gpie |= IXGBE_SDP1_GPIEN;
1139 /* Add for Module detection */
1140 if (hw->mac.type == ixgbe_mac_82599EB)
1141 gpie |= IXGBE_SDP2_GPIEN;
1143 /* Thermal Failure Detection */
1144 if (hw->mac.type == ixgbe_mac_X540)
1145 gpie |= IXGBE_SDP0_GPIEN;
1147 if (adapter->msix > 1) {
1148 /* Enable Enhanced MSIX mode */
1149 gpie |= IXGBE_GPIE_MSIX_MODE;
1150 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1153 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1156 if (ifp->if_mtu > ETHERMTU) {
1157 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1158 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1159 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1160 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1163 /* Now enable all the queues */
1165 for (int i = 0; i < adapter->num_queues; i++) {
1166 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1167 txdctl |= IXGBE_TXDCTL_ENABLE;
1168 /* Set WTHRESH to 8, burst writeback */
1169 txdctl |= (8 << 16);
1171 * When the internal queue falls below PTHRESH (32),
1172 * start prefetching as long as there are at least
1173 * HTHRESH (1) buffers ready. The values are taken
1174 * from the Intel linux driver 3.8.21.
1175 * Prefetching enables tx line rate even with 1 queue.
1177 txdctl |= (32 << 0) | (1 << 8);
1178 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1181 for (int i = 0; i < adapter->num_queues; i++) {
1182 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1183 if (hw->mac.type == ixgbe_mac_82598EB) {
1189 rxdctl &= ~0x3FFFFF;
1192 rxdctl |= IXGBE_RXDCTL_ENABLE;
1193 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1194 for (k = 0; k < 10; k++) {
1195 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1196 IXGBE_RXDCTL_ENABLE)
1204 * In netmap mode, we must preserve the buffers made
1205 * available to userspace before the if_init()
1206 * (this is true by default on the TX side, because
1207 * init makes all buffers available to userspace).
1209 * netmap_reset() and the device specific routines
1210 * (e.g. ixgbe_setup_receive_rings()) map these
1211 * buffers at the end of the NIC ring, so here we
1212 * must set the RDT (tail) register to make sure
1213 * they are not overwritten.
1215 * In this driver the NIC ring starts at RDH = 0,
1216 * RDT points to the last slot available for reception (?),
1217 * so RDT = num_rx_desc - 1 means the whole ring is available.
1219 if (ifp->if_capenable & IFCAP_NETMAP) {
1220 struct netmap_adapter *na = NA(adapter->ifp);
1221 struct netmap_kring *kring = &na->rx_rings[i];
1222 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1224 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1226 #endif /* DEV_NETMAP */
1227 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1230 /* Set up VLAN support and filter */
1231 ixgbe_setup_vlan_hw_support(adapter);
1233 /* Enable Receive engine */
1234 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1235 if (hw->mac.type == ixgbe_mac_82598EB)
1236 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1237 rxctrl |= IXGBE_RXCTRL_RXEN;
1238 ixgbe_enable_rx_dma(hw, rxctrl);
1240 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1242 /* Set up MSI/X routing */
1243 if (ixgbe_enable_msix) {
1244 ixgbe_configure_ivars(adapter);
1245 /* Set up auto-mask */
1246 if (hw->mac.type == ixgbe_mac_82598EB)
1247 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1249 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1250 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1252 } else { /* Simple settings for Legacy/MSI */
1253 ixgbe_set_ivar(adapter, 0, 0, 0);
1254 ixgbe_set_ivar(adapter, 0, 0, 1);
1255 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1259 /* Init Flow director */
1260 if (hw->mac.type != ixgbe_mac_82598EB) {
1261 u32 hdrm = 32 << fdir_pballoc;
1263 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1264 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1269 ** Check on any SFP devices that
1270 ** need to be kick-started
1272 if (hw->phy.type == ixgbe_phy_none) {
1273 int err = hw->phy.ops.identify(hw);
1274 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1276 "Unsupported SFP+ module type was detected.\n");
1281 /* Set moderation on the Link interrupt */
1282 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1284 /* Config/Enable Link */
1285 ixgbe_config_link(adapter);
1287 /* Hardware Packet Buffer & Flow Control setup */
1289 u32 rxpb, frame, size, tmp;
1291 frame = adapter->max_frame_size;
1293 /* Calculate High Water */
1294 if (hw->mac.type == ixgbe_mac_X540)
1295 tmp = IXGBE_DV_X540(frame, frame);
1297 tmp = IXGBE_DV(frame, frame);
1298 size = IXGBE_BT2KB(tmp);
1299 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1300 hw->fc.high_water[0] = rxpb - size;
1302 /* Now calculate Low Water */
1303 if (hw->mac.type == ixgbe_mac_X540)
1304 tmp = IXGBE_LOW_DV_X540(frame);
1306 tmp = IXGBE_LOW_DV(frame);
1307 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1309 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1310 hw->fc.pause_time = IXGBE_FC_PAUSE;
1311 hw->fc.send_xon = TRUE;
1313 /* Initialize the FC settings */
1316 /* And now turn on interrupts */
1317 ixgbe_enable_intr(adapter);
1319 /* Now inform the stack we're ready */
1320 ifp->if_flags |= IFF_RUNNING;
1321 ifp->if_flags &= ~IFF_OACTIVE;
1327 ixgbe_init(void *arg)
1329 struct adapter *adapter = arg;
1331 IXGBE_CORE_LOCK(adapter);
1332 ixgbe_init_locked(adapter);
1333 IXGBE_CORE_UNLOCK(adapter);
1340 ** MSIX Interrupt Handlers and Tasklets
1345 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1347 struct ixgbe_hw *hw = &adapter->hw;
1348 u64 queue = (u64)(1 << vector);
1351 if (hw->mac.type == ixgbe_mac_82598EB) {
1352 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1353 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1355 mask = (queue & 0xFFFFFFFF);
1357 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1358 mask = (queue >> 32);
1360 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1365 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1367 struct ixgbe_hw *hw = &adapter->hw;
1368 u64 queue = (u64)(1 << vector);
1371 if (hw->mac.type == ixgbe_mac_82598EB) {
1372 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1373 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1375 mask = (queue & 0xFFFFFFFF);
1377 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1378 mask = (queue >> 32);
1380 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1385 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1389 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1390 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1391 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1393 mask = (queues & 0xFFFFFFFF);
1394 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1395 mask = (queues >> 32);
1396 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1402 ixgbe_handle_que(void *context, int pending)
1404 struct ix_queue *que = context;
1405 struct adapter *adapter = que->adapter;
1406 struct tx_ring *txr = que->txr;
1407 struct ifnet *ifp = adapter->ifp;
1410 if (ifp->if_flags & IFF_RUNNING) {
1411 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1414 #if 0 /*__FreeBSD_version >= 800000*/
1415 if (!drbr_empty(ifp, txr->br))
1416 ixgbe_mq_start_locked(ifp, txr, NULL);
1418 if (!ifq_is_empty(&ifp->if_snd))
1419 ixgbe_start_locked(txr, ifp);
1421 IXGBE_TX_UNLOCK(txr);
1423 taskqueue_enqueue(que->tq, &que->que_task);
1428 /* Reenable this interrupt */
1429 ixgbe_enable_queue(adapter, que->msix);
1434 /*********************************************************************
1436 * Legacy Interrupt Service routine
1438 **********************************************************************/
1441 ixgbe_legacy_irq(void *arg)
1443 struct ix_queue *que = arg;
1444 struct adapter *adapter = que->adapter;
1445 struct ixgbe_hw *hw = &adapter->hw;
1446 struct tx_ring *txr = adapter->tx_rings;
1447 bool more_tx, more_rx;
1448 u32 reg_eicr, loop = MAX_LOOP;
1451 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1454 if (reg_eicr == 0) {
1455 ixgbe_enable_intr(adapter);
1459 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1463 more_tx = ixgbe_txeof(txr);
1464 } while (loop-- && more_tx);
1465 IXGBE_TX_UNLOCK(txr);
1467 if (more_rx || more_tx)
1468 taskqueue_enqueue(que->tq, &que->que_task);
1470 /* Check for fan failure */
1471 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1472 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1473 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1474 "REPLACE IMMEDIATELY!!\n");
1475 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1478 /* Link status change */
1479 if (reg_eicr & IXGBE_EICR_LSC)
1480 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1482 ixgbe_enable_intr(adapter);
1487 /*********************************************************************
1489 * MSIX Queue Interrupt Service routine
1491 **********************************************************************/
1493 ixgbe_msix_que(void *arg)
1495 struct ix_queue *que = arg;
1496 struct adapter *adapter = que->adapter;
1497 struct tx_ring *txr = que->txr;
1498 struct rx_ring *rxr = que->rxr;
1499 bool more_tx, more_rx;
1502 ixgbe_disable_queue(adapter, que->msix);
1505 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1508 more_tx = ixgbe_txeof(txr);
1510 ** Make certain that if the stack
1511 ** has anything queued the task gets
1512 ** scheduled to handle it.
1515 #if __FreeBSD_version < 800000
1516 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1518 if (!drbr_empty(adapter->ifp, txr->br))
1521 if (!ifq_is_empty(&adapter->ifp->if_snd))
1523 IXGBE_TX_UNLOCK(txr);
1527 if (ixgbe_enable_aim == FALSE)
1530 ** Do Adaptive Interrupt Moderation:
1531 ** - Write out last calculated setting
1532 ** - Calculate based on average size over
1533 ** the last interval.
1535 if (que->eitr_setting)
1536 IXGBE_WRITE_REG(&adapter->hw,
1537 IXGBE_EITR(que->msix), que->eitr_setting);
1539 que->eitr_setting = 0;
1541 /* Idle, do nothing */
1542 if ((txr->bytes == 0) && (rxr->bytes == 0))
1545 if ((txr->bytes) && (txr->packets))
1546 newitr = txr->bytes/txr->packets;
1547 if ((rxr->bytes) && (rxr->packets))
1548 newitr = max(newitr,
1549 (rxr->bytes / rxr->packets));
1550 newitr += 24; /* account for hardware frame, crc */
1552 /* set an upper boundary */
1553 newitr = min(newitr, 3000);
1555 /* Be nice to the mid range */
1556 if ((newitr > 300) && (newitr < 1200))
1557 newitr = (newitr / 3);
1559 newitr = (newitr / 2);
1561 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1562 newitr |= newitr << 16;
1564 newitr |= IXGBE_EITR_CNT_WDIS;
1566 /* save for next interrupt */
1567 que->eitr_setting = newitr;
1576 if (more_tx || more_rx)
1577 taskqueue_enqueue(que->tq, &que->que_task);
1578 else /* Reenable this interrupt */
1579 ixgbe_enable_queue(adapter, que->msix);
1585 ixgbe_msix_link(void *arg)
1587 struct adapter *adapter = arg;
1588 struct ixgbe_hw *hw = &adapter->hw;
1591 ++adapter->link_irq;
1593 /* First get the cause */
1594 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1595 /* Clear interrupt with write */
1596 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1598 /* Link status change */
1599 if (reg_eicr & IXGBE_EICR_LSC)
1600 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1602 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1604 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1605 /* This is probably overkill :) */
1606 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1608 /* Disable the interrupt */
1609 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1610 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1613 if (reg_eicr & IXGBE_EICR_ECC) {
1614 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1615 "Please Reboot!!\n");
1616 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1619 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1620 /* Clear the interrupt */
1621 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1622 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1623 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1624 /* Clear the interrupt */
1625 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1626 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1630 /* Check for fan failure */
1631 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1632 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1633 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1634 "REPLACE IMMEDIATELY!!\n");
1635 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1638 /* Check for over temp condition */
1639 if ((hw->mac.type == ixgbe_mac_X540) &&
1640 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1641 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1642 "PHY IS SHUT DOWN!!\n");
1643 device_printf(adapter->dev, "System shutdown required\n");
1644 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1647 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1651 /*********************************************************************
1653 * Media Ioctl callback
1655 * This routine is called whenever the user queries the status of
1656 * the interface using ifconfig.
1658 **********************************************************************/
1660 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1662 struct adapter *adapter = ifp->if_softc;
1664 INIT_DEBUGOUT("ixgbe_media_status: begin");
1665 IXGBE_CORE_LOCK(adapter);
1666 ixgbe_update_link_status(adapter);
1668 ifmr->ifm_status = IFM_AVALID;
1669 ifmr->ifm_active = IFM_ETHER;
1671 if (!adapter->link_active) {
1672 IXGBE_CORE_UNLOCK(adapter);
1676 ifmr->ifm_status |= IFM_ACTIVE;
1678 switch (adapter->link_speed) {
1679 case IXGBE_LINK_SPEED_100_FULL:
1680 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1682 case IXGBE_LINK_SPEED_1GB_FULL:
1683 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1685 case IXGBE_LINK_SPEED_10GB_FULL:
1686 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1690 IXGBE_CORE_UNLOCK(adapter);
1695 /*********************************************************************
1697 * Media Ioctl callback
1699 * This routine is called when the user changes speed/duplex using
1700 * media/mediopt option with ifconfig.
1702 **********************************************************************/
1704 ixgbe_media_change(struct ifnet * ifp)
1706 struct adapter *adapter = ifp->if_softc;
1707 struct ifmedia *ifm = &adapter->media;
1709 INIT_DEBUGOUT("ixgbe_media_change: begin");
1711 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1714 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1716 adapter->hw.phy.autoneg_advertised =
1717 IXGBE_LINK_SPEED_100_FULL |
1718 IXGBE_LINK_SPEED_1GB_FULL |
1719 IXGBE_LINK_SPEED_10GB_FULL;
1722 device_printf(adapter->dev, "Only auto media type\n");
1729 /*********************************************************************
1731 * This routine maps the mbufs to tx descriptors, allowing the
1732 * TX engine to transmit the packets.
1733 * - return 0 on success, positive on failure
1735 **********************************************************************/
1738 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1740 struct adapter *adapter = txr->adapter;
1741 u32 olinfo_status = 0, cmd_type_len;
1743 int i, j, error, nsegs, maxsegs;
1744 int first, last = 0;
1745 struct mbuf *m_head;
1746 bus_dma_segment_t segs[adapter->num_segs];
1748 struct ixgbe_tx_buf *txbuf;
1749 union ixgbe_adv_tx_desc *txd = NULL;
1753 /* Basic descriptor defines */
1754 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1755 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1757 if (m_head->m_flags & M_VLANTAG)
1758 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1761 * Important to capture the first descriptor
1762 * used because it will contain the index of
1763 * the one we tell the hardware to report back
1765 first = txr->next_avail_desc;
1766 txbuf = &txr->tx_buffers[first];
1770 * Map the packet for DMA.
1772 maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1773 if (maxsegs > adapter->num_segs)
1774 maxsegs = adapter->num_segs;
1776 error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1777 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1779 if (error == ENOBUFS)
1780 adapter->mbuf_defrag_failed++;
1782 adapter->no_tx_dma_setup++;
1789 /* Make certain there are enough descriptors */
1790 if (nsegs > txr->tx_avail - 2) {
1791 txr->no_desc_avail++;
1798 ** Set up the appropriate offload context
1799 ** this becomes the first descriptor of
1803 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1804 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1805 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1806 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1807 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1811 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1813 if (ixgbe_tx_ctx_setup(txr, m_head))
1814 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1816 #ifdef IXGBE_IEEE1588
1817 /* This is changing soon to an mtag detection */
1818 if (we detect this mbuf has a TSTAMP mtag)
1819 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1823 /* Do the flow director magic */
1824 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1826 if (txr->atr_count >= atr_sample_rate) {
1827 ixgbe_atr(txr, m_head);
1832 /* Record payload length */
1834 olinfo_status |= m_head->m_pkthdr.len <<
1835 IXGBE_ADVTXD_PAYLEN_SHIFT;
1837 i = txr->next_avail_desc;
1838 for (j = 0; j < nsegs; j++) {
1842 txbuf = &txr->tx_buffers[i];
1843 txd = &txr->tx_base[i];
1844 seglen = segs[j].ds_len;
1845 segaddr = htole64(segs[j].ds_addr);
1847 txd->read.buffer_addr = segaddr;
1848 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1849 cmd_type_len |seglen);
1850 txd->read.olinfo_status = htole32(olinfo_status);
1851 last = i; /* descriptor that will get completion IRQ */
1853 if (++i == adapter->num_tx_desc)
1856 txbuf->m_head = NULL;
1857 txbuf->eop_index = -1;
1860 txd->read.cmd_type_len |=
1861 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1862 txr->tx_avail -= nsegs;
1863 txr->next_avail_desc = i;
1865 txbuf->m_head = m_head;
1866 /* Swap the dma map between the first and last descriptor */
1867 txr->tx_buffers[first].map = txbuf->map;
1869 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1871 /* Set the index of the descriptor that will be marked done */
1872 txbuf = &txr->tx_buffers[first];
1873 txbuf->eop_index = last;
1875 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1876 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1878 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1879 * hardware that this frame is available to transmit.
1881 ++txr->total_packets;
1882 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1887 bus_dmamap_unload(txr->txtag, txbuf->map);
1893 ixgbe_set_promisc(struct adapter *adapter)
1896 struct ifnet *ifp = adapter->ifp;
1898 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1899 reg_rctl &= (~IXGBE_FCTRL_UPE);
1900 reg_rctl &= (~IXGBE_FCTRL_MPE);
1901 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1903 if (ifp->if_flags & IFF_PROMISC) {
1904 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1905 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1906 } else if (ifp->if_flags & IFF_ALLMULTI) {
1907 reg_rctl |= IXGBE_FCTRL_MPE;
1908 reg_rctl &= ~IXGBE_FCTRL_UPE;
1909 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1915 /*********************************************************************
1918 * This routine is called whenever multicast address list is updated.
1920 **********************************************************************/
1921 #define IXGBE_RAR_ENTRIES 16
1924 ixgbe_set_multi(struct adapter *adapter)
1929 struct ifmultiaddr *ifma;
1931 struct ifnet *ifp = adapter->ifp;
1933 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1936 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1937 MAX_NUM_MULTICAST_ADDRESSES);
1939 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1940 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1941 if (ifp->if_flags & IFF_PROMISC)
1942 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1943 else if (ifp->if_flags & IFF_ALLMULTI) {
1944 fctrl |= IXGBE_FCTRL_MPE;
1945 fctrl &= ~IXGBE_FCTRL_UPE;
1947 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1949 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1951 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1952 if (ifma->ifma_addr->sa_family != AF_LINK)
1954 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1955 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1956 IXGBE_ETH_LENGTH_OF_ADDRESS);
1961 ixgbe_update_mc_addr_list(&adapter->hw,
1962 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1968 * This is an iterator function now needed by the multicast
1969 * shared code. It simply feeds the shared code routine the
1970 * addresses in the array of ixgbe_set_multi() one by one.
1973 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1975 u8 *addr = *update_ptr;
1979 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1980 *update_ptr = newptr;
1985 /*********************************************************************
1988 * This routine checks for link status,updates statistics,
1989 * and runs the watchdog check.
1991 **********************************************************************/
1994 ixgbe_local_timer(void *arg)
1996 struct adapter *adapter = arg;
1997 device_t dev = adapter->dev;
1998 struct ifnet *ifp = adapter->ifp;
1999 struct ix_queue *que = adapter->queues;
2000 struct tx_ring *txr = adapter->tx_rings;
2001 int hung, busy, paused;
2003 IXGBE_CORE_LOCK(adapter);
2004 hung = busy = paused = 0;
2006 /* Check for pluggable optics */
2007 if (adapter->sfp_probe)
2008 if (!ixgbe_sfp_probe(adapter))
2009 goto out; /* Nothing to do */
2011 ixgbe_update_link_status(adapter);
2012 ixgbe_update_stats_counters(adapter);
2015 * If the interface has been paused
2016 * then don't do the watchdog check
2018 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2022 ** Check the TX queues status
2023 ** - central locked handling of OACTIVE
2024 ** - watchdog only if all queues show hung
2026 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2027 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2030 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2032 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2033 taskqueue_enqueue(que->tq, &que->que_task);
2035 /* Only truely watchdog if all queues show hung */
2036 if (hung == adapter->num_queues)
2038 /* Only turn off the stack flow when ALL are depleted */
2039 if (busy == adapter->num_queues)
2040 ifp->if_flags |= IFF_OACTIVE;
2041 else if ((ifp->if_flags & IFF_OACTIVE) &&
2042 (busy < adapter->num_queues))
2043 ifp->if_flags &= ~IFF_OACTIVE;
2046 ixgbe_rearm_queues(adapter, adapter->que_mask);
2047 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2048 IXGBE_CORE_UNLOCK(adapter);
2052 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2053 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2054 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2055 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2056 device_printf(dev,"TX(%d) desc avail = %d,"
2057 "Next TX to Clean = %d\n",
2058 txr->me, txr->tx_avail, txr->next_to_clean);
2059 adapter->ifp->if_flags &= ~IFF_RUNNING;
2060 adapter->watchdog_events++;
2061 ixgbe_init_locked(adapter);
2063 IXGBE_CORE_UNLOCK(adapter);
2067 ** Note: this routine updates the OS on the link state
2068 ** the real check of the hardware only happens with
2069 ** a link interrupt.
2072 ixgbe_update_link_status(struct adapter *adapter)
2074 struct ifnet *ifp = adapter->ifp;
2075 struct tx_ring *txr = adapter->tx_rings;
2076 device_t dev = adapter->dev;
2079 if (adapter->link_up){
2080 if (adapter->link_active == FALSE) {
2082 device_printf(dev,"Link is up %d Gbps %s \n",
2083 ((adapter->link_speed == 128)? 10:1),
2085 adapter->link_active = TRUE;
2086 /* Update any Flow Control changes */
2087 ixgbe_fc_enable(&adapter->hw);
2088 ifp->if_link_state = LINK_STATE_UP;
2089 if_link_state_change(ifp);
2091 } else { /* Link down */
2092 if (adapter->link_active == TRUE) {
2094 device_printf(dev,"Link is Down\n");
2095 ifp->if_link_state = LINK_STATE_DOWN;
2096 if_link_state_change(ifp);
2097 adapter->link_active = FALSE;
2098 for (int i = 0; i < adapter->num_queues;
2100 txr->queue_status = IXGBE_QUEUE_IDLE;
2108 /*********************************************************************
2110 * This routine disables all traffic on the adapter by issuing a
2111 * global reset on the MAC and deallocates TX/RX buffers.
2113 **********************************************************************/
2116 ixgbe_stop(void *arg)
2119 struct adapter *adapter = arg;
2120 struct ixgbe_hw *hw = &adapter->hw;
2123 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2125 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2126 ixgbe_disable_intr(adapter);
2127 callout_stop(&adapter->timer);
2129 /* Let the stack know...*/
2130 ifp->if_flags &= ~IFF_RUNNING;
2131 ifp->if_flags |= IFF_OACTIVE;
2134 hw->adapter_stopped = FALSE;
2135 ixgbe_stop_adapter(hw);
2136 /* Turn off the laser */
2137 if (hw->phy.multispeed_fiber)
2138 ixgbe_disable_tx_laser(hw);
2140 /* reprogram the RAR[0] in case user changed it. */
2141 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2147 /*********************************************************************
2149 * Determine hardware revision.
2151 **********************************************************************/
2153 ixgbe_identify_hardware(struct adapter *adapter)
2155 device_t dev = adapter->dev;
2156 struct ixgbe_hw *hw = &adapter->hw;
2158 /* Save off the information about this board */
2159 hw->vendor_id = pci_get_vendor(dev);
2160 hw->device_id = pci_get_device(dev);
2161 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2162 hw->subsystem_vendor_id =
2163 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2164 hw->subsystem_device_id =
2165 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2167 /* We need this here to set the num_segs below */
2168 ixgbe_set_mac_type(hw);
2170 /* Pick up the 82599 and VF settings */
2171 if (hw->mac.type != ixgbe_mac_82598EB) {
2172 hw->phy.smart_speed = ixgbe_smart_speed;
2173 adapter->num_segs = IXGBE_82599_SCATTER;
2175 adapter->num_segs = IXGBE_82598_SCATTER;
2180 /*********************************************************************
2182 * Determine optic type
2184 **********************************************************************/
2186 ixgbe_setup_optics(struct adapter *adapter)
2188 struct ixgbe_hw *hw = &adapter->hw;
2191 layer = ixgbe_get_supported_physical_layer(hw);
2193 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2194 adapter->optics = IFM_10G_T;
2198 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2199 adapter->optics = IFM_1000_T;
2203 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2204 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2205 adapter->optics = IFM_10G_LR;
2209 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2210 adapter->optics = IFM_10G_SR;
2214 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2215 adapter->optics = IFM_10G_TWINAX;
2219 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2220 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2221 adapter->optics = IFM_10G_CX4;
2225 /* If we get here just set the default */
2226 adapter->optics = IFM_ETHER | IFM_AUTO;
2230 /*********************************************************************
2232 * Setup the Legacy or MSI Interrupt handler
2234 **********************************************************************/
2236 ixgbe_allocate_legacy(struct adapter *adapter)
2238 device_t dev = adapter->dev;
2239 struct ix_queue *que = adapter->queues;
2241 unsigned int intr_flags;
2244 if (adapter->msix == 1)
2247 /* Try allocating a MSI interrupt first */
2248 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2251 /* We allocate a single interrupt resource */
2252 adapter->res = bus_alloc_resource_any(dev,
2253 SYS_RES_IRQ, &rid, intr_flags);
2254 if (adapter->res == NULL) {
2255 device_printf(dev, "Unable to allocate bus resource: "
2261 * Try allocating a fast interrupt and the associated deferred
2262 * processing contexts.
2264 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2265 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2266 taskqueue_thread_enqueue, &que->tq);
2267 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2268 device_get_nameunit(adapter->dev));
2270 /* Tasklets for Link, SFP and Multispeed Fiber */
2271 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2272 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2273 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2275 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2277 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2278 taskqueue_thread_enqueue, &adapter->tq);
2279 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2280 device_get_nameunit(adapter->dev));
2282 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2283 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2284 device_printf(dev, "Failed to register fast interrupt "
2285 "handler: %d\n", error);
2286 taskqueue_free(que->tq);
2287 taskqueue_free(adapter->tq);
2292 /* For simplicity in the handlers */
2293 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2299 /*********************************************************************
2301 * Setup MSIX Interrupt resources and handlers
2303 **********************************************************************/
2305 ixgbe_allocate_msix(struct adapter *adapter)
2307 device_t dev = adapter->dev;
2308 struct ix_queue *que = adapter->queues;
2309 int error, rid, vector = 0;
2311 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2313 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2314 RF_SHAREABLE | RF_ACTIVE);
2315 if (que->res == NULL) {
2316 device_printf(dev,"Unable to allocate"
2317 " bus resource: que interrupt [%d]\n", vector);
2320 /* Set the handler function */
2321 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2322 ixgbe_msix_que, que, &que->tag, &que->serializer);
2325 device_printf(dev, "Failed to register QUE handler");
2328 #if 0 /* __FreeBSD_version >= 800504 */
2329 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2332 adapter->que_mask |= (u64)(1 << que->msix);
2334 ** Bind the msix vector, and thus the
2335 ** ring to the corresponding cpu.
2338 if (adapter->num_queues > 1)
2339 bus_bind_intr(dev, que->res, i);
2342 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2343 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2344 taskqueue_thread_enqueue, &que->tq);
2345 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2346 device_get_nameunit(adapter->dev));
2351 adapter->res = bus_alloc_resource_any(dev,
2352 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2353 if (!adapter->res) {
2354 device_printf(dev,"Unable to allocate"
2355 " bus resource: Link interrupt [%d]\n", rid);
2358 /* Set the link handler function */
2359 error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2360 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2362 adapter->res = NULL;
2363 device_printf(dev, "Failed to register LINK handler");
2366 #if 0 /* __FreeBSD_version >= 800504 */
2367 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2369 adapter->linkvec = vector;
2370 /* Tasklets for Link, SFP and Multispeed Fiber */
2371 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2372 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2373 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2375 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2377 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2378 taskqueue_thread_enqueue, &adapter->tq);
2379 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2380 device_get_nameunit(adapter->dev));
2385 #if 0 /* HAVE_MSIX */
2387 * Setup Either MSI/X or MSI
2390 ixgbe_setup_msix(struct adapter *adapter)
2392 device_t dev = adapter->dev;
2393 int rid, want, queues, msgs;
2395 /* Override by tuneable */
2396 if (ixgbe_enable_msix == 0)
2399 /* First try MSI/X */
2400 rid = PCIR_BAR(MSIX_82598_BAR);
2401 adapter->msix_mem = bus_alloc_resource_any(dev,
2402 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2403 if (!adapter->msix_mem) {
2404 rid += 4; /* 82599 maps in higher BAR */
2405 adapter->msix_mem = bus_alloc_resource_any(dev,
2406 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2408 if (!adapter->msix_mem) {
2409 /* May not be enabled */
2410 device_printf(adapter->dev,
2411 "Unable to map MSIX table \n");
2415 msgs = pci_msix_count(dev);
2416 if (msgs == 0) { /* system has msix disabled */
2417 bus_release_resource(dev, SYS_RES_MEMORY,
2418 rid, adapter->msix_mem);
2419 adapter->msix_mem = NULL;
2423 /* Figure out a reasonable auto config value */
2424 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2426 if (ixgbe_num_queues != 0)
2427 queues = ixgbe_num_queues;
2428 /* Set max queues to 8 when autoconfiguring */
2429 else if ((ixgbe_num_queues == 0) && (queues > 8))
2433 ** Want one vector (RX/TX pair) per queue
2434 ** plus an additional for Link.
2440 device_printf(adapter->dev,
2441 "MSIX Configuration Problem, "
2442 "%d vectors but %d queues wanted!\n",
2444 return (0); /* Will go to Legacy setup */
2446 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2447 device_printf(adapter->dev,
2448 "Using MSIX interrupts with %d vectors\n", msgs);
2449 adapter->num_queues = queues;
2453 msgs = pci_msi_count(dev);
2454 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2455 device_printf(adapter->dev,"Using an MSI interrupt\n");
2457 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2464 ixgbe_allocate_pci_resources(struct adapter *adapter)
2467 device_t dev = adapter->dev;
2470 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2473 if (!(adapter->pci_mem)) {
2474 device_printf(dev,"Unable to allocate bus resource: memory\n");
2478 adapter->osdep.mem_bus_space_tag =
2479 rman_get_bustag(adapter->pci_mem);
2480 adapter->osdep.mem_bus_space_handle =
2481 rman_get_bushandle(adapter->pci_mem);
2482 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2484 /* Legacy defaults */
2485 adapter->num_queues = 1;
2486 adapter->hw.back = &adapter->osdep;
2489 ** Now setup MSI or MSI/X, should
2490 ** return us the number of supported
2491 ** vectors. (Will be 1 for MSI)
2493 #if 0 /* HAVE_MSIX */
2494 adapter->msix = ixgbe_setup_msix(adapter);
2500 ixgbe_free_pci_resources(struct adapter * adapter)
2502 struct ix_queue *que = adapter->queues;
2503 device_t dev = adapter->dev;
2506 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2507 memrid = PCIR_BAR(MSIX_82598_BAR);
2509 memrid = PCIR_BAR(MSIX_82599_BAR);
2512 ** There is a slight possibility of a failure mode
2513 ** in attach that will result in entering this function
2514 ** before interrupt resources have been initialized, and
2515 ** in that case we do not want to execute the loops below
2516 ** We can detect this reliably by the state of the adapter
2519 if (adapter->res == NULL)
2523 ** Release all msix queue resources:
2525 for (int i = 0; i < adapter->num_queues; i++, que++) {
2526 rid = que->msix + 1;
2527 if (que->tag != NULL) {
2528 bus_teardown_intr(dev, que->res, que->tag);
2531 if (que->res != NULL)
2532 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2536 /* Clean the Legacy or Link interrupt last */
2537 if (adapter->linkvec) /* we are doing MSIX */
2538 rid = adapter->linkvec + 1;
2540 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2542 if (adapter->tag != NULL) {
2543 bus_teardown_intr(dev, adapter->res, adapter->tag);
2544 adapter->tag = NULL;
2546 if (adapter->res != NULL)
2547 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2548 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2549 pci_release_msi(adapter->dev);
2553 pci_release_msi(dev);
2555 if (adapter->msix_mem != NULL)
2556 bus_release_resource(dev, SYS_RES_MEMORY,
2557 memrid, adapter->msix_mem);
2559 if (adapter->pci_mem != NULL)
2560 bus_release_resource(dev, SYS_RES_MEMORY,
2561 PCIR_BAR(0), adapter->pci_mem);
2566 /*********************************************************************
2568 * Setup networking device structure and register an interface.
2570 **********************************************************************/
2572 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2574 struct ixgbe_hw *hw = &adapter->hw;
2577 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2579 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2581 device_printf(dev, "can not allocate ifnet structure\n");
2584 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2585 ifp->if_baudrate = 1000000000;
2586 ifp->if_init = ixgbe_init;
2587 ifp->if_softc = adapter;
2588 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2589 ifp->if_ioctl = ixgbe_ioctl;
2590 ifp->if_start = ixgbe_start;
2591 #if 0 /* __FreeBSD_version >= 800000 */
2592 ifp->if_transmit = ixgbe_mq_start;
2593 ifp->if_qflush = ixgbe_qflush;
2595 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2597 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2599 adapter->max_frame_size =
2600 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2603 * Tell the upper layer(s) we support long frames.
2605 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2608 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2610 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2611 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2612 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2617 ifp->if_capenable = ifp->if_capabilities;
2619 /* Don't enable LRO by default */
2621 ifp->if_capabilities |= IFCAP_LRO;
2625 ** Don't turn this on by default, if vlans are
2626 ** created on another pseudo device (eg. lagg)
2627 ** then vlan events are not passed thru, breaking
2628 ** operation, but with HW FILTER off it works. If
2629 ** using vlans directly on the ixgbe driver you can
2630 ** enable this and get full hardware tag filtering.
2632 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2635 * Specify the media types supported by this adapter and register
2636 * callbacks to update media and link information
2638 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2639 ixgbe_media_status);
2640 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2641 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2642 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2643 ifmedia_add(&adapter->media,
2644 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2645 ifmedia_add(&adapter->media,
2646 IFM_ETHER | IFM_1000_T, 0, NULL);
2648 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2649 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2655 ixgbe_config_link(struct adapter *adapter)
2657 struct ixgbe_hw *hw = &adapter->hw;
2658 u32 autoneg, err = 0;
2659 bool sfp, negotiate;
2661 sfp = ixgbe_is_sfp(hw);
2664 if (hw->phy.multispeed_fiber) {
2665 hw->mac.ops.setup_sfp(hw);
2666 ixgbe_enable_tx_laser(hw);
2667 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2669 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2671 if (hw->mac.ops.check_link)
2672 err = ixgbe_check_link(hw, &autoneg,
2673 &adapter->link_up, FALSE);
2676 autoneg = hw->phy.autoneg_advertised;
2677 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2678 err = hw->mac.ops.get_link_capabilities(hw,
2679 &autoneg, &negotiate);
2682 if (hw->mac.ops.setup_link)
2683 err = hw->mac.ops.setup_link(hw, autoneg,
2684 negotiate, adapter->link_up);
2690 /********************************************************************
2691 * Manage DMA'able memory.
2692 *******************************************************************/
2694 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2698 *(bus_addr_t *) arg = segs->ds_addr;
2703 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2704 struct ixgbe_dma_alloc *dma, int mapflags)
2706 device_t dev = adapter->dev;
2709 r = bus_dma_tag_create(NULL, /* parent */
2710 DBA_ALIGN, 0, /* alignment, bounds */
2711 BUS_SPACE_MAXADDR, /* lowaddr */
2712 BUS_SPACE_MAXADDR, /* highaddr */
2713 NULL, NULL, /* filter, filterarg */
2716 size, /* maxsegsize */
2717 BUS_DMA_ALLOCNOW, /* flags */
2720 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2724 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2725 BUS_DMA_NOWAIT, &dma->dma_map);
2727 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2731 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2735 mapflags | BUS_DMA_NOWAIT);
2737 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2741 dma->dma_size = size;
2744 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2746 bus_dma_tag_destroy(dma->dma_tag);
2748 dma->dma_map = NULL;
2749 dma->dma_tag = NULL;
2754 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2756 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2757 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2758 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2759 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2760 bus_dma_tag_destroy(dma->dma_tag);
2764 /*********************************************************************
2766 * Allocate memory for the transmit and receive rings, and then
2767 * the descriptors associated with each, called only once at attach.
2769 **********************************************************************/
2771 ixgbe_allocate_queues(struct adapter *adapter)
2773 device_t dev = adapter->dev;
2774 struct ix_queue *que;
2775 struct tx_ring *txr;
2776 struct rx_ring *rxr;
2777 int rsize, tsize, error = IXGBE_SUCCESS;
2778 int txconf = 0, rxconf = 0;
2780 /* First allocate the top level queue structs */
2781 if (!(adapter->queues =
2782 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2783 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2784 device_printf(dev, "Unable to allocate queue memory\n");
2789 /* First allocate the TX ring struct memory */
2790 if (!(adapter->tx_rings =
2791 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2792 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2793 device_printf(dev, "Unable to allocate TX ring memory\n");
2798 /* Next allocate the RX */
2799 if (!(adapter->rx_rings =
2800 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2801 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2802 device_printf(dev, "Unable to allocate RX ring memory\n");
2807 /* For the ring itself */
2808 tsize = roundup2(adapter->num_tx_desc *
2809 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2812 * Now set up the TX queues, txconf is needed to handle the
2813 * possibility that things fail midcourse and we need to
2814 * undo memory gracefully
2816 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2817 /* Set up some basics */
2818 txr = &adapter->tx_rings[i];
2819 txr->adapter = adapter;
2822 /* Initialize the TX side lock */
2823 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2824 device_get_nameunit(dev), txr->me);
2825 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2827 if (ixgbe_dma_malloc(adapter, tsize,
2828 &txr->txdma, BUS_DMA_NOWAIT)) {
2830 "Unable to allocate TX Descriptor memory\n");
2834 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2835 bzero((void *)txr->tx_base, tsize);
2837 /* Now allocate transmit buffers for the ring */
2838 if (ixgbe_allocate_transmit_buffers(txr)) {
2840 "Critical Failure setting up transmit buffers\n");
2844 #if 0 /* __FreeBSD_version >= 800000 */
2845 /* Allocate a buf ring */
2846 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2847 M_WAITOK, &txr->tx_mtx);
2848 if (txr->br == NULL) {
2850 "Critical Failure setting up buf ring\n");
2858 * Next the RX queues...
2860 rsize = roundup2(adapter->num_rx_desc *
2861 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2862 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2863 rxr = &adapter->rx_rings[i];
2864 /* Set up some basics */
2865 rxr->adapter = adapter;
2868 /* Initialize the RX side lock */
2869 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2870 device_get_nameunit(dev), rxr->me);
2871 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2873 if (ixgbe_dma_malloc(adapter, rsize,
2874 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2876 "Unable to allocate RxDescriptor memory\n");
2880 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2881 bzero((void *)rxr->rx_base, rsize);
2883 /* Allocate receive buffers for the ring*/
2884 if (ixgbe_allocate_receive_buffers(rxr)) {
2886 "Critical Failure setting up receive buffers\n");
2893 ** Finally set up the queue holding structs
2895 for (int i = 0; i < adapter->num_queues; i++) {
2896 que = &adapter->queues[i];
2897 que->adapter = adapter;
2898 que->txr = &adapter->tx_rings[i];
2899 que->rxr = &adapter->rx_rings[i];
2905 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2906 ixgbe_dma_free(adapter, &rxr->rxdma);
2908 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2909 ixgbe_dma_free(adapter, &txr->txdma);
2910 kfree(adapter->rx_rings, M_DEVBUF);
2912 kfree(adapter->tx_rings, M_DEVBUF);
2914 kfree(adapter->queues, M_DEVBUF);
2919 /*********************************************************************
2921 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2922 * the information needed to transmit a packet on the wire. This is
2923 * called only once at attach, setup is done every reset.
2925 **********************************************************************/
2927 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2929 struct adapter *adapter = txr->adapter;
2930 device_t dev = adapter->dev;
2931 struct ixgbe_tx_buf *txbuf;
2935 * Setup DMA descriptor areas.
2937 if ((error = bus_dma_tag_create(
2939 1, 0, /* alignment, bounds */
2940 BUS_SPACE_MAXADDR, /* lowaddr */
2941 BUS_SPACE_MAXADDR, /* highaddr */
2942 NULL, NULL, /* filter, filterarg */
2943 IXGBE_TSO_SIZE, /* maxsize */
2944 adapter->num_segs, /* nsegments */
2945 PAGE_SIZE, /* maxsegsize */
2948 device_printf(dev,"Unable to allocate TX DMA tag\n");
2952 if (!(txr->tx_buffers =
2953 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2954 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2955 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2960 /* Create the descriptor buffer dma maps */
2961 txbuf = txr->tx_buffers;
2962 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2963 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2965 device_printf(dev, "Unable to create TX DMA map\n");
2972 /* We free all, it handles case where we are in the middle */
2973 ixgbe_free_transmit_structures(adapter);
2977 /*********************************************************************
2979 * Initialize a transmit ring.
2981 **********************************************************************/
2983 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2985 struct adapter *adapter = txr->adapter;
2986 struct ixgbe_tx_buf *txbuf;
2989 struct netmap_adapter *na = NA(adapter->ifp);
2990 struct netmap_slot *slot;
2991 #endif /* DEV_NETMAP */
2993 /* Clear the old ring contents */
2997 * (under lock): if in netmap mode, do some consistency
2998 * checks and set slot to entry 0 of the netmap ring.
3000 slot = netmap_reset(na, NR_TX, txr->me, 0);
3001 #endif /* DEV_NETMAP */
3002 bzero((void *)txr->tx_base,
3003 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3005 txr->next_avail_desc = 0;
3006 txr->next_to_clean = 0;
3008 /* Free any existing tx buffers. */
3009 txbuf = txr->tx_buffers;
3010 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3011 if (txbuf->m_head != NULL) {
3012 bus_dmamap_sync(txr->txtag, txbuf->map,
3013 BUS_DMASYNC_POSTWRITE);
3014 bus_dmamap_unload(txr->txtag, txbuf->map);
3015 m_freem(txbuf->m_head);
3016 txbuf->m_head = NULL;
3020 * In netmap mode, set the map for the packet buffer.
3021 * NOTE: Some drivers (not this one) also need to set
3022 * the physical buffer address in the NIC ring.
3023 * Slots in the netmap ring (indexed by "si") are
3024 * kring->nkr_hwofs positions "ahead" wrt the
3025 * corresponding slot in the NIC ring. In some drivers
3026 * (not here) nkr_hwofs can be negative. Function
3027 * netmap_idx_n2k() handles wraparounds properly.
3030 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3031 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3033 #endif /* DEV_NETMAP */
3034 /* Clear the EOP index */
3035 txbuf->eop_index = -1;
3039 /* Set the rate at which we sample packets */
3040 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3041 txr->atr_sample = atr_sample_rate;
3044 /* Set number of descriptors available */
3045 txr->tx_avail = adapter->num_tx_desc;
3047 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3048 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3049 IXGBE_TX_UNLOCK(txr);
3052 /*********************************************************************
3054 * Initialize all transmit rings.
3056 **********************************************************************/
3058 ixgbe_setup_transmit_structures(struct adapter *adapter)
3060 struct tx_ring *txr = adapter->tx_rings;
3062 for (int i = 0; i < adapter->num_queues; i++, txr++)
3063 ixgbe_setup_transmit_ring(txr);
3068 /*********************************************************************
3070 * Enable transmit unit.
3072 **********************************************************************/
3074 ixgbe_initialize_transmit_units(struct adapter *adapter)
3076 struct tx_ring *txr = adapter->tx_rings;
3077 struct ixgbe_hw *hw = &adapter->hw;
3079 /* Setup the Base and Length of the Tx Descriptor Ring */
3081 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3082 u64 tdba = txr->txdma.dma_paddr;
3085 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3086 (tdba & 0x00000000ffffffffULL));
3087 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3088 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3089 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3091 /* Setup the HW Tx Head and Tail descriptor pointers */
3092 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3093 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3095 /* Setup Transmit Descriptor Cmd Settings */
3096 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3097 txr->queue_status = IXGBE_QUEUE_IDLE;
3099 /* Disable Head Writeback */
3100 switch (hw->mac.type) {
3101 case ixgbe_mac_82598EB:
3102 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3104 case ixgbe_mac_82599EB:
3105 case ixgbe_mac_X540:
3107 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3110 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3111 switch (hw->mac.type) {
3112 case ixgbe_mac_82598EB:
3113 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3115 case ixgbe_mac_82599EB:
3116 case ixgbe_mac_X540:
3118 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3124 if (hw->mac.type != ixgbe_mac_82598EB) {
3125 u32 dmatxctl, rttdcs;
3126 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3127 dmatxctl |= IXGBE_DMATXCTL_TE;
3128 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3129 /* Disable arbiter to set MTQC */
3130 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3131 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3132 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3133 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3134 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3135 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3141 /*********************************************************************
3143 * Free all transmit rings.
3145 **********************************************************************/
3147 ixgbe_free_transmit_structures(struct adapter *adapter)
3149 struct tx_ring *txr = adapter->tx_rings;
3151 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3153 ixgbe_free_transmit_buffers(txr);
3154 ixgbe_dma_free(adapter, &txr->txdma);
3155 IXGBE_TX_UNLOCK(txr);
3156 IXGBE_TX_LOCK_DESTROY(txr);
3158 kfree(adapter->tx_rings, M_DEVBUF);
3161 /*********************************************************************
3163 * Free transmit ring related data structures.
3165 **********************************************************************/
3167 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3169 struct adapter *adapter = txr->adapter;
3170 struct ixgbe_tx_buf *tx_buffer;
3173 INIT_DEBUGOUT("free_transmit_ring: begin");
3175 if (txr->tx_buffers == NULL)
3178 tx_buffer = txr->tx_buffers;
3179 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3180 if (tx_buffer->m_head != NULL) {
3181 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3182 BUS_DMASYNC_POSTWRITE);
3183 bus_dmamap_unload(txr->txtag,
3185 m_freem(tx_buffer->m_head);
3186 tx_buffer->m_head = NULL;
3187 if (tx_buffer->map != NULL) {
3188 bus_dmamap_destroy(txr->txtag,
3190 tx_buffer->map = NULL;
3192 } else if (tx_buffer->map != NULL) {
3193 bus_dmamap_unload(txr->txtag,
3195 bus_dmamap_destroy(txr->txtag,
3197 tx_buffer->map = NULL;
3200 #if 0 /* __FreeBSD_version >= 800000 */
3201 if (txr->br != NULL)
3202 buf_ring_free(txr->br, M_DEVBUF);
3204 if (txr->tx_buffers != NULL) {
3205 kfree(txr->tx_buffers, M_DEVBUF);
3206 txr->tx_buffers = NULL;
3208 if (txr->txtag != NULL) {
3209 bus_dma_tag_destroy(txr->txtag);
3215 /*********************************************************************
3217 * Advanced Context Descriptor setup for VLAN or CSUM
3219 **********************************************************************/
3222 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3224 struct adapter *adapter = txr->adapter;
3225 struct ixgbe_adv_tx_context_desc *TXD;
3226 struct ixgbe_tx_buf *tx_buffer;
3227 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3228 struct ether_vlan_header *eh;
3230 struct ip6_hdr *ip6;
3231 int ehdrlen, ip_hlen = 0;
3234 bool offload = TRUE;
3235 int ctxd = txr->next_avail_desc;
3239 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3242 tx_buffer = &txr->tx_buffers[ctxd];
3243 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3246 ** In advanced descriptors the vlan tag must
3247 ** be placed into the descriptor itself.
3249 if (mp->m_flags & M_VLANTAG) {
3250 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3251 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3252 } else if (offload == FALSE)
3256 * Determine where frame payload starts.
3257 * Jump over vlan headers if already present,
3258 * helpful for QinQ too.
3260 eh = mtod(mp, struct ether_vlan_header *);
3261 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3262 etype = ntohs(eh->evl_proto);
3263 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3265 etype = ntohs(eh->evl_encap_proto);
3266 ehdrlen = ETHER_HDR_LEN;
3269 /* Set the ether header length */
3270 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3274 ip = (struct ip *)(mp->m_data + ehdrlen);
3275 ip_hlen = ip->ip_hl << 2;
3277 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3279 case ETHERTYPE_IPV6:
3280 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3281 ip_hlen = sizeof(struct ip6_hdr);
3282 /* XXX-BZ this will go badly in case of ext hdrs. */
3283 ipproto = ip6->ip6_nxt;
3284 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3291 vlan_macip_lens |= ip_hlen;
3292 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3296 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3297 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3301 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3302 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3307 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3308 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3316 /* Now copy bits into descriptor */
3317 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3318 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3319 TXD->seqnum_seed = htole32(0);
3320 TXD->mss_l4len_idx = htole32(0);
3322 tx_buffer->m_head = NULL;
3323 tx_buffer->eop_index = -1;
3325 /* We've consumed the first desc, adjust counters */
3326 if (++ctxd == adapter->num_tx_desc)
3328 txr->next_avail_desc = ctxd;
3334 /**********************************************************************
3336 * Setup work for hardware segmentation offload (TSO) on
3337 * adapters using advanced tx descriptors
3339 **********************************************************************/
3342 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3345 struct adapter *adapter = txr->adapter;
3346 struct ixgbe_adv_tx_context_desc *TXD;
3347 struct ixgbe_tx_buf *tx_buffer;
3348 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3349 u16 vtag = 0, eh_type;
3350 u32 mss_l4len_idx = 0, len;
3351 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3352 struct ether_vlan_header *eh;
3354 struct ip6_hdr *ip6;
3363 * Determine where frame payload starts.
3364 * Jump over vlan headers if already present
3366 eh = mtod(mp, struct ether_vlan_header *);
3367 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3368 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3369 eh_type = eh->evl_proto;
3371 ehdrlen = ETHER_HDR_LEN;
3372 eh_type = eh->evl_encap_proto;
3375 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3376 len = ehdrlen + sizeof(struct tcphdr);
3377 switch (ntohs(eh_type)) {
3379 case ETHERTYPE_IPV6:
3380 if (mp->m_len < len + sizeof(struct ip6_hdr))
3382 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3383 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3384 if (ip6->ip6_nxt != IPPROTO_TCP)
3386 ip_hlen = sizeof(struct ip6_hdr);
3387 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3388 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3389 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3394 if (mp->m_len < len + sizeof(struct ip))
3396 ip = (struct ip *)(mp->m_data + ehdrlen);
3397 if (ip->ip_p != IPPROTO_TCP)
3400 ip_hlen = ip->ip_hl << 2;
3401 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3402 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3403 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3404 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3405 /* Tell transmit desc to also do IPv4 checksum. */
3406 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3410 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3411 __func__, ntohs(eh_type));
3415 ctxd = txr->next_avail_desc;
3416 tx_buffer = &txr->tx_buffers[ctxd];
3417 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3419 tcp_hlen = th->th_off << 2;
3421 /* This is used in the transmit desc in encap */
3422 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3424 /* VLAN MACLEN IPLEN */
3425 if (mp->m_flags & M_VLANTAG) {
3426 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3427 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3430 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3431 vlan_macip_lens |= ip_hlen;
3432 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3434 /* ADV DTYPE TUCMD */
3435 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3436 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3437 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3440 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3441 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3442 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3444 TXD->seqnum_seed = htole32(0);
3445 tx_buffer->m_head = NULL;
3446 tx_buffer->eop_index = -1;
3448 if (++ctxd == adapter->num_tx_desc)
3452 txr->next_avail_desc = ctxd;
3459 ** This routine parses packet headers so that Flow
3460 ** Director can make a hashed filter table entry
3461 ** allowing traffic flows to be identified and kept
3462 ** on the same cpu. This would be a performance
3463 ** hit, but we only do it at IXGBE_FDIR_RATE of
3467 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3469 struct adapter *adapter = txr->adapter;
3470 struct ix_queue *que;
3474 struct ether_vlan_header *eh;
3475 union ixgbe_atr_hash_dword input = {.dword = 0};
3476 union ixgbe_atr_hash_dword common = {.dword = 0};
3477 int ehdrlen, ip_hlen;
3480 eh = mtod(mp, struct ether_vlan_header *);
3481 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3482 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3483 etype = eh->evl_proto;
3485 ehdrlen = ETHER_HDR_LEN;
3486 etype = eh->evl_encap_proto;
3489 /* Only handling IPv4 */
3490 if (etype != htons(ETHERTYPE_IP))
3493 ip = (struct ip *)(mp->m_data + ehdrlen);
3494 ip_hlen = ip->ip_hl << 2;
3496 /* check if we're UDP or TCP */
3499 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3500 /* src and dst are inverted */
3501 common.port.dst ^= th->th_sport;
3502 common.port.src ^= th->th_dport;
3503 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3506 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3507 /* src and dst are inverted */
3508 common.port.dst ^= uh->uh_sport;
3509 common.port.src ^= uh->uh_dport;
3510 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3516 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3517 if (mp->m_pkthdr.ether_vtag)
3518 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3520 common.flex_bytes ^= etype;
3521 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3523 que = &adapter->queues[txr->me];
3525 ** This assumes the Rx queue and Tx
3526 ** queue are bound to the same CPU
3528 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3529 input, common, que->msix);
3531 #endif /* IXGBE_FDIR */
3533 /**********************************************************************
3535 * Examine each tx_buffer in the used queue. If the hardware is done
3536 * processing the packet then free associated resources. The
3537 * tx_buffer is put back on the free queue.
3539 **********************************************************************/
3541 ixgbe_txeof(struct tx_ring *txr)
3543 struct adapter *adapter = txr->adapter;
3544 struct ifnet *ifp = adapter->ifp;
3545 u32 first, last, done, processed;
3546 struct ixgbe_tx_buf *tx_buffer;
3547 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3549 KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3552 if (ifp->if_capenable & IFCAP_NETMAP) {
3553 struct netmap_adapter *na = NA(ifp);
3554 struct netmap_kring *kring = &na->tx_rings[txr->me];
3556 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3558 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3559 BUS_DMASYNC_POSTREAD);
3561 * In netmap mode, all the work is done in the context
3562 * of the client thread. Interrupt handlers only wake up
3563 * clients, which may be sleeping on individual rings
3564 * or on a global resource for all rings.
3565 * To implement tx interrupt mitigation, we wake up the client
3566 * thread roughly every half ring, even if the NIC interrupts
3567 * more frequently. This is implemented as follows:
3568 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3569 * the slot that should wake up the thread (nkr_num_slots
3570 * means the user thread should not be woken up);
3571 * - the driver ignores tx interrupts unless netmap_mitigate=0
3572 * or the slot has the DD bit set.
3574 * When the driver has separate locks, we need to
3575 * release and re-acquire txlock to avoid deadlocks.
3576 * XXX see if we can find a better way.
3578 if (!netmap_mitigate ||
3579 (kring->nr_kflags < kring->nkr_num_slots &&
3580 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3581 kring->nr_kflags = kring->nkr_num_slots;
3582 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3583 IXGBE_TX_UNLOCK(txr);
3584 IXGBE_CORE_LOCK(adapter);
3585 selwakeuppri(&na->tx_si, PI_NET);
3586 IXGBE_CORE_UNLOCK(adapter);
3591 #endif /* DEV_NETMAP */
3593 if (txr->tx_avail == adapter->num_tx_desc) {
3594 txr->queue_status = IXGBE_QUEUE_IDLE;
3599 first = txr->next_to_clean;
3600 tx_buffer = &txr->tx_buffers[first];
3601 /* For cleanup we just use legacy struct */
3602 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3603 last = tx_buffer->eop_index;
3606 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3609 ** Get the index of the first descriptor
3610 ** BEYOND the EOP and call that 'done'.
3611 ** I do this so the comparison in the
3612 ** inner while loop below can be simple
3614 if (++last == adapter->num_tx_desc) last = 0;
3617 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3618 BUS_DMASYNC_POSTREAD);
3620 ** Only the EOP descriptor of a packet now has the DD
3621 ** bit set, this is what we look for...
3623 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3624 /* We clean the range of the packet */
3625 while (first != done) {
3626 tx_desc->upper.data = 0;
3627 tx_desc->lower.data = 0;
3628 tx_desc->buffer_addr = 0;
3632 if (tx_buffer->m_head) {
3634 tx_buffer->m_head->m_pkthdr.len;
3635 bus_dmamap_sync(txr->txtag,
3637 BUS_DMASYNC_POSTWRITE);
3638 bus_dmamap_unload(txr->txtag,
3640 m_freem(tx_buffer->m_head);
3641 tx_buffer->m_head = NULL;
3642 tx_buffer->map = NULL;
3644 tx_buffer->eop_index = -1;
3645 txr->watchdog_time = ticks;
3647 if (++first == adapter->num_tx_desc)
3650 tx_buffer = &txr->tx_buffers[first];
3652 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3656 /* See if there is more work now */
3657 last = tx_buffer->eop_index;
3660 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3661 /* Get next done point */
3662 if (++last == adapter->num_tx_desc) last = 0;
3667 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3668 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3670 txr->next_to_clean = first;
3673 ** Watchdog calculation, we know there's
3674 ** work outstanding or the first return
3675 ** would have been taken, so none processed
3676 ** for too long indicates a hang.
3678 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3679 txr->queue_status = IXGBE_QUEUE_HUNG;
3681 /* With a minimum free clear the depleted state bit. */
3682 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3683 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3685 if (txr->tx_avail == adapter->num_tx_desc) {
3686 txr->queue_status = IXGBE_QUEUE_IDLE;
3693 /*********************************************************************
3695 * Refresh mbuf buffers for RX descriptor rings
3696 * - now keeps its own state so discards due to resource
3697 * exhaustion are unnecessary, if an mbuf cannot be obtained
3698 * it just returns, keeping its placeholder, thus it can simply
3699 * be recalled to try again.
3701 **********************************************************************/
3703 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3705 struct adapter *adapter = rxr->adapter;
3706 bus_dma_segment_t hseg[1];
3707 bus_dma_segment_t pseg[1];
3708 struct ixgbe_rx_buf *rxbuf;
3709 struct mbuf *mh, *mp;
3710 int i, j, nsegs, error;
3711 bool refreshed = FALSE;
3713 i = j = rxr->next_to_refresh;
3714 /* Control the loop with one beyond */
3715 if (++j == adapter->num_rx_desc)
3718 while (j != limit) {
3719 rxbuf = &rxr->rx_buffers[i];
3720 if (rxr->hdr_split == FALSE)
3723 if (rxbuf->m_head == NULL) {
3724 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3730 mh->m_pkthdr.len = mh->m_len = MHLEN;
3732 mh->m_flags |= M_PKTHDR;
3733 /* Get the memory mapping */
3734 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3735 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3737 kprintf("Refresh mbufs: hdr dmamap load"
3738 " failure - %d\n", error);
3740 rxbuf->m_head = NULL;
3744 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3745 BUS_DMASYNC_PREREAD);
3746 rxr->rx_base[i].read.hdr_addr =
3747 htole64(hseg[0].ds_addr);
3750 if (rxbuf->m_pack == NULL) {
3751 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3752 M_PKTHDR, adapter->rx_mbuf_sz);
3758 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3759 /* Get the memory mapping */
3760 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3761 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3763 kprintf("Refresh mbufs: payload dmamap load"
3764 " failure - %d\n", error);
3766 rxbuf->m_pack = NULL;
3770 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3771 BUS_DMASYNC_PREREAD);
3772 rxr->rx_base[i].read.pkt_addr =
3773 htole64(pseg[0].ds_addr);
3776 /* Next is precalculated */
3778 rxr->next_to_refresh = i;
3779 if (++j == adapter->num_rx_desc)
3783 if (refreshed) /* Update hardware tail index */
3784 IXGBE_WRITE_REG(&adapter->hw,
3785 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3789 /*********************************************************************
3791 * Allocate memory for rx_buffer structures. Since we use one
3792 * rx_buffer per received packet, the maximum number of rx_buffer's
3793 * that we'll need is equal to the number of receive descriptors
3794 * that we've allocated.
3796 **********************************************************************/
3798 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3800 struct adapter *adapter = rxr->adapter;
3801 device_t dev = adapter->dev;
3802 struct ixgbe_rx_buf *rxbuf;
3803 int i, bsize, error;
3805 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3806 if (!(rxr->rx_buffers =
3807 (struct ixgbe_rx_buf *) kmalloc(bsize,
3808 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3809 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3814 if ((error = bus_dma_tag_create(NULL, /* parent */
3815 1, 0, /* alignment, bounds */
3816 BUS_SPACE_MAXADDR, /* lowaddr */
3817 BUS_SPACE_MAXADDR, /* highaddr */
3818 NULL, NULL, /* filter, filterarg */
3819 MSIZE, /* maxsize */
3821 MSIZE, /* maxsegsize */
3824 device_printf(dev, "Unable to create RX DMA tag\n");
3828 if ((error = bus_dma_tag_create(NULL, /* parent */
3829 1, 0, /* alignment, bounds */
3830 BUS_SPACE_MAXADDR, /* lowaddr */
3831 BUS_SPACE_MAXADDR, /* highaddr */
3832 NULL, NULL, /* filter, filterarg */
3833 MJUM16BYTES, /* maxsize */
3835 MJUM16BYTES, /* maxsegsize */
3838 device_printf(dev, "Unable to create RX DMA tag\n");
3842 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3843 rxbuf = &rxr->rx_buffers[i];
3844 error = bus_dmamap_create(rxr->htag,
3845 BUS_DMA_NOWAIT, &rxbuf->hmap);
3847 device_printf(dev, "Unable to create RX head map\n");
3850 error = bus_dmamap_create(rxr->ptag,
3851 BUS_DMA_NOWAIT, &rxbuf->pmap);
3853 device_printf(dev, "Unable to create RX pkt map\n");
3861 /* Frees all, but can handle partial completion */
3862 ixgbe_free_receive_structures(adapter);
3867 ** Used to detect a descriptor that has
3868 ** been merged by Hardware RSC.
3871 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3873 return (le32toh(rx->wb.lower.lo_dword.data) &
3874 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3877 /*********************************************************************
3879 * Initialize Hardware RSC (LRO) feature on 82599
3880 * for an RX ring, this is toggled by the LRO capability
3881 * even though it is transparent to the stack.
3883 **********************************************************************/
3886 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3888 struct adapter *adapter = rxr->adapter;
3889 struct ixgbe_hw *hw = &adapter->hw;
3890 u32 rscctrl, rdrxctl;
3892 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3893 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3894 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3895 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3896 #endif /* DEV_NETMAP */
3897 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3898 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3899 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3901 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3902 rscctrl |= IXGBE_RSCCTL_RSCEN;
3904 ** Limit the total number of descriptors that
3905 ** can be combined, so it does not exceed 64K
3907 if (adapter->rx_mbuf_sz == MCLBYTES)
3908 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3909 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3910 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3911 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3912 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3913 else /* Using 16K cluster */
3914 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3916 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3918 /* Enable TCP header recognition */
3919 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3920 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3921 IXGBE_PSRTYPE_TCPHDR));
3923 /* Disable RSC for ACK packets */
3924 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3925 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3932 ixgbe_free_receive_ring(struct rx_ring *rxr)
3934 struct adapter *adapter;
3935 struct ixgbe_rx_buf *rxbuf;
3938 adapter = rxr->adapter;
3939 for (i = 0; i < adapter->num_rx_desc; i++) {
3940 rxbuf = &rxr->rx_buffers[i];
3941 if (rxbuf->m_head != NULL) {
3942 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3943 BUS_DMASYNC_POSTREAD);
3944 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3945 rxbuf->m_head->m_flags |= M_PKTHDR;
3946 m_freem(rxbuf->m_head);
3948 if (rxbuf->m_pack != NULL) {
3949 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3950 BUS_DMASYNC_POSTREAD);
3951 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3952 rxbuf->m_pack->m_flags |= M_PKTHDR;
3953 m_freem(rxbuf->m_pack);
3955 rxbuf->m_head = NULL;
3956 rxbuf->m_pack = NULL;
3961 /*********************************************************************
3963 * Initialize a receive ring and its buffers.
3965 **********************************************************************/
3967 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3969 struct adapter *adapter;
3972 struct ixgbe_rx_buf *rxbuf;
3973 bus_dma_segment_t pseg[1], hseg[1];
3975 struct lro_ctrl *lro = &rxr->lro;
3977 int rsize, nsegs, error = 0;
3979 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3980 struct netmap_slot *slot;
3981 #endif /* DEV_NETMAP */
3983 adapter = rxr->adapter;
3987 /* Clear the ring contents */
3990 /* same as in ixgbe_setup_transmit_ring() */
3991 slot = netmap_reset(na, NR_RX, rxr->me, 0);
3992 #endif /* DEV_NETMAP */
3993 rsize = roundup2(adapter->num_rx_desc *
3994 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3995 bzero((void *)rxr->rx_base, rsize);
3997 /* Free current RX buffer structs and their mbufs */
3998 ixgbe_free_receive_ring(rxr);
4000 /* Configure header split? */
4001 if (ixgbe_header_split)
4002 rxr->hdr_split = TRUE;
4004 /* Now replenish the mbufs */
4005 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4006 struct mbuf *mh, *mp;
4008 rxbuf = &rxr->rx_buffers[j];
4011 * In netmap mode, fill the map and set the buffer
4012 * address in the NIC ring, considering the offset
4013 * between the netmap and NIC rings (see comment in
4014 * ixgbe_setup_transmit_ring() ). No need to allocate
4015 * an mbuf, so end the block with a continue;
4018 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4022 addr = PNMB(slot + sj, &paddr);
4023 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4024 /* Update descriptor */
4025 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4028 #endif /* DEV_NETMAP */
4030 ** Don't allocate mbufs if not
4031 ** doing header split, its wasteful
4033 if (rxr->hdr_split == FALSE)
4036 /* First the header */
4037 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4038 if (rxbuf->m_head == NULL) {
4042 m_adj(rxbuf->m_head, ETHER_ALIGN);
4044 mh->m_len = mh->m_pkthdr.len = MHLEN;
4045 mh->m_flags |= M_PKTHDR;
4046 /* Get the memory mapping */
4047 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4048 rxbuf->hmap, rxbuf->m_head, hseg, 1,
4049 &nsegs, BUS_DMA_NOWAIT);
4051 if (error != 0) /* Nothing elegant to do here */
4053 bus_dmamap_sync(rxr->htag,
4054 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4055 /* Update descriptor */
4056 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4059 /* Now the payload cluster */
4060 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4061 M_PKTHDR, adapter->rx_mbuf_sz);
4062 if (rxbuf->m_pack == NULL) {
4067 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4068 /* Get the memory mapping */
4069 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4070 rxbuf->pmap, mp, hseg, 1,
4071 &nsegs, BUS_DMA_NOWAIT);
4074 bus_dmamap_sync(rxr->ptag,
4075 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4076 /* Update descriptor */
4077 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4081 /* Setup our descriptor indices */
4082 rxr->next_to_check = 0;
4083 rxr->next_to_refresh = 0;
4084 rxr->lro_enabled = FALSE;
4085 rxr->rx_split_packets = 0;
4087 rxr->discard = FALSE;
4088 rxr->vtag_strip = FALSE;
4090 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4091 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4094 ** Now set up the LRO interface:
4095 ** 82598 uses software LRO, the
4096 ** 82599 and X540 use a hardware assist.
4099 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4100 (ifp->if_capenable & IFCAP_RXCSUM) &&
4101 (ifp->if_capenable & IFCAP_LRO))
4102 ixgbe_setup_hw_rsc(rxr);
4103 else if (ifp->if_capenable & IFCAP_LRO) {
4104 int err = tcp_lro_init(lro);
4106 device_printf(dev, "LRO Initialization failed!\n");
4109 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4110 rxr->lro_enabled = TRUE;
4111 lro->ifp = adapter->ifp;
4115 IXGBE_RX_UNLOCK(rxr);
4119 ixgbe_free_receive_ring(rxr);
4120 IXGBE_RX_UNLOCK(rxr);
4124 /*********************************************************************
4126 * Initialize all receive rings.
4128 **********************************************************************/
4130 ixgbe_setup_receive_structures(struct adapter *adapter)
4132 struct rx_ring *rxr = adapter->rx_rings;
4135 for (j = 0; j < adapter->num_queues; j++, rxr++)
4136 if (ixgbe_setup_receive_ring(rxr))
4142 * Free RX buffers allocated so far, we will only handle
4143 * the rings that completed, the failing case will have
4144 * cleaned up for itself. 'j' failed, so its the terminus.
4146 for (int i = 0; i < j; ++i) {
4147 rxr = &adapter->rx_rings[i];
4148 ixgbe_free_receive_ring(rxr);
4154 /*********************************************************************
4156 * Setup receive registers and features.
4158 **********************************************************************/
4159 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4161 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4164 ixgbe_initialize_receive_units(struct adapter *adapter)
4166 struct rx_ring *rxr = adapter->rx_rings;
4167 struct ixgbe_hw *hw = &adapter->hw;
4168 struct ifnet *ifp = adapter->ifp;
4169 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
4170 u32 reta, mrqc = 0, hlreg, random[10];
4174 * Make sure receives are disabled while
4175 * setting up the descriptor ring
4177 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4178 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4179 rxctrl & ~IXGBE_RXCTRL_RXEN);
4181 /* Enable broadcasts */
4182 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4183 fctrl |= IXGBE_FCTRL_BAM;
4184 fctrl |= IXGBE_FCTRL_DPF;
4185 fctrl |= IXGBE_FCTRL_PMCF;
4186 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4188 /* Set for Jumbo Frames? */
4189 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4190 if (ifp->if_mtu > ETHERMTU)
4191 hlreg |= IXGBE_HLREG0_JUMBOEN;
4193 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4195 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4196 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4197 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4199 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4200 #endif /* DEV_NETMAP */
4201 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4203 bufsz = (adapter->rx_mbuf_sz +
4204 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4206 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4207 u64 rdba = rxr->rxdma.dma_paddr;
4209 /* Setup the Base and Length of the Rx Descriptor Ring */
4210 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4211 (rdba & 0x00000000ffffffffULL));
4212 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4213 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4214 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4216 /* Set up the SRRCTL register */
4217 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4218 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4219 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4221 if (rxr->hdr_split) {
4222 /* Use a standard mbuf for the header */
4223 srrctl |= ((IXGBE_RX_HDR <<
4224 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4225 & IXGBE_SRRCTL_BSIZEHDR_MASK);
4226 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4228 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4229 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4231 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4232 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4233 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4236 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4237 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4238 IXGBE_PSRTYPE_UDPHDR |
4239 IXGBE_PSRTYPE_IPV4HDR |
4240 IXGBE_PSRTYPE_IPV6HDR;
4241 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4244 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4247 if (adapter->num_queues > 1) {
4251 /* set up random bits */
4252 karc4rand(&random, sizeof(random));
4254 /* Set up the redirection table */
4255 for (i = 0, j = 0; i < 128; i++, j++) {
4256 if (j == adapter->num_queues) j = 0;
4257 reta = (reta << 8) | (j * 0x11);
4259 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4262 /* Now fill our hash function seeds */
4263 for (int i = 0; i < 10; i++)
4264 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4266 /* Perform hash on these packet types */
4267 mrqc = IXGBE_MRQC_RSSEN
4268 | IXGBE_MRQC_RSS_FIELD_IPV4
4269 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4270 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4271 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4272 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4273 | IXGBE_MRQC_RSS_FIELD_IPV6
4274 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4275 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4276 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4277 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4279 /* RSS and RX IPP Checksum are mutually exclusive */
4280 rxcsum |= IXGBE_RXCSUM_PCSD;
4283 if (ifp->if_capenable & IFCAP_RXCSUM)
4284 rxcsum |= IXGBE_RXCSUM_PCSD;
4286 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4287 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4289 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4294 /*********************************************************************
4296 * Free all receive rings.
4298 **********************************************************************/
4300 ixgbe_free_receive_structures(struct adapter *adapter)
4302 struct rx_ring *rxr = adapter->rx_rings;
4304 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4306 struct lro_ctrl *lro = &rxr->lro;
4308 ixgbe_free_receive_buffers(rxr);
4309 /* Free LRO memory */
4313 /* Free the ring memory as well */
4314 ixgbe_dma_free(adapter, &rxr->rxdma);
4317 kfree(adapter->rx_rings, M_DEVBUF);
4321 /*********************************************************************
4323 * Free receive ring data structures
4325 **********************************************************************/
4327 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4329 struct adapter *adapter = rxr->adapter;
4330 struct ixgbe_rx_buf *rxbuf;
4332 INIT_DEBUGOUT("free_receive_structures: begin");
4334 /* Cleanup any existing buffers */
4335 if (rxr->rx_buffers != NULL) {
4336 for (int i = 0; i < adapter->num_rx_desc; i++) {
4337 rxbuf = &rxr->rx_buffers[i];
4338 if (rxbuf->m_head != NULL) {
4339 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4340 BUS_DMASYNC_POSTREAD);
4341 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4342 rxbuf->m_head->m_flags |= M_PKTHDR;
4343 m_freem(rxbuf->m_head);
4345 if (rxbuf->m_pack != NULL) {
4346 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4347 BUS_DMASYNC_POSTREAD);
4348 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4349 rxbuf->m_pack->m_flags |= M_PKTHDR;
4350 m_freem(rxbuf->m_pack);
4352 rxbuf->m_head = NULL;
4353 rxbuf->m_pack = NULL;
4354 if (rxbuf->hmap != NULL) {
4355 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4358 if (rxbuf->pmap != NULL) {
4359 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4363 if (rxr->rx_buffers != NULL) {
4364 kfree(rxr->rx_buffers, M_DEVBUF);
4365 rxr->rx_buffers = NULL;
4369 if (rxr->htag != NULL) {
4370 bus_dma_tag_destroy(rxr->htag);
4373 if (rxr->ptag != NULL) {
4374 bus_dma_tag_destroy(rxr->ptag);
4381 static __inline void
4382 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4386 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4387 * should be computed by hardware. Also it should not have VLAN tag in
4388 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4391 if (rxr->lro_enabled &&
4392 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4393 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4394 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4395 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4396 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4397 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4398 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4399 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4401 * Send to the stack if:
4402 ** - LRO not enabled, or
4403 ** - no LRO resources, or
4404 ** - lro enqueue fails
4406 if (rxr->lro.lro_cnt != 0)
4407 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4411 IXGBE_RX_UNLOCK(rxr);
4412 (*ifp->if_input)(ifp, m);
4416 static __inline void
4417 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4419 struct ixgbe_rx_buf *rbuf;
4421 rbuf = &rxr->rx_buffers[i];
4423 if (rbuf->fmp != NULL) {/* Partial chain ? */
4424 rbuf->fmp->m_flags |= M_PKTHDR;
4430 ** With advanced descriptors the writeback
4431 ** clobbers the buffer addrs, so its easier
4432 ** to just free the existing mbufs and take
4433 ** the normal refresh path to get new buffers
4437 m_free(rbuf->m_head);
4438 rbuf->m_head = NULL;
4442 m_free(rbuf->m_pack);
4443 rbuf->m_pack = NULL;
4450 /*********************************************************************
4452 * This routine executes in interrupt context. It replenishes
4453 * the mbufs in the descriptor and sends data which has been
4454 * dma'ed into host memory to upper layer.
4456 * We loop at most count times if count is > 0, or until done if
4459 * Return TRUE for more work, FALSE for all clean.
4460 *********************************************************************/
4462 ixgbe_rxeof(struct ix_queue *que, int count)
4464 struct adapter *adapter = que->adapter;
4465 struct rx_ring *rxr = que->rxr;
4466 struct ifnet *ifp = adapter->ifp;
4468 struct lro_ctrl *lro = &rxr->lro;
4469 struct lro_entry *queued;
4471 int i, nextp, processed = 0;
4473 union ixgbe_adv_rx_desc *cur;
4474 struct ixgbe_rx_buf *rbuf, *nbuf;
4479 if (ifp->if_capenable & IFCAP_NETMAP) {
4481 * Same as the txeof routine: only wakeup clients on intr.
4482 * NKR_PENDINTR in nr_kflags is used to implement interrupt
4483 * mitigation (ixgbe_rxsync() will not look for new packets
4484 * unless NKR_PENDINTR is set).
4486 struct netmap_adapter *na = NA(ifp);
4488 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4489 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4490 IXGBE_RX_UNLOCK(rxr);
4491 IXGBE_CORE_LOCK(adapter);
4492 selwakeuppri(&na->rx_si, PI_NET);
4493 IXGBE_CORE_UNLOCK(adapter);
4496 #endif /* DEV_NETMAP */
4497 for (i = rxr->next_to_check; count != 0;) {
4498 struct mbuf *sendmp, *mh, *mp;
4500 u16 hlen, plen, hdr;
4504 /* Sync the ring. */
4505 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4506 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4508 cur = &rxr->rx_base[i];
4509 staterr = le32toh(cur->wb.upper.status_error);
4511 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4513 if ((ifp->if_flags & IFF_RUNNING) == 0)
4520 cur->wb.upper.status_error = 0;
4521 rbuf = &rxr->rx_buffers[i];
4525 plen = le16toh(cur->wb.upper.length);
4526 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4527 IXGBE_RXDADV_PKTTYPE_MASK;
4528 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4529 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4531 /* Process vlan info */
4532 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4533 vtag = le16toh(cur->wb.upper.vlan);
4535 /* Make sure bad packets are discarded */
4536 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4539 rxr->rx_discarded++;
4541 rxr->discard = FALSE;
4543 rxr->discard = TRUE;
4544 ixgbe_rx_discard(rxr, i);
4549 ** On 82599 which supports a hardware
4550 ** LRO (called HW RSC), packets need
4551 ** not be fragmented across sequential
4552 ** descriptors, rather the next descriptor
4553 ** is indicated in bits of the descriptor.
4554 ** This also means that we might proceses
4555 ** more than one packet at a time, something
4556 ** that has never been true before, it
4557 ** required eliminating global chain pointers
4558 ** in favor of what we are doing here. -jfv
4562 ** Figure out the next descriptor
4565 if (rxr->hw_rsc == TRUE) {
4566 rsc = ixgbe_rsc_count(cur);
4567 rxr->rsc_num += (rsc - 1);
4569 if (rsc) { /* Get hardware index */
4571 IXGBE_RXDADV_NEXTP_MASK) >>
4572 IXGBE_RXDADV_NEXTP_SHIFT);
4573 } else { /* Just sequential */
4575 if (nextp == adapter->num_rx_desc)
4578 nbuf = &rxr->rx_buffers[nextp];
4582 ** The header mbuf is ONLY used when header
4583 ** split is enabled, otherwise we get normal
4584 ** behavior, ie, both header and payload
4585 ** are DMA'd into the payload buffer.
4587 ** Rather than using the fmp/lmp global pointers
4588 ** we now keep the head of a packet chain in the
4589 ** buffer struct and pass this along from one
4590 ** descriptor to the next, until we get EOP.
4592 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4593 /* This must be an initial descriptor */
4594 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4595 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4596 if (hlen > IXGBE_RX_HDR)
4597 hlen = IXGBE_RX_HDR;
4599 mh->m_flags |= M_PKTHDR;
4601 mh->m_pkthdr.len = mh->m_len;
4602 /* Null buf pointer so it is refreshed */
4603 rbuf->m_head = NULL;
4605 ** Check the payload length, this
4606 ** could be zero if its a small
4612 mp->m_flags &= ~M_PKTHDR;
4614 mh->m_pkthdr.len += mp->m_len;
4615 /* Null buf pointer so it is refreshed */
4616 rbuf->m_pack = NULL;
4617 rxr->rx_split_packets++;
4620 ** Now create the forward
4621 ** chain so when complete
4625 /* stash the chain head */
4627 /* Make forward chain */
4629 mp->m_next = nbuf->m_pack;
4631 mh->m_next = nbuf->m_pack;
4633 /* Singlet, prepare to send */
4635 /* If hardware handled vtag */
4637 sendmp->m_pkthdr.ether_vlantag = vtag;
4638 sendmp->m_flags |= M_VLANTAG;
4643 ** Either no header split, or a
4644 ** secondary piece of a fragmented
4649 ** See if there is a stored head
4650 ** that determines what we are
4653 rbuf->m_pack = rbuf->fmp = NULL;
4655 if (sendmp != NULL) { /* secondary frag */
4656 mp->m_flags &= ~M_PKTHDR;
4657 sendmp->m_pkthdr.len += mp->m_len;
4659 /* first desc of a non-ps chain */
4661 sendmp->m_flags |= M_PKTHDR;
4662 sendmp->m_pkthdr.len = mp->m_len;
4663 if (staterr & IXGBE_RXD_STAT_VP) {
4664 sendmp->m_pkthdr.ether_vlantag = vtag;
4665 sendmp->m_flags |= M_VLANTAG;
4668 /* Pass the head pointer on */
4672 mp->m_next = nbuf->m_pack;
4676 /* Sending this frame? */
4678 sendmp->m_pkthdr.rcvif = ifp;
4681 /* capture data for AIM */
4682 rxr->bytes += sendmp->m_pkthdr.len;
4683 rxr->rx_bytes += sendmp->m_pkthdr.len;
4684 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4685 ixgbe_rx_checksum(staterr, sendmp, ptype);
4686 #if 0 /* __FreeBSD_version >= 800000 */
4687 sendmp->m_pkthdr.flowid = que->msix;
4688 sendmp->m_flags |= M_FLOWID;
4692 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4693 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4695 /* Advance our pointers to the next descriptor. */
4696 if (++i == adapter->num_rx_desc)
4699 /* Now send to the stack or do LRO */
4700 if (sendmp != NULL) {
4701 rxr->next_to_check = i;
4702 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4703 i = rxr->next_to_check;
4706 /* Every 8 descriptors we go to refresh mbufs */
4707 if (processed == 8) {
4708 ixgbe_refresh_mbufs(rxr, i);
4713 /* Refresh any remaining buf structs */
4714 if (ixgbe_rx_unrefreshed(rxr))
4715 ixgbe_refresh_mbufs(rxr, i);
4717 rxr->next_to_check = i;
4720 * Flush any outstanding LRO work
4723 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4724 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4725 tcp_lro_flush(lro, queued);
4729 IXGBE_RX_UNLOCK(rxr);
4732 ** We still have cleaning to do?
4733 ** Schedule another interrupt if so.
4735 if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4736 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4744 /*********************************************************************
4746 * Verify that the hardware indicated that the checksum is valid.
4747 * Inform the stack about the status of checksum so that stack
4748 * doesn't spend time verifying the checksum.
4750 *********************************************************************/
4752 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4754 u16 status = (u16) staterr;
4755 u8 errors = (u8) (staterr >> 24);
4758 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4759 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4762 if (status & IXGBE_RXD_STAT_IPCS) {
4763 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4764 /* IP Checksum Good */
4765 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4766 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4769 mp->m_pkthdr.csum_flags = 0;
4771 if (status & IXGBE_RXD_STAT_L4CS) {
4772 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4775 type = CSUM_SCTP_VALID;
4777 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4778 mp->m_pkthdr.csum_flags |= type;
4780 mp->m_pkthdr.csum_data = htons(0xffff);
4788 ** This routine is run via an vlan config EVENT,
4789 ** it enables us to use the HW Filter table since
4790 ** we can get the vlan id. This just creates the
4791 ** entry in the soft version of the VFTA, init will
4792 ** repopulate the real table.
4795 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4797 struct adapter *adapter = ifp->if_softc;
4800 if (ifp->if_softc != arg) /* Not our event */
4803 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4806 IXGBE_CORE_LOCK(adapter);
4807 index = (vtag >> 5) & 0x7F;
4809 adapter->shadow_vfta[index] |= (1 << bit);
4810 ++adapter->num_vlans;
4811 ixgbe_init_locked(adapter);
4812 IXGBE_CORE_UNLOCK(adapter);
4816 ** This routine is run via an vlan
4817 ** unconfig EVENT, remove our entry
4818 ** in the soft vfta.
4821 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4823 struct adapter *adapter = ifp->if_softc;
4826 if (ifp->if_softc != arg)
4829 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4832 IXGBE_CORE_LOCK(adapter);
4833 index = (vtag >> 5) & 0x7F;
4835 adapter->shadow_vfta[index] &= ~(1 << bit);
4836 --adapter->num_vlans;
4837 /* Re-init to load the changes */
4838 ixgbe_init_locked(adapter);
4839 IXGBE_CORE_UNLOCK(adapter);
4843 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4845 struct ifnet *ifp = adapter->ifp;
4846 struct ixgbe_hw *hw = &adapter->hw;
4847 struct rx_ring *rxr;
4851 ** We get here thru init_locked, meaning
4852 ** a soft reset, this has already cleared
4853 ** the VFTA and other state, so if there
4854 ** have been no vlan's registered do nothing.
4856 if (adapter->num_vlans == 0)
4860 ** A soft reset zero's out the VFTA, so
4861 ** we need to repopulate it now.
4863 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4864 if (adapter->shadow_vfta[i] != 0)
4865 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4866 adapter->shadow_vfta[i]);
4868 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4869 /* Enable the Filter Table if enabled */
4870 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4871 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4872 ctrl |= IXGBE_VLNCTRL_VFE;
4874 if (hw->mac.type == ixgbe_mac_82598EB)
4875 ctrl |= IXGBE_VLNCTRL_VME;
4876 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4878 /* Setup the queues for vlans */
4879 for (int i = 0; i < adapter->num_queues; i++) {
4880 rxr = &adapter->rx_rings[i];
4881 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4882 if (hw->mac.type != ixgbe_mac_82598EB) {
4883 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4884 ctrl |= IXGBE_RXDCTL_VME;
4885 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4887 rxr->vtag_strip = TRUE;
4892 ixgbe_enable_intr(struct adapter *adapter)
4894 struct ixgbe_hw *hw = &adapter->hw;
4895 struct ix_queue *que = adapter->queues;
4896 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4899 /* Enable Fan Failure detection */
4900 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4901 mask |= IXGBE_EIMS_GPI_SDP1;
4903 mask |= IXGBE_EIMS_ECC;
4904 mask |= IXGBE_EIMS_GPI_SDP0;
4905 mask |= IXGBE_EIMS_GPI_SDP1;
4906 mask |= IXGBE_EIMS_GPI_SDP2;
4908 mask |= IXGBE_EIMS_FLOW_DIR;
4912 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4914 /* With RSS we use auto clear */
4915 if (adapter->msix_mem) {
4916 mask = IXGBE_EIMS_ENABLE_MASK;
4917 /* Don't autoclear Link */
4918 mask &= ~IXGBE_EIMS_OTHER;
4919 mask &= ~IXGBE_EIMS_LSC;
4920 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4924 ** Now enable all queues, this is done separately to
4925 ** allow for handling the extended (beyond 32) MSIX
4926 ** vectors that can be used by 82599
4928 for (int i = 0; i < adapter->num_queues; i++, que++)
4929 ixgbe_enable_queue(adapter, que->msix);
4931 IXGBE_WRITE_FLUSH(hw);
4937 ixgbe_disable_intr(struct adapter *adapter)
4939 if (adapter->msix_mem)
4940 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4941 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4942 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4944 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4945 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4946 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4948 IXGBE_WRITE_FLUSH(&adapter->hw);
4953 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4957 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4964 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4966 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4973 ** Setup the correct IVAR register for a particular MSIX interrupt
4974 ** (yes this is all very magic and confusing :)
4975 ** - entry is the register array entry
4976 ** - vector is the MSIX vector for this queue
4977 ** - type is RX/TX/MISC
4980 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
4982 struct ixgbe_hw *hw = &adapter->hw;
4985 vector |= IXGBE_IVAR_ALLOC_VAL;
4987 switch (hw->mac.type) {
4989 case ixgbe_mac_82598EB:
4991 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
4993 entry += (type * 64);
4994 index = (entry >> 2) & 0x1F;
4995 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
4996 ivar &= ~(0xFF << (8 * (entry & 0x3)));
4997 ivar |= (vector << (8 * (entry & 0x3)));
4998 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5001 case ixgbe_mac_82599EB:
5002 case ixgbe_mac_X540:
5003 if (type == -1) { /* MISC IVAR */
5004 index = (entry & 1) * 8;
5005 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5006 ivar &= ~(0xFF << index);
5007 ivar |= (vector << index);
5008 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5009 } else { /* RX/TX IVARS */
5010 index = (16 * (entry & 1)) + (8 * type);
5011 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5012 ivar &= ~(0xFF << index);
5013 ivar |= (vector << index);
5014 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5023 ixgbe_configure_ivars(struct adapter *adapter)
5025 struct ix_queue *que = adapter->queues;
5028 if (ixgbe_max_interrupt_rate > 0)
5029 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5033 for (int i = 0; i < adapter->num_queues; i++, que++) {
5034 /* First the RX queue entry */
5035 ixgbe_set_ivar(adapter, i, que->msix, 0);
5036 /* ... and the TX */
5037 ixgbe_set_ivar(adapter, i, que->msix, 1);
5038 /* Set an Initial EITR value */
5039 IXGBE_WRITE_REG(&adapter->hw,
5040 IXGBE_EITR(que->msix), newitr);
5043 /* For the Link interrupt */
5044 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5048 ** ixgbe_sfp_probe - called in the local timer to
5049 ** determine if a port had optics inserted.
5051 static bool ixgbe_sfp_probe(struct adapter *adapter)
5053 struct ixgbe_hw *hw = &adapter->hw;
5054 device_t dev = adapter->dev;
5055 bool result = FALSE;
5057 if ((hw->phy.type == ixgbe_phy_nl) &&
5058 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5059 s32 ret = hw->phy.ops.identify_sfp(hw);
5062 ret = hw->phy.ops.reset(hw);
5063 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5064 device_printf(dev,"Unsupported SFP+ module detected!");
5065 kprintf(" Reload driver with supported module.\n");
5066 adapter->sfp_probe = FALSE;
5069 device_printf(dev,"SFP+ module detected!\n");
5070 /* We now have supported optics */
5071 adapter->sfp_probe = FALSE;
5072 /* Set the optics type so system reports correctly */
5073 ixgbe_setup_optics(adapter);
5081 ** Tasklet handler for MSIX Link interrupts
5082 ** - do outside interrupt since it might sleep
5085 ixgbe_handle_link(void *context, int pending)
5087 struct adapter *adapter = context;
5089 ixgbe_check_link(&adapter->hw,
5090 &adapter->link_speed, &adapter->link_up, 0);
5091 ixgbe_update_link_status(adapter);
5095 ** Tasklet for handling SFP module interrupts
5098 ixgbe_handle_mod(void *context, int pending)
5100 struct adapter *adapter = context;
5101 struct ixgbe_hw *hw = &adapter->hw;
5102 device_t dev = adapter->dev;
5105 err = hw->phy.ops.identify_sfp(hw);
5106 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5108 "Unsupported SFP+ module type was detected.\n");
5111 err = hw->mac.ops.setup_sfp(hw);
5112 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5114 "Setup failure - unsupported SFP+ module type.\n");
5117 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5123 ** Tasklet for handling MSF (multispeed fiber) interrupts
5126 ixgbe_handle_msf(void *context, int pending)
5128 struct adapter *adapter = context;
5129 struct ixgbe_hw *hw = &adapter->hw;
5133 autoneg = hw->phy.autoneg_advertised;
5134 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5135 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5136 if (hw->mac.ops.setup_link)
5137 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5143 ** Tasklet for reinitializing the Flow Director filter table
5146 ixgbe_reinit_fdir(void *context, int pending)
5148 struct adapter *adapter = context;
5149 struct ifnet *ifp = adapter->ifp;
5151 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5153 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5154 adapter->fdir_reinit = 0;
5155 /* re-enable flow director interrupts */
5156 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5157 /* Restart the interface */
5158 ifp->if_drv_flags |= IFF_DRV_RUNNING;
5163 /**********************************************************************
5165 * Update the board statistics counters.
5167 **********************************************************************/
5169 ixgbe_update_stats_counters(struct adapter *adapter)
5171 struct ifnet *ifp = adapter->ifp;
5172 struct ixgbe_hw *hw = &adapter->hw;
5173 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5174 u64 total_missed_rx = 0;
5176 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5177 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5178 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5179 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5181 for (int i = 0; i < 8; i++) {
5183 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5184 /* missed_rx tallies misses for the gprc workaround */
5186 /* global total per queue */
5187 adapter->stats.mpc[i] += mp;
5188 /* Running comprehensive total for stats display */
5189 total_missed_rx += adapter->stats.mpc[i];
5190 if (hw->mac.type == ixgbe_mac_82598EB)
5191 adapter->stats.rnbc[i] +=
5192 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5193 adapter->stats.pxontxc[i] +=
5194 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5195 adapter->stats.pxonrxc[i] +=
5196 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5197 adapter->stats.pxofftxc[i] +=
5198 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5199 adapter->stats.pxoffrxc[i] +=
5200 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5201 adapter->stats.pxon2offc[i] +=
5202 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5204 for (int i = 0; i < 16; i++) {
5205 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5206 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5207 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5208 adapter->stats.qbrc[i] +=
5209 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5210 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5211 adapter->stats.qbtc[i] +=
5212 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5213 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5215 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5216 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5217 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5219 /* Hardware workaround, gprc counts missed packets */
5220 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5221 adapter->stats.gprc -= missed_rx;
5223 if (hw->mac.type != ixgbe_mac_82598EB) {
5224 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5225 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5226 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5227 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5228 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5229 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5230 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5231 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5233 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5234 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5235 /* 82598 only has a counter in the high register */
5236 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5237 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5238 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5242 * Workaround: mprc hardware is incorrectly counting
5243 * broadcasts, so for now we subtract those.
5245 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5246 adapter->stats.bprc += bprc;
5247 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5248 if (hw->mac.type == ixgbe_mac_82598EB)
5249 adapter->stats.mprc -= bprc;
5251 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5252 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5253 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5254 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5255 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5256 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5258 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5259 adapter->stats.lxontxc += lxon;
5260 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5261 adapter->stats.lxofftxc += lxoff;
5262 total = lxon + lxoff;
5264 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5265 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5266 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5267 adapter->stats.gptc -= total;
5268 adapter->stats.mptc -= total;
5269 adapter->stats.ptc64 -= total;
5270 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5272 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5273 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5274 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5275 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5276 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5277 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5278 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5279 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5280 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5281 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5282 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5283 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5284 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5285 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5286 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5287 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5288 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5289 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5290 /* Only read FCOE on 82599 */
5291 if (hw->mac.type != ixgbe_mac_82598EB) {
5292 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5293 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5294 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5295 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5296 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5299 /* Fill out the OS statistics structure */
5300 ifp->if_ipackets = adapter->stats.gprc;
5301 ifp->if_opackets = adapter->stats.gptc;
5302 ifp->if_ibytes = adapter->stats.gorc;
5303 ifp->if_obytes = adapter->stats.gotc;
5304 ifp->if_imcasts = adapter->stats.mprc;
5305 ifp->if_collisions = 0;
5308 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5309 adapter->stats.rlec;
5312 /** ixgbe_sysctl_tdh_handler - Handler function
5313 * Retrieves the TDH value from the hardware
5316 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5320 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5323 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5324 error = sysctl_handle_int(oidp, &val, 0, req);
5325 if (error || !req->newptr)
5330 /** ixgbe_sysctl_tdt_handler - Handler function
5331 * Retrieves the TDT value from the hardware
5334 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5338 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5341 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5342 error = sysctl_handle_int(oidp, &val, 0, req);
5343 if (error || !req->newptr)
5348 /** ixgbe_sysctl_rdh_handler - Handler function
5349 * Retrieves the RDH value from the hardware
5352 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5356 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5359 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5360 error = sysctl_handle_int(oidp, &val, 0, req);
5361 if (error || !req->newptr)
5366 /** ixgbe_sysctl_rdt_handler - Handler function
5367 * Retrieves the RDT value from the hardware
5370 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5374 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5377 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5378 error = sysctl_handle_int(oidp, &val, 0, req);
5379 if (error || !req->newptr)
5385 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5388 struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5389 unsigned int reg, usec, rate;
5391 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5392 usec = ((reg & 0x0FF8) >> 3);
5394 rate = 500000 / usec;
5397 error = sysctl_handle_int(oidp, &rate, 0, req);
5398 if (error || !req->newptr)
5400 reg &= ~0xfff; /* default, no limitation */
5401 ixgbe_max_interrupt_rate = 0;
5402 if (rate > 0 && rate < 500000) {
5405 ixgbe_max_interrupt_rate = rate;
5406 reg |= ((4000000/rate) & 0xff8 );
5408 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5413 * Add sysctl variables, one per statistic, to the system.
5416 ixgbe_add_hw_stats(struct adapter *adapter)
5418 struct tx_ring *txr = adapter->tx_rings;
5419 struct rx_ring *rxr = adapter->rx_rings;
5421 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5422 struct sysctl_oid *tree = adapter->sysctl_tree;
5423 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5424 struct ixgbe_hw_stats *stats = &adapter->stats;
5426 struct sysctl_oid *stat_node, *queue_node;
5427 struct sysctl_oid_list *stat_list, *queue_list;
5429 #define QUEUE_NAME_LEN 32
5430 char namebuf[QUEUE_NAME_LEN];
5432 /* Driver Statistics */
5433 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5434 CTLFLAG_RD, &adapter->dropped_pkts,
5435 "Driver dropped packets");
5436 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5437 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5438 "m_defrag() failed");
5439 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5440 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5441 "Driver tx dma failure in xmit");
5442 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5443 CTLFLAG_RD, &adapter->watchdog_events,
5444 "Watchdog timeouts");
5446 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5447 CTLFLAG_RD, &adapter->tso_tx,
5450 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5451 CTLFLAG_RD, &adapter->link_irq,
5452 "Link MSIX IRQ Handled");
5454 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5455 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5456 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5457 CTLFLAG_RD, NULL, "Queue Name");
5458 queue_list = SYSCTL_CHILDREN(queue_node);
5460 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5461 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5462 sizeof(&adapter->queues[i]),
5463 ixgbe_sysctl_interrupt_rate_handler, "IU",
5465 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5466 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5467 "irqs on this queue");
5468 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5469 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5470 ixgbe_sysctl_tdh_handler, "IU",
5471 "Transmit Descriptor Head");
5472 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5473 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5474 ixgbe_sysctl_tdt_handler, "IU",
5475 "Transmit Descriptor Tail");
5476 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5477 CTLFLAG_RD, &txr->no_desc_avail, 0,
5478 "Queue No Descriptor Available");
5479 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5480 CTLFLAG_RD, &txr->total_packets, 0,
5481 "Queue Packets Transmitted");
5484 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5485 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5486 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5487 CTLFLAG_RD, NULL, "Queue Name");
5488 queue_list = SYSCTL_CHILDREN(queue_node);
5491 struct lro_ctrl *lro = &rxr->lro;
5494 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5495 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5496 CTLFLAG_RD, NULL, "Queue Name");
5497 queue_list = SYSCTL_CHILDREN(queue_node);
5499 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5500 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5501 ixgbe_sysctl_rdh_handler, "IU",
5502 "Receive Descriptor Head");
5503 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5504 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5505 ixgbe_sysctl_rdt_handler, "IU",
5506 "Receive Descriptor Tail");
5507 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5508 CTLFLAG_RD, &rxr->rx_packets, 0,
5509 "Queue Packets Received");
5510 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5511 CTLFLAG_RD, &rxr->rx_bytes, 0,
5512 "Queue Bytes Received");
5514 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5515 CTLFLAG_RD, &lro->lro_queued, 0,
5517 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5518 CTLFLAG_RD, &lro->lro_flushed, 0,
5523 /* MAC stats get the own sub node */
5525 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5526 CTLFLAG_RD, NULL, "MAC Statistics");
5527 stat_list = SYSCTL_CHILDREN(stat_node);
5529 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5530 CTLFLAG_RD, &stats->crcerrs, 0,
5532 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5533 CTLFLAG_RD, &stats->illerrc, 0,
5534 "Illegal Byte Errors");
5535 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5536 CTLFLAG_RD, &stats->errbc, 0,
5538 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5539 CTLFLAG_RD, &stats->mspdc, 0,
5540 "MAC Short Packets Discarded");
5541 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5542 CTLFLAG_RD, &stats->mlfc, 0,
5543 "MAC Local Faults");
5544 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5545 CTLFLAG_RD, &stats->mrfc, 0,
5546 "MAC Remote Faults");
5547 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5548 CTLFLAG_RD, &stats->rlec, 0,
5549 "Receive Length Errors");
5550 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5551 CTLFLAG_RD, &stats->lxontxc, 0,
5552 "Link XON Transmitted");
5553 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5554 CTLFLAG_RD, &stats->lxonrxc, 0,
5555 "Link XON Received");
5556 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5557 CTLFLAG_RD, &stats->lxofftxc, 0,
5558 "Link XOFF Transmitted");
5559 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5560 CTLFLAG_RD, &stats->lxoffrxc, 0,
5561 "Link XOFF Received");
5563 /* Packet Reception Stats */
5564 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5565 CTLFLAG_RD, &stats->tor, 0,
5566 "Total Octets Received");
5567 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5568 CTLFLAG_RD, &stats->gorc, 0,
5569 "Good Octets Received");
5570 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5571 CTLFLAG_RD, &stats->tpr, 0,
5572 "Total Packets Received");
5573 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5574 CTLFLAG_RD, &stats->gprc, 0,
5575 "Good Packets Received");
5576 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5577 CTLFLAG_RD, &stats->mprc, 0,
5578 "Multicast Packets Received");
5579 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5580 CTLFLAG_RD, &stats->bprc, 0,
5581 "Broadcast Packets Received");
5582 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5583 CTLFLAG_RD, &stats->prc64, 0,
5584 "64 byte frames received ");
5585 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5586 CTLFLAG_RD, &stats->prc127, 0,
5587 "65-127 byte frames received");
5588 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5589 CTLFLAG_RD, &stats->prc255, 0,
5590 "128-255 byte frames received");
5591 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5592 CTLFLAG_RD, &stats->prc511, 0,
5593 "256-511 byte frames received");
5594 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5595 CTLFLAG_RD, &stats->prc1023, 0,
5596 "512-1023 byte frames received");
5597 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5598 CTLFLAG_RD, &stats->prc1522, 0,
5599 "1023-1522 byte frames received");
5600 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5601 CTLFLAG_RD, &stats->ruc, 0,
5602 "Receive Undersized");
5603 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5604 CTLFLAG_RD, &stats->rfc, 0,
5605 "Fragmented Packets Received ");
5606 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5607 CTLFLAG_RD, &stats->roc, 0,
5608 "Oversized Packets Received");
5609 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5610 CTLFLAG_RD, &stats->rjc, 0,
5612 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5613 CTLFLAG_RD, &stats->mngprc, 0,
5614 "Management Packets Received");
5615 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5616 CTLFLAG_RD, &stats->mngptc, 0,
5617 "Management Packets Dropped");
5618 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5619 CTLFLAG_RD, &stats->xec, 0,
5622 /* Packet Transmission Stats */
5623 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5624 CTLFLAG_RD, &stats->gotc, 0,
5625 "Good Octets Transmitted");
5626 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5627 CTLFLAG_RD, &stats->tpt, 0,
5628 "Total Packets Transmitted");
5629 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5630 CTLFLAG_RD, &stats->gptc, 0,
5631 "Good Packets Transmitted");
5632 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5633 CTLFLAG_RD, &stats->bptc, 0,
5634 "Broadcast Packets Transmitted");
5635 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5636 CTLFLAG_RD, &stats->mptc, 0,
5637 "Multicast Packets Transmitted");
5638 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5639 CTLFLAG_RD, &stats->mngptc, 0,
5640 "Management Packets Transmitted");
5641 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5642 CTLFLAG_RD, &stats->ptc64, 0,
5643 "64 byte frames transmitted ");
5644 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5645 CTLFLAG_RD, &stats->ptc127, 0,
5646 "65-127 byte frames transmitted");
5647 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5648 CTLFLAG_RD, &stats->ptc255, 0,
5649 "128-255 byte frames transmitted");
5650 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5651 CTLFLAG_RD, &stats->ptc511, 0,
5652 "256-511 byte frames transmitted");
5653 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5654 CTLFLAG_RD, &stats->ptc1023, 0,
5655 "512-1023 byte frames transmitted");
5656 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5657 CTLFLAG_RD, &stats->ptc1522, 0,
5658 "1024-1522 byte frames transmitted");
5661 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5662 CTLFLAG_RD, &stats->fccrc, 0,
5664 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5665 CTLFLAG_RD, &stats->fclast, 0,
5667 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5668 CTLFLAG_RD, &stats->fcoerpdc, 0,
5669 "FCoE Packets Dropped");
5670 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5671 CTLFLAG_RD, &stats->fcoeprc, 0,
5672 "FCoE Packets Received");
5673 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5674 CTLFLAG_RD, &stats->fcoeptc, 0,
5675 "FCoE Packets Transmitted");
5676 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5677 CTLFLAG_RD, &stats->fcoedwrc, 0,
5678 "FCoE DWords Received");
5679 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5680 CTLFLAG_RD, &stats->fcoedwtc, 0,
5681 "FCoE DWords Transmitted");
5685 ** Set flow control using sysctl:
5686 ** Flow control values:
5693 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5696 struct adapter *adapter = (struct adapter *) arg1;
5699 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5700 if ((error) || (req->newptr == NULL))
5703 /* Don't bother if it's not changed */
5704 if (adapter->fc == last)
5707 switch (adapter->fc) {
5708 case ixgbe_fc_rx_pause:
5709 case ixgbe_fc_tx_pause:
5711 adapter->hw.fc.requested_mode = adapter->fc;
5715 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5717 /* Don't autoneg if forcing a value */
5718 adapter->hw.fc.disable_fc_autoneg = TRUE;
5719 ixgbe_fc_enable(&adapter->hw);
5724 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5725 const char *description, int *limit, int value)
5728 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5729 SYSCTL_CHILDREN(adapter->sysctl_tree),
5730 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5734 ** Control link advertise speed:
5735 ** 1 - advertise only 1G
5736 ** 2 - advertise 100Mb
5737 ** 3 - advertise normal
5740 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5743 struct adapter *adapter;
5745 struct ixgbe_hw *hw;
5746 ixgbe_link_speed speed, last;
5748 adapter = (struct adapter *) arg1;
5751 last = adapter->advertise;
5753 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5754 if ((error) || (adapter->advertise == -1))
5757 if (adapter->advertise == last) /* no change */
5760 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5761 (hw->phy.multispeed_fiber)))
5764 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5765 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5769 if (adapter->advertise == 1)
5770 speed = IXGBE_LINK_SPEED_1GB_FULL;
5771 else if (adapter->advertise == 2)
5772 speed = IXGBE_LINK_SPEED_100_FULL;
5773 else if (adapter->advertise == 3)
5774 speed = IXGBE_LINK_SPEED_1GB_FULL |
5775 IXGBE_LINK_SPEED_10GB_FULL;
5776 else /* bogus value */
5779 hw->mac.autotry_restart = TRUE;
5780 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5786 ** Thermal Shutdown Trigger
5787 ** - cause a Thermal Overtemp IRQ
5790 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5792 int error, fire = 0;
5793 struct adapter *adapter = (struct adapter *) arg1;
5794 struct ixgbe_hw *hw = &adapter->hw;
5797 if (hw->mac.type != ixgbe_mac_X540)
5800 error = sysctl_handle_int(oidp, &fire, 0, req);
5801 if ((error) || (req->newptr == NULL))
5805 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5806 reg |= IXGBE_EICR_TS;
5807 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);