1 /******************************************************************************
3 Copyright (c) 2001-2012, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.69 2012/06/07 22:57:26 emax Exp $*/
36 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.5";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
84 /* required last entry */
88 /*********************************************************************
89 * Table of branding strings
90 *********************************************************************/
92 static char *ixgbe_strings[] = {
93 "Intel(R) PRO/10GbE PCI-Express Network Driver"
96 /*********************************************************************
98 *********************************************************************/
99 static int ixgbe_probe(device_t);
100 static int ixgbe_attach(device_t);
101 static int ixgbe_detach(device_t);
102 static int ixgbe_shutdown(device_t);
103 static void ixgbe_start(struct ifnet *);
104 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
105 #if 0 /* __FreeBSD_version >= 800000 */
106 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
107 static int ixgbe_mq_start_locked(struct ifnet *,
108 struct tx_ring *, struct mbuf *);
109 static void ixgbe_qflush(struct ifnet *);
111 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
112 static void ixgbe_init(void *);
113 static void ixgbe_init_locked(struct adapter *);
114 static void ixgbe_stop(void *);
115 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
116 static int ixgbe_media_change(struct ifnet *);
117 static void ixgbe_identify_hardware(struct adapter *);
118 static int ixgbe_allocate_pci_resources(struct adapter *);
119 static int ixgbe_allocate_msix(struct adapter *);
120 static int ixgbe_allocate_legacy(struct adapter *);
121 static int ixgbe_allocate_queues(struct adapter *);
122 #if 0 /* HAVE_MSIX */
123 static int ixgbe_setup_msix(struct adapter *);
125 static void ixgbe_free_pci_resources(struct adapter *);
126 static void ixgbe_local_timer(void *);
127 static int ixgbe_setup_interface(device_t, struct adapter *);
128 static void ixgbe_config_link(struct adapter *);
130 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
131 static int ixgbe_setup_transmit_structures(struct adapter *);
132 static void ixgbe_setup_transmit_ring(struct tx_ring *);
133 static void ixgbe_initialize_transmit_units(struct adapter *);
134 static void ixgbe_free_transmit_structures(struct adapter *);
135 static void ixgbe_free_transmit_buffers(struct tx_ring *);
137 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
138 static int ixgbe_setup_receive_structures(struct adapter *);
139 static int ixgbe_setup_receive_ring(struct rx_ring *);
140 static void ixgbe_initialize_receive_units(struct adapter *);
141 static void ixgbe_free_receive_structures(struct adapter *);
142 static void ixgbe_free_receive_buffers(struct rx_ring *);
144 static void ixgbe_setup_hw_rsc(struct rx_ring *);
147 static void ixgbe_enable_intr(struct adapter *);
148 static void ixgbe_disable_intr(struct adapter *);
149 static void ixgbe_update_stats_counters(struct adapter *);
150 static bool ixgbe_txeof(struct tx_ring *);
151 static bool ixgbe_rxeof(struct ix_queue *, int);
152 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void ixgbe_set_promisc(struct adapter *);
154 static void ixgbe_set_multi(struct adapter *);
155 static void ixgbe_update_link_status(struct adapter *);
156 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
162 struct ixgbe_dma_alloc *, int);
163 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
165 const char *, int *, int);
166 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
168 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
170 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
171 static void ixgbe_configure_ivars(struct adapter *);
172 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
174 static void ixgbe_setup_vlan_hw_support(struct adapter *);
175 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
176 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
178 static void ixgbe_add_hw_stats(struct adapter *adapter);
180 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
181 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
184 /* Support for pluggable optic modules */
185 static bool ixgbe_sfp_probe(struct adapter *);
186 static void ixgbe_setup_optics(struct adapter *);
188 /* Legacy (single vector interrupt handler */
189 static void ixgbe_legacy_irq(void *);
191 /* The MSI/X Interrupt handlers */
192 static void ixgbe_msix_que(void *);
193 static void ixgbe_msix_link(void *);
195 /* Deferred interrupt tasklets */
196 static void ixgbe_handle_que(void *, int);
197 static void ixgbe_handle_link(void *, int);
198 static void ixgbe_handle_msf(void *, int);
199 static void ixgbe_handle_mod(void *, int);
202 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
203 static void ixgbe_reinit_fdir(void *, int);
206 /*********************************************************************
207 * FreeBSD Device Interface Entry Points
208 *********************************************************************/
210 static device_method_t ixgbe_methods[] = {
211 /* Device interface */
212 DEVMETHOD(device_probe, ixgbe_probe),
213 DEVMETHOD(device_attach, ixgbe_attach),
214 DEVMETHOD(device_detach, ixgbe_detach),
215 DEVMETHOD(device_shutdown, ixgbe_shutdown),
219 static driver_t ixgbe_driver = {
220 "ix", ixgbe_methods, sizeof(struct adapter),
223 devclass_t ixgbe_devclass;
224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230 ** TUNEABLE PARAMETERS:
234 ** AIM: Adaptive Interrupt Moderation
235 ** which means that the interrupt rate
236 ** is varied over time based on the
237 ** traffic for that interrupt vector
239 static int ixgbe_enable_aim = TRUE;
240 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
242 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
243 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
245 /* How many packets rxeof tries to clean at a time */
246 static int ixgbe_rx_process_limit = 128;
247 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
249 /* Flow control setting, default to full */
250 static int ixgbe_flow_control = ixgbe_fc_full;
251 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
254 ** Smart speed setting, default to on
255 ** this only works as a compile option
256 ** right now as its during attach, set
257 ** this to 'ixgbe_smart_speed_off' to
260 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
263 * MSIX should be the default for best performance,
264 * but this allows it to be forced off for testing.
266 static int ixgbe_enable_msix = 1;
267 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
270 * Header split: this causes the hardware to DMA
271 * the header into a separate mbuf from the payload,
272 * it can be a performance win in some workloads, but
273 * in others it actually hurts, its off by default.
275 static int ixgbe_header_split = FALSE;
276 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
279 * Number of Queues, can be set to 0,
280 * it then autoconfigures based on the
281 * number of cpus with a max of 8. This
282 * can be overriden manually here.
284 static int ixgbe_num_queues = 0;
285 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
288 ** Number of TX descriptors per ring,
289 ** setting higher than RX as this seems
290 ** the better performing choice.
292 static int ixgbe_txd = PERFORM_TXD;
293 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
295 /* Number of RX descriptors per ring */
296 static int ixgbe_rxd = PERFORM_RXD;
297 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
299 /* Keep running tab on them for sanity check */
300 static int ixgbe_total_ports;
304 ** For Flow Director: this is the
305 ** number of TX packets we sample
306 ** for the filter pool, this means
307 ** every 20th packet will be probed.
309 ** This feature can be disabled by
310 ** setting this to 0.
312 static int atr_sample_rate = 20;
314 ** Flow Director actually 'steals'
315 ** part of the packet buffer as its
316 ** filter pool, this variable controls
318 ** 0 = 64K, 1 = 128K, 2 = 256K
320 static int fdir_pballoc = 1;
325 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
326 * be a reference on how to implement netmap support in a driver.
327 * Additional comments are in ixgbe_netmap.h .
329 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
330 * that extend the standard driver.
332 #include <dev/netmap/ixgbe_netmap.h>
333 #endif /* DEV_NETMAP */
335 /*********************************************************************
336 * Device identification routine
338 * ixgbe_probe determines if the driver should be loaded on
339 * adapter based on PCI vendor/device id of the adapter.
341 * return BUS_PROBE_DEFAULT on success, positive on failure
342 *********************************************************************/
345 ixgbe_probe(device_t dev)
347 ixgbe_vendor_info_t *ent;
349 u16 pci_vendor_id = 0;
350 u16 pci_device_id = 0;
351 u16 pci_subvendor_id = 0;
352 u16 pci_subdevice_id = 0;
353 char adapter_name[256];
355 INIT_DEBUGOUT("ixgbe_probe: begin");
357 pci_vendor_id = pci_get_vendor(dev);
358 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
361 pci_device_id = pci_get_device(dev);
362 pci_subvendor_id = pci_get_subvendor(dev);
363 pci_subdevice_id = pci_get_subdevice(dev);
365 ent = ixgbe_vendor_info_array;
366 while (ent->vendor_id != 0) {
367 if ((pci_vendor_id == ent->vendor_id) &&
368 (pci_device_id == ent->device_id) &&
370 ((pci_subvendor_id == ent->subvendor_id) ||
371 (ent->subvendor_id == 0)) &&
373 ((pci_subdevice_id == ent->subdevice_id) ||
374 (ent->subdevice_id == 0))) {
375 ksprintf(adapter_name, "%s, Version - %s",
376 ixgbe_strings[ent->index],
377 ixgbe_driver_version);
378 device_set_desc_copy(dev, adapter_name);
380 return (BUS_PROBE_DEFAULT);
387 /*********************************************************************
388 * Device initialization routine
390 * The attach entry point is called when the driver is being loaded.
391 * This routine identifies the type of hardware, allocates all resources
392 * and initializes the hardware.
394 * return 0 on success, positive on failure
395 *********************************************************************/
398 ixgbe_attach(device_t dev)
400 struct adapter *adapter;
406 INIT_DEBUGOUT("ixgbe_attach: begin");
408 if (resource_disabled("ixgbe", device_get_unit(dev))) {
409 device_printf(dev, "Disabled by device hint\n");
413 /* Allocate, clear, and link in our adapter structure */
414 adapter = device_get_softc(dev);
415 adapter->dev = adapter->osdep.dev = dev;
419 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
420 spin_init(&adapter->mcast_spin);
424 sysctl_ctx_init(&adapter->sysctl_ctx);
425 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
426 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
427 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
428 if (adapter->sysctl_tree == NULL) {
429 device_printf(adapter->dev, "can't add sysctl node\n");
432 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
433 SYSCTL_CHILDREN(adapter->sysctl_tree),
434 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
435 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
437 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
438 SYSCTL_CHILDREN(adapter->sysctl_tree),
439 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
440 &ixgbe_enable_aim, 1, "Interrupt Moderation");
443 ** Allow a kind of speed control by forcing the autoneg
444 ** advertised speed list to only a certain value, this
445 ** supports 1G on 82599 devices, and 100Mb on x540.
447 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
448 SYSCTL_CHILDREN(adapter->sysctl_tree),
449 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
450 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
452 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
453 SYSCTL_CHILDREN(adapter->sysctl_tree),
454 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
455 0, ixgbe_set_thermal_test, "I", "Thermal Test");
457 /* Set up the timer callout */
458 /* XXX: shouldn't this be a spin lock ? */
459 lockinit(&adapter->core_lock, "ixgbe core lock", 0, LK_CANRECURSE);
460 callout_init(&adapter->timer);
462 /* Determine hardware revision */
463 ixgbe_identify_hardware(adapter);
465 /* Do base PCI setup - map BAR0 */
466 if (ixgbe_allocate_pci_resources(adapter)) {
467 device_printf(dev, "Allocation of PCI resources failed\n");
472 /* Do descriptor calc and sanity checks */
473 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
474 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
475 device_printf(dev, "TXD config issue, using default!\n");
476 adapter->num_tx_desc = DEFAULT_TXD;
478 adapter->num_tx_desc = ixgbe_txd;
481 ** With many RX rings it is easy to exceed the
482 ** system mbuf allocation. Tuning nmbclusters
483 ** can alleviate this.
485 if (nmbclusters > 0 ) {
487 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
488 if (s > nmbclusters) {
489 device_printf(dev, "RX Descriptors exceed "
490 "system mbuf max, using default instead!\n");
491 ixgbe_rxd = DEFAULT_RXD;
495 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
496 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
497 device_printf(dev, "RXD config issue, using default!\n");
498 adapter->num_rx_desc = DEFAULT_RXD;
500 adapter->num_rx_desc = ixgbe_rxd;
502 /* Allocate our TX/RX Queues */
503 if (ixgbe_allocate_queues(adapter)) {
508 /* Allocate multicast array memory. */
509 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
510 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
511 if (adapter->mta == NULL) {
512 device_printf(dev, "Can not allocate multicast setup array\n");
517 /* Initialize the shared code */
518 error = ixgbe_init_shared_code(hw);
519 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
521 ** No optics in this port, set up
522 ** so the timer routine will probe
523 ** for later insertion.
525 adapter->sfp_probe = TRUE;
527 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
528 device_printf(dev,"Unsupported SFP+ module detected!\n");
532 device_printf(dev,"Unable to initialize the shared code\n");
537 /* Make sure we have a good EEPROM before we read from it */
538 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
539 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
544 /* Get Hardware Flow Control setting */
545 hw->fc.requested_mode = ixgbe_fc_full;
546 adapter->fc = hw->fc.requested_mode;
547 hw->fc.pause_time = IXGBE_FC_PAUSE;
548 hw->fc.low_water = IXGBE_FC_LO;
549 hw->fc.high_water[0] = IXGBE_FC_HI;
550 hw->fc.send_xon = TRUE;
552 error = ixgbe_init_hw(hw);
553 if (error == IXGBE_ERR_EEPROM_VERSION) {
554 device_printf(dev, "This device is a pre-production adapter/"
555 "LOM. Please be aware there may be issues associated "
556 "with your hardware.\n If you are experiencing problems "
557 "please contact your Intel or hardware representative "
558 "who provided you with this hardware.\n");
559 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED)
560 device_printf(dev,"Unsupported SFP+ Module\n");
564 device_printf(dev,"Hardware Initialization Failure\n");
568 /* Detect and set physical type */
569 ixgbe_setup_optics(adapter);
571 if ((adapter->msix > 1) && (ixgbe_enable_msix))
572 error = ixgbe_allocate_msix(adapter);
574 error = ixgbe_allocate_legacy(adapter);
578 /* Setup OS specific network interface */
579 if (ixgbe_setup_interface(dev, adapter) != 0)
582 /* Sysctl for limiting the amount of work done in the taskqueue */
583 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
584 "max number of rx packets to process", &adapter->rx_process_limit,
585 ixgbe_rx_process_limit);
587 /* Initialize statistics */
588 ixgbe_update_stats_counters(adapter);
590 /* Register for VLAN events */
591 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
592 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
593 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
594 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
596 /* Print PCIE bus type/speed/width info */
597 ixgbe_get_bus_info(hw);
598 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
599 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
600 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
601 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
602 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
603 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
606 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
607 (hw->bus.speed == ixgbe_bus_speed_2500)) {
608 device_printf(dev, "PCI-Express bandwidth available"
609 " for this card\n is not sufficient for"
610 " optimal performance.\n");
611 device_printf(dev, "For optimal performance a x8 "
612 "PCIE, or x4 PCIE 2 slot is required.\n");
615 /* let hardware know driver is loaded */
616 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
617 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
618 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
620 ixgbe_add_hw_stats(adapter);
623 ixgbe_netmap_attach(adapter);
624 #endif /* DEV_NETMAP */
625 INIT_DEBUGOUT("ixgbe_attach: end");
628 ixgbe_free_transmit_structures(adapter);
629 ixgbe_free_receive_structures(adapter);
631 if (adapter->ifp != NULL)
632 if_free(adapter->ifp);
633 ixgbe_free_pci_resources(adapter);
634 kfree(adapter->mta, M_DEVBUF);
639 /*********************************************************************
640 * Device removal routine
642 * The detach entry point is called when the driver is being removed.
643 * This routine stops the adapter and deallocates all the resources
644 * that were allocated for driver operation.
646 * return 0 on success, positive on failure
647 *********************************************************************/
650 ixgbe_detach(device_t dev)
652 struct adapter *adapter = device_get_softc(dev);
653 struct ix_queue *que = adapter->queues;
656 INIT_DEBUGOUT("ixgbe_detach: begin");
659 /* Make sure VLANS are not using driver */
660 if (adapter->ifp->if_vlantrunk != NULL) {
661 device_printf(dev,"Vlan in use, detach first\n");
666 IXGBE_CORE_LOCK(adapter);
668 IXGBE_CORE_UNLOCK(adapter);
670 for (int i = 0; i < adapter->num_queues; i++, que++) {
672 taskqueue_drain(que->tq, &que->que_task);
673 taskqueue_free(que->tq);
677 /* Drain the Link queue */
679 taskqueue_drain(adapter->tq, &adapter->link_task);
680 taskqueue_drain(adapter->tq, &adapter->mod_task);
681 taskqueue_drain(adapter->tq, &adapter->msf_task);
683 taskqueue_drain(adapter->tq, &adapter->fdir_task);
685 taskqueue_free(adapter->tq);
688 /* let hardware know driver is unloading */
689 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
690 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
691 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
693 /* Unregister VLAN events */
695 if (adapter->vlan_attach != NULL)
696 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
697 if (adapter->vlan_detach != NULL)
698 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
701 ether_ifdetach(adapter->ifp);
702 callout_stop(&adapter->timer);
703 lockuninit(&adapter->core_lock);
705 netmap_detach(adapter->ifp);
706 #endif /* DEV_NETMAP */
707 ixgbe_free_pci_resources(adapter);
708 bus_generic_detach(dev);
709 if_free(adapter->ifp);
711 ixgbe_free_transmit_structures(adapter);
712 ixgbe_free_receive_structures(adapter);
713 kfree(adapter->mta, M_DEVBUF);
714 sysctl_ctx_free(&adapter->sysctl_ctx);
716 spin_uninit(&adapter->mcast_spin);
717 IXGBE_CORE_LOCK_DESTROY(adapter);
721 /*********************************************************************
723 * Shutdown entry point
725 **********************************************************************/
728 ixgbe_shutdown(device_t dev)
730 struct adapter *adapter = device_get_softc(dev);
731 IXGBE_CORE_LOCK(adapter);
733 IXGBE_CORE_UNLOCK(adapter);
738 /*********************************************************************
739 * Transmit entry point
741 * ixgbe_start is called by the stack to initiate a transmit.
742 * The driver will remain in this routine as long as there are
743 * packets to transmit and transmit resources are available.
744 * In case resources are not available stack is notified and
745 * the packet is requeued.
746 **********************************************************************/
749 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
752 struct adapter *adapter = txr->adapter;
754 IXGBE_TX_LOCK_ASSERT(txr);
756 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
758 if (!adapter->link_active)
761 while (!ifq_is_empty(&ifp->if_snd)) {
762 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
763 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
767 m_head = ifq_dequeue(&ifp->if_snd, NULL);
771 if (ixgbe_xmit(txr, &m_head)) {
772 #if 0 /* XXX: prepend to an ALTQ queue ? */
774 IF_PREPEND(&ifp->if_snd, m_head);
776 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
777 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
780 /* Send a copy of the frame to the BPF listener */
781 ETHER_BPF_MTAP(ifp, m_head);
783 /* Set watchdog on */
784 txr->watchdog_time = ticks;
785 txr->queue_status = IXGBE_QUEUE_WORKING;
792 * Legacy TX start - called by the stack, this
793 * always uses the first tx ring, and should
794 * not be used with multiqueue tx enabled.
797 ixgbe_start(struct ifnet *ifp)
799 struct adapter *adapter = ifp->if_softc;
800 struct tx_ring *txr = adapter->tx_rings;
802 if (ifp->if_flags & IFF_RUNNING) {
804 ixgbe_start_locked(txr, ifp);
805 IXGBE_TX_UNLOCK(txr);
810 #if 0 /* __FreeBSD_version >= 800000 */
812 ** Multiqueue Transmit driver
816 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
818 struct adapter *adapter = ifp->if_softc;
819 struct ix_queue *que;
823 /* Which queue to use */
824 if ((m->m_flags & M_FLOWID) != 0)
825 i = m->m_pkthdr.flowid % adapter->num_queues;
827 i = curcpu % adapter->num_queues;
829 txr = &adapter->tx_rings[i];
830 que = &adapter->queues[i];
832 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
833 IXGBE_TX_TRYLOCK(txr)) {
834 err = ixgbe_mq_start_locked(ifp, txr, m);
835 IXGBE_TX_UNLOCK(txr);
837 err = drbr_enqueue(ifp, txr->br, m);
838 taskqueue_enqueue(que->tq, &que->que_task);
845 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
847 struct adapter *adapter = txr->adapter;
849 int enqueued, err = 0;
851 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
852 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
853 adapter->link_active == 0) {
855 err = drbr_enqueue(ifp, txr->br, m);
861 next = drbr_dequeue(ifp, txr->br);
862 } else if (drbr_needs_enqueue(ifp, txr->br)) {
863 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
865 next = drbr_dequeue(ifp, txr->br);
869 /* Process the queue */
870 while (next != NULL) {
871 if ((err = ixgbe_xmit(txr, &next)) != 0) {
873 err = drbr_enqueue(ifp, txr->br, next);
877 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
878 /* Send a copy of the frame to the BPF listener */
879 ETHER_BPF_MTAP(ifp, next);
880 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
882 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
884 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
885 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
888 next = drbr_dequeue(ifp, txr->br);
892 /* Set watchdog on */
893 txr->queue_status |= IXGBE_QUEUE_WORKING;
894 txr->watchdog_time = ticks;
897 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
904 ** Flush all ring buffers
907 ixgbe_qflush(struct ifnet *ifp)
909 struct adapter *adapter = ifp->if_softc;
910 struct tx_ring *txr = adapter->tx_rings;
913 for (int i = 0; i < adapter->num_queues; i++, txr++) {
915 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
917 IXGBE_TX_UNLOCK(txr);
921 #endif /* __FreeBSD_version >= 800000 */
923 /*********************************************************************
926 * ixgbe_ioctl is called when the user wants to configure the
929 * return 0 on success, positive on failure
930 **********************************************************************/
933 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
935 struct adapter *adapter = ifp->if_softc;
936 struct ifreq *ifr = (struct ifreq *) data;
937 #if defined(INET) || defined(INET6)
938 struct ifaddr *ifa = (struct ifaddr *)data;
939 bool avoid_reset = FALSE;
947 if (ifa->ifa_addr->sa_family == AF_INET)
951 if (ifa->ifa_addr->sa_family == AF_INET6)
954 #if defined(INET) || defined(INET6)
956 ** Calling init results in link renegotiation,
957 ** so we avoid doing it when possible.
960 ifp->if_flags |= IFF_UP;
961 if (!(ifp->if_flags & IFF_RUNNING))
963 if (!(ifp->if_flags & IFF_NOARP))
964 arp_ifinit(ifp, ifa);
966 error = ether_ioctl(ifp, command, data);
970 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
971 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
974 IXGBE_CORE_LOCK(adapter);
975 ifp->if_mtu = ifr->ifr_mtu;
976 adapter->max_frame_size =
977 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
978 ixgbe_init_locked(adapter);
979 IXGBE_CORE_UNLOCK(adapter);
983 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
984 IXGBE_CORE_LOCK(adapter);
985 if (ifp->if_flags & IFF_UP) {
986 if ((ifp->if_flags & IFF_RUNNING)) {
987 if ((ifp->if_flags ^ adapter->if_flags) &
988 (IFF_PROMISC | IFF_ALLMULTI)) {
989 ixgbe_set_promisc(adapter);
992 ixgbe_init_locked(adapter);
994 if (ifp->if_flags & IFF_RUNNING)
996 adapter->if_flags = ifp->if_flags;
997 IXGBE_CORE_UNLOCK(adapter);
1001 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1002 if (ifp->if_flags & IFF_RUNNING) {
1003 IXGBE_CORE_LOCK(adapter);
1004 ixgbe_disable_intr(adapter);
1005 ixgbe_set_multi(adapter);
1006 ixgbe_enable_intr(adapter);
1007 IXGBE_CORE_UNLOCK(adapter);
1012 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1013 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1017 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1018 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1019 if (mask & IFCAP_HWCSUM)
1020 ifp->if_capenable ^= IFCAP_HWCSUM;
1022 if (mask & IFCAP_TSO4)
1023 ifp->if_capenable ^= IFCAP_TSO4;
1024 if (mask & IFCAP_TSO6)
1025 ifp->if_capenable ^= IFCAP_TSO6;
1028 if (mask & IFCAP_LRO)
1029 ifp->if_capenable ^= IFCAP_LRO;
1031 if (mask & IFCAP_VLAN_HWTAGGING)
1032 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1033 if (mask & IFCAP_VLAN_HWFILTER)
1034 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1036 if (mask & IFCAP_VLAN_HWTSO)
1037 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1039 if (ifp->if_flags & IFF_RUNNING) {
1040 IXGBE_CORE_LOCK(adapter);
1041 ixgbe_init_locked(adapter);
1042 IXGBE_CORE_UNLOCK(adapter);
1045 VLAN_CAPABILITIES(ifp);
1051 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1052 error = ether_ioctl(ifp, command, data);
1059 /*********************************************************************
1062 * This routine is used in two ways. It is used by the stack as
1063 * init entry point in network interface structure. It is also used
1064 * by the driver as a hw/sw initialization routine to get to a
1067 * return 0 on success, positive on failure
1068 **********************************************************************/
1069 #define IXGBE_MHADD_MFS_SHIFT 16
1072 ixgbe_init_locked(struct adapter *adapter)
1074 struct ifnet *ifp = adapter->ifp;
1075 device_t dev = adapter->dev;
1076 struct ixgbe_hw *hw = &adapter->hw;
1077 u32 k, txdctl, mhadd, gpie;
1080 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1081 INIT_DEBUGOUT("ixgbe_init: begin");
1082 hw->adapter_stopped = FALSE;
1083 ixgbe_stop_adapter(hw);
1084 callout_stop(&adapter->timer);
1086 /* reprogram the RAR[0] in case user changed it. */
1087 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1089 /* Get the latest mac address, User can use a LAA */
1090 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1091 IXGBE_ETH_LENGTH_OF_ADDRESS);
1092 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1093 hw->addr_ctrl.rar_used_count = 1;
1095 /* Set the various hardware offload abilities */
1096 ifp->if_hwassist = 0;
1098 if (ifp->if_capenable & IFCAP_TSO)
1099 ifp->if_hwassist |= CSUM_TSO;
1101 if (ifp->if_capenable & IFCAP_TXCSUM) {
1102 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1104 if (hw->mac.type != ixgbe_mac_82598EB)
1105 ifp->if_hwassist |= CSUM_SCTP;
1109 /* Prepare transmit descriptors and buffers */
1110 if (ixgbe_setup_transmit_structures(adapter)) {
1111 device_printf(dev,"Could not setup transmit structures\n");
1112 ixgbe_stop(adapter);
1117 ixgbe_initialize_transmit_units(adapter);
1119 /* Setup Multicast table */
1120 ixgbe_set_multi(adapter);
1123 ** Determine the correct mbuf pool
1124 ** for doing jumbo/headersplit
1126 if (adapter->max_frame_size <= 2048)
1127 adapter->rx_mbuf_sz = MCLBYTES;
1128 else if (adapter->max_frame_size <= 4096)
1129 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1130 else if (adapter->max_frame_size <= 9216)
1131 adapter->rx_mbuf_sz = MJUM9BYTES;
1133 adapter->rx_mbuf_sz = MJUM16BYTES;
1135 /* Prepare receive descriptors and buffers */
1136 if (ixgbe_setup_receive_structures(adapter)) {
1137 device_printf(dev,"Could not setup receive structures\n");
1138 ixgbe_stop(adapter);
1142 /* Configure RX settings */
1143 ixgbe_initialize_receive_units(adapter);
1145 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1147 /* Enable Fan Failure Interrupt */
1148 gpie |= IXGBE_SDP1_GPIEN;
1150 /* Add for Module detection */
1151 if (hw->mac.type == ixgbe_mac_82599EB)
1152 gpie |= IXGBE_SDP2_GPIEN;
1154 /* Thermal Failure Detection */
1155 if (hw->mac.type == ixgbe_mac_X540)
1156 gpie |= IXGBE_SDP0_GPIEN;
1158 if (adapter->msix > 1) {
1159 /* Enable Enhanced MSIX mode */
1160 gpie |= IXGBE_GPIE_MSIX_MODE;
1161 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1164 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1167 if (ifp->if_mtu > ETHERMTU) {
1168 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1169 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1170 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1171 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1174 /* Now enable all the queues */
1176 for (int i = 0; i < adapter->num_queues; i++) {
1177 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1178 txdctl |= IXGBE_TXDCTL_ENABLE;
1179 /* Set WTHRESH to 8, burst writeback */
1180 txdctl |= (8 << 16);
1182 * When the internal queue falls below PTHRESH (32),
1183 * start prefetching as long as there are at least
1184 * HTHRESH (1) buffers ready. The values are taken
1185 * from the Intel linux driver 3.8.21.
1186 * Prefetching enables tx line rate even with 1 queue.
1188 txdctl |= (32 << 0) | (1 << 8);
1189 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1192 for (int i = 0; i < adapter->num_queues; i++) {
1193 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1194 if (hw->mac.type == ixgbe_mac_82598EB) {
1200 rxdctl &= ~0x3FFFFF;
1203 rxdctl |= IXGBE_RXDCTL_ENABLE;
1204 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1205 for (k = 0; k < 10; k++) {
1206 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1207 IXGBE_RXDCTL_ENABLE)
1215 * In netmap mode, we must preserve the buffers made
1216 * available to userspace before the if_init()
1217 * (this is true by default on the TX side, because
1218 * init makes all buffers available to userspace).
1220 * netmap_reset() and the device specific routines
1221 * (e.g. ixgbe_setup_receive_rings()) map these
1222 * buffers at the end of the NIC ring, so here we
1223 * must set the RDT (tail) register to make sure
1224 * they are not overwritten.
1226 * In this driver the NIC ring starts at RDH = 0,
1227 * RDT points to the last slot available for reception (?),
1228 * so RDT = num_rx_desc - 1 means the whole ring is available.
1230 if (ifp->if_capenable & IFCAP_NETMAP) {
1231 struct netmap_adapter *na = NA(adapter->ifp);
1232 struct netmap_kring *kring = &na->rx_rings[i];
1233 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1235 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1237 #endif /* DEV_NETMAP */
1238 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1241 /* Set up VLAN support and filter */
1242 ixgbe_setup_vlan_hw_support(adapter);
1244 /* Enable Receive engine */
1245 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1246 if (hw->mac.type == ixgbe_mac_82598EB)
1247 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1248 rxctrl |= IXGBE_RXCTRL_RXEN;
1249 ixgbe_enable_rx_dma(hw, rxctrl);
1251 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1253 /* Set up MSI/X routing */
1254 if (ixgbe_enable_msix) {
1255 ixgbe_configure_ivars(adapter);
1256 /* Set up auto-mask */
1257 if (hw->mac.type == ixgbe_mac_82598EB)
1258 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1260 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1261 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1263 } else { /* Simple settings for Legacy/MSI */
1264 ixgbe_set_ivar(adapter, 0, 0, 0);
1265 ixgbe_set_ivar(adapter, 0, 0, 1);
1266 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1270 /* Init Flow director */
1271 if (hw->mac.type != ixgbe_mac_82598EB) {
1272 u32 hdrm = 64 << fdir_pballoc;
1274 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1275 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1280 ** Check on any SFP devices that
1281 ** need to be kick-started
1283 if (hw->phy.type == ixgbe_phy_none) {
1284 int err = hw->phy.ops.identify(hw);
1285 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1287 "Unsupported SFP+ module type was detected.\n");
1292 /* Set moderation on the Link interrupt */
1293 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1295 /* Config/Enable Link */
1296 ixgbe_config_link(adapter);
1298 /* And now turn on interrupts */
1299 ixgbe_enable_intr(adapter);
1301 /* Now inform the stack we're ready */
1302 ifp->if_flags |= IFF_RUNNING;
1303 ifp->if_flags &= ~IFF_OACTIVE;
1309 ixgbe_init(void *arg)
1311 struct adapter *adapter = arg;
1313 IXGBE_CORE_LOCK(adapter);
1314 ixgbe_init_locked(adapter);
1315 IXGBE_CORE_UNLOCK(adapter);
1322 ** MSIX Interrupt Handlers and Tasklets
1327 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1329 struct ixgbe_hw *hw = &adapter->hw;
1330 u64 queue = (u64)(1 << vector);
1333 if (hw->mac.type == ixgbe_mac_82598EB) {
1334 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1335 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1337 mask = (queue & 0xFFFFFFFF);
1339 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1340 mask = (queue >> 32);
1342 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1347 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1349 struct ixgbe_hw *hw = &adapter->hw;
1350 u64 queue = (u64)(1 << vector);
1353 if (hw->mac.type == ixgbe_mac_82598EB) {
1354 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1355 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1357 mask = (queue & 0xFFFFFFFF);
1359 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1360 mask = (queue >> 32);
1362 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1367 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1371 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1372 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1373 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1375 mask = (queues & 0xFFFFFFFF);
1376 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1377 mask = (queues >> 32);
1378 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1384 ixgbe_handle_que(void *context, int pending)
1386 struct ix_queue *que = context;
1387 struct adapter *adapter = que->adapter;
1388 struct tx_ring *txr = que->txr;
1389 struct ifnet *ifp = adapter->ifp;
1392 if (ifp->if_flags & IFF_RUNNING) {
1393 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1396 #if 0 /*__FreeBSD_version >= 800000*/
1397 if (!drbr_empty(ifp, txr->br))
1398 ixgbe_mq_start_locked(ifp, txr, NULL);
1400 if (!ifq_is_empty(&ifp->if_snd))
1401 ixgbe_start_locked(txr, ifp);
1403 IXGBE_TX_UNLOCK(txr);
1405 taskqueue_enqueue(que->tq, &que->que_task);
1410 /* Reenable this interrupt */
1411 ixgbe_enable_queue(adapter, que->msix);
1416 /*********************************************************************
1418 * Legacy Interrupt Service routine
1420 **********************************************************************/
1423 ixgbe_legacy_irq(void *arg)
1425 struct ix_queue *que = arg;
1426 struct adapter *adapter = que->adapter;
1427 struct ixgbe_hw *hw = &adapter->hw;
1428 struct tx_ring *txr = adapter->tx_rings;
1429 bool more_tx, more_rx;
1430 u32 reg_eicr, loop = MAX_LOOP;
1433 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1436 if (reg_eicr == 0) {
1437 ixgbe_enable_intr(adapter);
1441 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1445 more_tx = ixgbe_txeof(txr);
1446 } while (loop-- && more_tx);
1447 IXGBE_TX_UNLOCK(txr);
1449 if (more_rx || more_tx)
1450 taskqueue_enqueue(que->tq, &que->que_task);
1452 /* Check for fan failure */
1453 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1454 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1455 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1456 "REPLACE IMMEDIATELY!!\n");
1457 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1460 /* Link status change */
1461 if (reg_eicr & IXGBE_EICR_LSC)
1462 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1464 ixgbe_enable_intr(adapter);
1469 /*********************************************************************
1471 * MSIX Queue Interrupt Service routine
1473 **********************************************************************/
1475 ixgbe_msix_que(void *arg)
1477 struct ix_queue *que = arg;
1478 struct adapter *adapter = que->adapter;
1479 struct tx_ring *txr = que->txr;
1480 struct rx_ring *rxr = que->rxr;
1481 bool more_tx, more_rx;
1484 ixgbe_disable_queue(adapter, que->msix);
1487 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1490 more_tx = ixgbe_txeof(txr);
1492 ** Make certain that if the stack
1493 ** has anything queued the task gets
1494 ** scheduled to handle it.
1497 #if __FreeBSD_version < 800000
1498 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1500 if (!drbr_empty(adapter->ifp, txr->br))
1503 if (!ifq_is_empty(&adapter->ifp->if_snd))
1505 IXGBE_TX_UNLOCK(txr);
1509 if (ixgbe_enable_aim == FALSE)
1512 ** Do Adaptive Interrupt Moderation:
1513 ** - Write out last calculated setting
1514 ** - Calculate based on average size over
1515 ** the last interval.
1517 if (que->eitr_setting)
1518 IXGBE_WRITE_REG(&adapter->hw,
1519 IXGBE_EITR(que->msix), que->eitr_setting);
1521 que->eitr_setting = 0;
1523 /* Idle, do nothing */
1524 if ((txr->bytes == 0) && (rxr->bytes == 0))
1527 if ((txr->bytes) && (txr->packets))
1528 newitr = txr->bytes/txr->packets;
1529 if ((rxr->bytes) && (rxr->packets))
1530 newitr = max(newitr,
1531 (rxr->bytes / rxr->packets));
1532 newitr += 24; /* account for hardware frame, crc */
1534 /* set an upper boundary */
1535 newitr = min(newitr, 3000);
1537 /* Be nice to the mid range */
1538 if ((newitr > 300) && (newitr < 1200))
1539 newitr = (newitr / 3);
1541 newitr = (newitr / 2);
1543 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1544 newitr |= newitr << 16;
1546 newitr |= IXGBE_EITR_CNT_WDIS;
1548 /* save for next interrupt */
1549 que->eitr_setting = newitr;
1558 if (more_tx || more_rx)
1559 taskqueue_enqueue(que->tq, &que->que_task);
1560 else /* Reenable this interrupt */
1561 ixgbe_enable_queue(adapter, que->msix);
1567 ixgbe_msix_link(void *arg)
1569 struct adapter *adapter = arg;
1570 struct ixgbe_hw *hw = &adapter->hw;
1573 ++adapter->link_irq;
1575 /* First get the cause */
1576 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1577 /* Clear interrupt with write */
1578 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1580 /* Link status change */
1581 if (reg_eicr & IXGBE_EICR_LSC)
1582 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1584 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1586 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1587 /* This is probably overkill :) */
1588 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1590 /* Clear the interrupt */
1591 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR);
1592 /* Turn off the interface */
1593 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1594 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1597 if (reg_eicr & IXGBE_EICR_ECC) {
1598 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1599 "Please Reboot!!\n");
1600 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1603 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1604 /* Clear the interrupt */
1605 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1606 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1607 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1608 /* Clear the interrupt */
1609 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1610 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1614 /* Check for fan failure */
1615 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1616 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1617 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1618 "REPLACE IMMEDIATELY!!\n");
1619 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1622 /* Check for over temp condition */
1623 if ((hw->mac.type == ixgbe_mac_X540) &&
1624 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1625 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1626 "PHY IS SHUT DOWN!!\n");
1627 device_printf(adapter->dev, "System shutdown required\n");
1628 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1631 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1635 /*********************************************************************
1637 * Media Ioctl callback
1639 * This routine is called whenever the user queries the status of
1640 * the interface using ifconfig.
1642 **********************************************************************/
1644 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1646 struct adapter *adapter = ifp->if_softc;
1648 INIT_DEBUGOUT("ixgbe_media_status: begin");
1649 IXGBE_CORE_LOCK(adapter);
1650 ixgbe_update_link_status(adapter);
1652 ifmr->ifm_status = IFM_AVALID;
1653 ifmr->ifm_active = IFM_ETHER;
1655 if (!adapter->link_active) {
1656 IXGBE_CORE_UNLOCK(adapter);
1660 ifmr->ifm_status |= IFM_ACTIVE;
1662 switch (adapter->link_speed) {
1663 case IXGBE_LINK_SPEED_100_FULL:
1664 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1666 case IXGBE_LINK_SPEED_1GB_FULL:
1667 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1669 case IXGBE_LINK_SPEED_10GB_FULL:
1670 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1674 IXGBE_CORE_UNLOCK(adapter);
1679 /*********************************************************************
1681 * Media Ioctl callback
1683 * This routine is called when the user changes speed/duplex using
1684 * media/mediopt option with ifconfig.
1686 **********************************************************************/
1688 ixgbe_media_change(struct ifnet * ifp)
1690 struct adapter *adapter = ifp->if_softc;
1691 struct ifmedia *ifm = &adapter->media;
1693 INIT_DEBUGOUT("ixgbe_media_change: begin");
1695 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1698 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1700 adapter->hw.phy.autoneg_advertised =
1701 IXGBE_LINK_SPEED_100_FULL |
1702 IXGBE_LINK_SPEED_1GB_FULL |
1703 IXGBE_LINK_SPEED_10GB_FULL;
1706 device_printf(adapter->dev, "Only auto media type\n");
1713 /*********************************************************************
1715 * This routine maps the mbufs to tx descriptors, allowing the
1716 * TX engine to transmit the packets.
1717 * - return 0 on success, positive on failure
1719 **********************************************************************/
1722 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1724 struct adapter *adapter = txr->adapter;
1725 u32 olinfo_status = 0, cmd_type_len;
1727 int i, j, error, nsegs;
1728 int first, last = 0;
1729 struct mbuf *m_head;
1730 bus_dma_segment_t segs[1];
1732 struct ixgbe_tx_buf *txbuf;
1733 union ixgbe_adv_tx_desc *txd = NULL;
1737 /* Basic descriptor defines */
1738 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1739 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1741 if (m_head->m_flags & M_VLANTAG)
1742 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1745 * Important to capture the first descriptor
1746 * used because it will contain the index of
1747 * the one we tell the hardware to report back
1749 first = txr->next_avail_desc;
1750 txbuf = &txr->tx_buffers[first];
1754 * Map the packet for DMA.
1756 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1757 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1759 if (error == EFBIG) {
1762 m = m_defrag(*m_headp, MB_DONTWAIT);
1764 adapter->mbuf_defrag_failed++;
1772 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1773 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1775 if (error == ENOMEM) {
1776 adapter->no_tx_dma_setup++;
1778 } else if (error != 0) {
1779 adapter->no_tx_dma_setup++;
1784 } else if (error == ENOMEM) {
1785 adapter->no_tx_dma_setup++;
1787 } else if (error != 0) {
1788 adapter->no_tx_dma_setup++;
1794 /* Make certain there are enough descriptors */
1795 if (nsegs > txr->tx_avail - 2) {
1796 txr->no_desc_avail++;
1803 ** Set up the appropriate offload context
1804 ** this becomes the first descriptor of
1808 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1809 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1810 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1811 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1812 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1816 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1818 if (ixgbe_tx_ctx_setup(txr, m_head))
1819 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1821 #ifdef IXGBE_IEEE1588
1822 /* This is changing soon to an mtag detection */
1823 if (we detect this mbuf has a TSTAMP mtag)
1824 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1828 /* Do the flow director magic */
1829 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1831 if (txr->atr_count >= atr_sample_rate) {
1832 ixgbe_atr(txr, m_head);
1837 /* Record payload length */
1839 olinfo_status |= m_head->m_pkthdr.len <<
1840 IXGBE_ADVTXD_PAYLEN_SHIFT;
1842 i = txr->next_avail_desc;
1843 for (j = 0; j < nsegs; j++) {
1847 txbuf = &txr->tx_buffers[i];
1848 txd = &txr->tx_base[i];
1849 seglen = segs[j].ds_len;
1850 segaddr = htole64(segs[j].ds_addr);
1852 txd->read.buffer_addr = segaddr;
1853 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1854 cmd_type_len |seglen);
1855 txd->read.olinfo_status = htole32(olinfo_status);
1856 last = i; /* descriptor that will get completion IRQ */
1858 if (++i == adapter->num_tx_desc)
1861 txbuf->m_head = NULL;
1862 txbuf->eop_index = -1;
1865 txd->read.cmd_type_len |=
1866 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1867 txr->tx_avail -= nsegs;
1868 txr->next_avail_desc = i;
1870 txbuf->m_head = m_head;
1871 /* Swap the dma map between the first and last descriptor */
1872 txr->tx_buffers[first].map = txbuf->map;
1874 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1876 /* Set the index of the descriptor that will be marked done */
1877 txbuf = &txr->tx_buffers[first];
1878 txbuf->eop_index = last;
1880 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1881 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1883 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1884 * hardware that this frame is available to transmit.
1886 ++txr->total_packets;
1887 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1892 bus_dmamap_unload(txr->txtag, txbuf->map);
1898 ixgbe_set_promisc(struct adapter *adapter)
1901 struct ifnet *ifp = adapter->ifp;
1903 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1904 reg_rctl &= (~IXGBE_FCTRL_UPE);
1905 reg_rctl &= (~IXGBE_FCTRL_MPE);
1906 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1908 if (ifp->if_flags & IFF_PROMISC) {
1909 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1910 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1911 } else if (ifp->if_flags & IFF_ALLMULTI) {
1912 reg_rctl |= IXGBE_FCTRL_MPE;
1913 reg_rctl &= ~IXGBE_FCTRL_UPE;
1914 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1920 /*********************************************************************
1923 * This routine is called whenever multicast address list is updated.
1925 **********************************************************************/
1926 #define IXGBE_RAR_ENTRIES 16
1929 ixgbe_set_multi(struct adapter *adapter)
1934 struct ifmultiaddr *ifma;
1936 struct ifnet *ifp = adapter->ifp;
1938 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1941 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1942 MAX_NUM_MULTICAST_ADDRESSES);
1944 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1945 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1946 if (ifp->if_flags & IFF_PROMISC)
1947 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1948 else if (ifp->if_flags & IFF_ALLMULTI) {
1949 fctrl |= IXGBE_FCTRL_MPE;
1950 fctrl &= ~IXGBE_FCTRL_UPE;
1952 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1954 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1956 spin_lock(&adapter->mcast_spin);
1957 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1958 if (ifma->ifma_addr->sa_family != AF_LINK)
1960 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1961 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1962 IXGBE_ETH_LENGTH_OF_ADDRESS);
1965 spin_unlock(&adapter->mcast_spin);
1968 ixgbe_update_mc_addr_list(&adapter->hw,
1969 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1975 * This is an iterator function now needed by the multicast
1976 * shared code. It simply feeds the shared code routine the
1977 * addresses in the array of ixgbe_set_multi() one by one.
1980 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1982 u8 *addr = *update_ptr;
1986 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1987 *update_ptr = newptr;
1992 /*********************************************************************
1995 * This routine checks for link status,updates statistics,
1996 * and runs the watchdog check.
1998 **********************************************************************/
2001 ixgbe_local_timer(void *arg)
2003 struct adapter *adapter = arg;
2004 device_t dev = adapter->dev;
2005 struct ifnet *ifp = adapter->ifp;
2006 struct ix_queue *que = adapter->queues;
2007 struct tx_ring *txr = adapter->tx_rings;
2008 int hung, busy, paused;
2010 lockmgr(&adapter->core_lock, LK_EXCLUSIVE);
2011 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2012 hung = busy = paused = 0;
2014 /* Check for pluggable optics */
2015 if (adapter->sfp_probe)
2016 if (!ixgbe_sfp_probe(adapter))
2017 goto out; /* Nothing to do */
2019 ixgbe_update_link_status(adapter);
2020 ixgbe_update_stats_counters(adapter);
2023 * If the interface has been paused
2024 * then don't do the watchdog check
2026 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2030 ** Check the TX queues status
2031 ** - central locked handling of OACTIVE
2032 ** - watchdog only if all queues show hung
2034 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2035 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2038 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2040 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2041 taskqueue_enqueue(que->tq, &que->que_task);
2043 /* Only truely watchdog if all queues show hung */
2044 if (hung == adapter->num_queues)
2046 /* Only turn off the stack flow when ALL are depleted */
2047 if (busy == adapter->num_queues)
2048 ifp->if_flags |= IFF_OACTIVE;
2049 else if ((ifp->if_flags & IFF_OACTIVE) &&
2050 (busy < adapter->num_queues))
2051 ifp->if_flags &= ~IFF_OACTIVE;
2054 ixgbe_rearm_queues(adapter, adapter->que_mask);
2055 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2056 lockmgr(&adapter->core_lock, LK_RELEASE);
2060 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2061 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2062 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2063 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2064 device_printf(dev,"TX(%d) desc avail = %d,"
2065 "Next TX to Clean = %d\n",
2066 txr->me, txr->tx_avail, txr->next_to_clean);
2067 adapter->ifp->if_flags &= ~IFF_RUNNING;
2068 adapter->watchdog_events++;
2069 ixgbe_init_locked(adapter);
2071 lockmgr(&adapter->core_lock, LK_RELEASE);
2075 ** Note: this routine updates the OS on the link state
2076 ** the real check of the hardware only happens with
2077 ** a link interrupt.
2080 ixgbe_update_link_status(struct adapter *adapter)
2082 struct ifnet *ifp = adapter->ifp;
2083 struct tx_ring *txr = adapter->tx_rings;
2084 device_t dev = adapter->dev;
2087 if (adapter->link_up){
2088 if (adapter->link_active == FALSE) {
2090 device_printf(dev,"Link is up %d Gbps %s \n",
2091 ((adapter->link_speed == 128)? 10:1),
2093 adapter->link_active = TRUE;
2094 ifp->if_link_state = LINK_STATE_UP;
2095 if_link_state_change(ifp);
2097 } else { /* Link down */
2098 if (adapter->link_active == TRUE) {
2100 device_printf(dev,"Link is Down\n");
2101 ifp->if_link_state = LINK_STATE_DOWN;
2102 if_link_state_change(ifp);
2103 adapter->link_active = FALSE;
2104 for (int i = 0; i < adapter->num_queues;
2106 txr->queue_status = IXGBE_QUEUE_IDLE;
2114 /*********************************************************************
2116 * This routine disables all traffic on the adapter by issuing a
2117 * global reset on the MAC and deallocates TX/RX buffers.
2119 **********************************************************************/
2122 ixgbe_stop(void *arg)
2125 struct adapter *adapter = arg;
2126 struct ixgbe_hw *hw = &adapter->hw;
2129 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2131 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2132 ixgbe_disable_intr(adapter);
2133 callout_stop(&adapter->timer);
2135 /* Let the stack know...*/
2136 ifp->if_flags &= ~IFF_RUNNING;
2137 ifp->if_flags |= IFF_OACTIVE;
2140 hw->adapter_stopped = FALSE;
2141 ixgbe_stop_adapter(hw);
2142 /* Turn off the laser */
2143 if (hw->phy.multispeed_fiber)
2144 ixgbe_disable_tx_laser(hw);
2146 /* reprogram the RAR[0] in case user changed it. */
2147 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2153 /*********************************************************************
2155 * Determine hardware revision.
2157 **********************************************************************/
2159 ixgbe_identify_hardware(struct adapter *adapter)
2161 device_t dev = adapter->dev;
2162 struct ixgbe_hw *hw = &adapter->hw;
2164 /* Save off the information about this board */
2165 hw->vendor_id = pci_get_vendor(dev);
2166 hw->device_id = pci_get_device(dev);
2167 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2168 hw->subsystem_vendor_id =
2169 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2170 hw->subsystem_device_id =
2171 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2173 /* We need this here to set the num_segs below */
2174 ixgbe_set_mac_type(hw);
2176 /* Pick up the 82599 and VF settings */
2177 if (hw->mac.type != ixgbe_mac_82598EB) {
2178 hw->phy.smart_speed = ixgbe_smart_speed;
2179 adapter->num_segs = IXGBE_82599_SCATTER;
2181 adapter->num_segs = IXGBE_82598_SCATTER;
2186 /*********************************************************************
2188 * Determine optic type
2190 **********************************************************************/
2192 ixgbe_setup_optics(struct adapter *adapter)
2194 struct ixgbe_hw *hw = &adapter->hw;
2197 layer = ixgbe_get_supported_physical_layer(hw);
2199 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2200 adapter->optics = IFM_10G_T;
2204 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2205 adapter->optics = IFM_1000_T;
2209 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2210 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2211 adapter->optics = IFM_10G_LR;
2215 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2216 adapter->optics = IFM_10G_SR;
2220 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2221 adapter->optics = IFM_10G_TWINAX;
2225 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2226 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2227 adapter->optics = IFM_10G_CX4;
2231 /* If we get here just set the default */
2232 adapter->optics = IFM_ETHER | IFM_AUTO;
2236 /*********************************************************************
2238 * Setup the Legacy or MSI Interrupt handler
2240 **********************************************************************/
2242 ixgbe_allocate_legacy(struct adapter *adapter)
2244 device_t dev = adapter->dev;
2245 struct ix_queue *que = adapter->queues;
2249 if (adapter->msix == 1)
2252 /* We allocate a single interrupt resource */
2253 adapter->res = bus_alloc_resource_any(dev,
2254 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2255 if (adapter->res == NULL) {
2256 device_printf(dev, "Unable to allocate bus resource: "
2262 * Try allocating a fast interrupt and the associated deferred
2263 * processing contexts.
2265 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2266 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2267 taskqueue_thread_enqueue, &que->tq);
2268 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2269 device_get_nameunit(adapter->dev));
2271 /* Tasklets for Link, SFP and Multispeed Fiber */
2272 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2273 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2274 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2276 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2278 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2279 taskqueue_thread_enqueue, &adapter->tq);
2280 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2281 device_get_nameunit(adapter->dev));
2283 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2284 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2285 device_printf(dev, "Failed to register fast interrupt "
2286 "handler: %d\n", error);
2287 taskqueue_free(que->tq);
2288 taskqueue_free(adapter->tq);
2293 /* For simplicity in the handlers */
2294 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2300 /*********************************************************************
2302 * Setup MSIX Interrupt resources and handlers
2304 **********************************************************************/
2306 ixgbe_allocate_msix(struct adapter *adapter)
2308 device_t dev = adapter->dev;
2309 struct ix_queue *que = adapter->queues;
2310 int error, rid, vector = 0;
2312 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2314 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2315 RF_SHAREABLE | RF_ACTIVE);
2316 if (que->res == NULL) {
2317 device_printf(dev,"Unable to allocate"
2318 " bus resource: que interrupt [%d]\n", vector);
2321 /* Set the handler function */
2322 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2323 ixgbe_msix_que, que, &que->tag, &que->serializer);
2326 device_printf(dev, "Failed to register QUE handler");
2329 #if 0 /* __FreeBSD_version >= 800504 */
2330 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2333 adapter->que_mask |= (u64)(1 << que->msix);
2335 ** Bind the msix vector, and thus the
2336 ** ring to the corresponding cpu.
2339 if (adapter->num_queues > 1)
2340 bus_bind_intr(dev, que->res, i);
2343 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2344 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2345 taskqueue_thread_enqueue, &que->tq);
2346 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2347 device_get_nameunit(adapter->dev));
2352 adapter->res = bus_alloc_resource_any(dev,
2353 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2354 if (!adapter->res) {
2355 device_printf(dev,"Unable to allocate"
2356 " bus resource: Link interrupt [%d]\n", rid);
2359 /* Set the link handler function */
2360 error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2361 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2363 adapter->res = NULL;
2364 device_printf(dev, "Failed to register LINK handler");
2367 #if 0 /* __FreeBSD_version >= 800504 */
2368 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2370 adapter->linkvec = vector;
2371 /* Tasklets for Link, SFP and Multispeed Fiber */
2372 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2373 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2374 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2376 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2378 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2379 taskqueue_thread_enqueue, &adapter->tq);
2380 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2381 device_get_nameunit(adapter->dev));
2386 #if 0 /* HAVE_MSIX */
2388 * Setup Either MSI/X or MSI
2391 ixgbe_setup_msix(struct adapter *adapter)
2393 device_t dev = adapter->dev;
2394 int rid, want, queues, msgs;
2396 /* Override by tuneable */
2397 if (ixgbe_enable_msix == 0)
2400 /* First try MSI/X */
2401 rid = PCIR_BAR(MSIX_82598_BAR);
2402 adapter->msix_mem = bus_alloc_resource_any(dev,
2403 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2404 if (!adapter->msix_mem) {
2405 rid += 4; /* 82599 maps in higher BAR */
2406 adapter->msix_mem = bus_alloc_resource_any(dev,
2407 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2409 if (!adapter->msix_mem) {
2410 /* May not be enabled */
2411 device_printf(adapter->dev,
2412 "Unable to map MSIX table \n");
2416 msgs = pci_msix_count(dev);
2417 if (msgs == 0) { /* system has msix disabled */
2418 bus_release_resource(dev, SYS_RES_MEMORY,
2419 rid, adapter->msix_mem);
2420 adapter->msix_mem = NULL;
2424 /* Figure out a reasonable auto config value */
2425 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2427 if (ixgbe_num_queues != 0)
2428 queues = ixgbe_num_queues;
2429 /* Set max queues to 8 when autoconfiguring */
2430 else if ((ixgbe_num_queues == 0) && (queues > 8))
2434 ** Want one vector (RX/TX pair) per queue
2435 ** plus an additional for Link.
2441 device_printf(adapter->dev,
2442 "MSIX Configuration Problem, "
2443 "%d vectors but %d queues wanted!\n",
2445 return (0); /* Will go to Legacy setup */
2447 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2448 device_printf(adapter->dev,
2449 "Using MSIX interrupts with %d vectors\n", msgs);
2450 adapter->num_queues = queues;
2454 msgs = pci_msi_count(dev);
2455 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2456 device_printf(adapter->dev,"Using an MSI interrupt\n");
2458 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2465 ixgbe_allocate_pci_resources(struct adapter *adapter)
2468 device_t dev = adapter->dev;
2471 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2474 if (!(adapter->pci_mem)) {
2475 device_printf(dev,"Unable to allocate bus resource: memory\n");
2479 adapter->osdep.mem_bus_space_tag =
2480 rman_get_bustag(adapter->pci_mem);
2481 adapter->osdep.mem_bus_space_handle =
2482 rman_get_bushandle(adapter->pci_mem);
2483 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2485 /* Legacy defaults */
2486 adapter->num_queues = 1;
2487 adapter->hw.back = &adapter->osdep;
2490 ** Now setup MSI or MSI/X, should
2491 ** return us the number of supported
2492 ** vectors. (Will be 1 for MSI)
2494 #if 0 /* HAVE_MSIX */
2495 adapter->msix = ixgbe_setup_msix(adapter);
2501 ixgbe_free_pci_resources(struct adapter * adapter)
2503 struct ix_queue *que = adapter->queues;
2504 device_t dev = adapter->dev;
2507 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2508 memrid = PCIR_BAR(MSIX_82598_BAR);
2510 memrid = PCIR_BAR(MSIX_82599_BAR);
2513 ** There is a slight possibility of a failure mode
2514 ** in attach that will result in entering this function
2515 ** before interrupt resources have been initialized, and
2516 ** in that case we do not want to execute the loops below
2517 ** We can detect this reliably by the state of the adapter
2520 if (adapter->res == NULL)
2524 ** Release all msix queue resources:
2526 for (int i = 0; i < adapter->num_queues; i++, que++) {
2527 rid = que->msix + 1;
2528 if (que->tag != NULL) {
2529 bus_teardown_intr(dev, que->res, que->tag);
2532 if (que->res != NULL)
2533 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2537 /* Clean the Legacy or Link interrupt last */
2538 if (adapter->linkvec) /* we are doing MSIX */
2539 rid = adapter->linkvec + 1;
2541 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2543 if (adapter->tag != NULL) {
2544 bus_teardown_intr(dev, adapter->res, adapter->tag);
2545 adapter->tag = NULL;
2547 if (adapter->res != NULL)
2548 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2552 pci_release_msi(dev);
2554 if (adapter->msix_mem != NULL)
2555 bus_release_resource(dev, SYS_RES_MEMORY,
2556 memrid, adapter->msix_mem);
2558 if (adapter->pci_mem != NULL)
2559 bus_release_resource(dev, SYS_RES_MEMORY,
2560 PCIR_BAR(0), adapter->pci_mem);
2565 /*********************************************************************
2567 * Setup networking device structure and register an interface.
2569 **********************************************************************/
2571 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2573 struct ixgbe_hw *hw = &adapter->hw;
2576 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2578 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2580 device_printf(dev, "can not allocate ifnet structure\n");
2583 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2584 ifp->if_baudrate = 1000000000;
2585 ifp->if_init = ixgbe_init;
2586 ifp->if_softc = adapter;
2587 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2588 ifp->if_ioctl = ixgbe_ioctl;
2589 ifp->if_start = ixgbe_start;
2590 #if 0 /* __FreeBSD_version >= 800000 */
2591 ifp->if_transmit = ixgbe_mq_start;
2592 ifp->if_qflush = ixgbe_qflush;
2594 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2596 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2598 adapter->max_frame_size =
2599 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2602 * Tell the upper layer(s) we support long frames.
2604 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2607 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2609 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2610 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2611 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2616 ifp->if_capenable = ifp->if_capabilities;
2618 /* Don't enable LRO by default */
2620 ifp->if_capabilities |= IFCAP_LRO;
2624 ** Don't turn this on by default, if vlans are
2625 ** created on another pseudo device (eg. lagg)
2626 ** then vlan events are not passed thru, breaking
2627 ** operation, but with HW FILTER off it works. If
2628 ** using vlans directly on the ixgbe driver you can
2629 ** enable this and get full hardware tag filtering.
2631 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2634 * Specify the media types supported by this adapter and register
2635 * callbacks to update media and link information
2637 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2638 ixgbe_media_status);
2639 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2640 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2641 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2642 ifmedia_add(&adapter->media,
2643 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2644 ifmedia_add(&adapter->media,
2645 IFM_ETHER | IFM_1000_T, 0, NULL);
2647 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2648 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2654 ixgbe_config_link(struct adapter *adapter)
2656 struct ixgbe_hw *hw = &adapter->hw;
2657 u32 autoneg, err = 0;
2658 bool sfp, negotiate;
2660 sfp = ixgbe_is_sfp(hw);
2663 if (hw->phy.multispeed_fiber) {
2664 hw->mac.ops.setup_sfp(hw);
2665 ixgbe_enable_tx_laser(hw);
2666 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2668 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2670 if (hw->mac.ops.check_link)
2671 err = ixgbe_check_link(hw, &autoneg,
2672 &adapter->link_up, FALSE);
2675 autoneg = hw->phy.autoneg_advertised;
2676 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2677 err = hw->mac.ops.get_link_capabilities(hw,
2678 &autoneg, &negotiate);
2681 if (hw->mac.ops.setup_link)
2682 err = hw->mac.ops.setup_link(hw, autoneg,
2683 negotiate, adapter->link_up);
2689 /********************************************************************
2690 * Manage DMA'able memory.
2691 *******************************************************************/
2693 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2697 *(bus_addr_t *) arg = segs->ds_addr;
2702 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2703 struct ixgbe_dma_alloc *dma, int mapflags)
2705 device_t dev = adapter->dev;
2708 r = bus_dma_tag_create(NULL, /* parent */
2709 DBA_ALIGN, 0, /* alignment, bounds */
2710 BUS_SPACE_MAXADDR, /* lowaddr */
2711 BUS_SPACE_MAXADDR, /* highaddr */
2712 NULL, NULL, /* filter, filterarg */
2715 size, /* maxsegsize */
2716 BUS_DMA_ALLOCNOW, /* flags */
2719 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2723 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2724 BUS_DMA_NOWAIT, &dma->dma_map);
2726 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2730 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2734 mapflags | BUS_DMA_NOWAIT);
2736 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2740 dma->dma_size = size;
2743 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2745 bus_dma_tag_destroy(dma->dma_tag);
2747 dma->dma_map = NULL;
2748 dma->dma_tag = NULL;
2753 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2755 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2756 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2757 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2758 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2759 bus_dma_tag_destroy(dma->dma_tag);
2763 /*********************************************************************
2765 * Allocate memory for the transmit and receive rings, and then
2766 * the descriptors associated with each, called only once at attach.
2768 **********************************************************************/
2770 ixgbe_allocate_queues(struct adapter *adapter)
2772 device_t dev = adapter->dev;
2773 struct ix_queue *que;
2774 struct tx_ring *txr;
2775 struct rx_ring *rxr;
2776 int rsize, tsize, error = IXGBE_SUCCESS;
2777 int txconf = 0, rxconf = 0;
2779 /* First allocate the top level queue structs */
2780 if (!(adapter->queues =
2781 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2782 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2783 device_printf(dev, "Unable to allocate queue memory\n");
2788 /* First allocate the TX ring struct memory */
2789 if (!(adapter->tx_rings =
2790 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2791 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2792 device_printf(dev, "Unable to allocate TX ring memory\n");
2797 /* Next allocate the RX */
2798 if (!(adapter->rx_rings =
2799 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2800 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2801 device_printf(dev, "Unable to allocate RX ring memory\n");
2806 /* For the ring itself */
2807 tsize = roundup2(adapter->num_tx_desc *
2808 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2811 * Now set up the TX queues, txconf is needed to handle the
2812 * possibility that things fail midcourse and we need to
2813 * undo memory gracefully
2815 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2816 /* Set up some basics */
2817 txr = &adapter->tx_rings[i];
2818 txr->adapter = adapter;
2821 /* Initialize the TX side lock */
2822 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2823 device_get_nameunit(dev), txr->me);
2824 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2826 if (ixgbe_dma_malloc(adapter, tsize,
2827 &txr->txdma, BUS_DMA_NOWAIT)) {
2829 "Unable to allocate TX Descriptor memory\n");
2833 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2834 bzero((void *)txr->tx_base, tsize);
2836 /* Now allocate transmit buffers for the ring */
2837 if (ixgbe_allocate_transmit_buffers(txr)) {
2839 "Critical Failure setting up transmit buffers\n");
2843 #if 0 /* __FreeBSD_version >= 800000 */
2844 /* Allocate a buf ring */
2845 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2846 M_WAITOK, &txr->tx_mtx);
2847 if (txr->br == NULL) {
2849 "Critical Failure setting up buf ring\n");
2857 * Next the RX queues...
2859 rsize = roundup2(adapter->num_rx_desc *
2860 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2861 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2862 rxr = &adapter->rx_rings[i];
2863 /* Set up some basics */
2864 rxr->adapter = adapter;
2867 /* Initialize the RX side lock */
2868 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2869 device_get_nameunit(dev), rxr->me);
2870 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2872 if (ixgbe_dma_malloc(adapter, rsize,
2873 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2875 "Unable to allocate RxDescriptor memory\n");
2879 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2880 bzero((void *)rxr->rx_base, rsize);
2882 /* Allocate receive buffers for the ring*/
2883 if (ixgbe_allocate_receive_buffers(rxr)) {
2885 "Critical Failure setting up receive buffers\n");
2892 ** Finally set up the queue holding structs
2894 for (int i = 0; i < adapter->num_queues; i++) {
2895 que = &adapter->queues[i];
2896 que->adapter = adapter;
2897 que->txr = &adapter->tx_rings[i];
2898 que->rxr = &adapter->rx_rings[i];
2904 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2905 ixgbe_dma_free(adapter, &rxr->rxdma);
2907 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2908 ixgbe_dma_free(adapter, &txr->txdma);
2909 kfree(adapter->rx_rings, M_DEVBUF);
2911 kfree(adapter->tx_rings, M_DEVBUF);
2913 kfree(adapter->queues, M_DEVBUF);
2918 /*********************************************************************
2920 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2921 * the information needed to transmit a packet on the wire. This is
2922 * called only once at attach, setup is done every reset.
2924 **********************************************************************/
2926 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2928 struct adapter *adapter = txr->adapter;
2929 device_t dev = adapter->dev;
2930 struct ixgbe_tx_buf *txbuf;
2934 * Setup DMA descriptor areas.
2936 if ((error = bus_dma_tag_create(
2938 1, 0, /* alignment, bounds */
2939 BUS_SPACE_MAXADDR, /* lowaddr */
2940 BUS_SPACE_MAXADDR, /* highaddr */
2941 NULL, NULL, /* filter, filterarg */
2942 IXGBE_TSO_SIZE, /* maxsize */
2943 adapter->num_segs, /* nsegments */
2944 PAGE_SIZE, /* maxsegsize */
2947 device_printf(dev,"Unable to allocate TX DMA tag\n");
2951 if (!(txr->tx_buffers =
2952 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2953 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2954 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2959 /* Create the descriptor buffer dma maps */
2960 txbuf = txr->tx_buffers;
2961 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2962 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2964 device_printf(dev, "Unable to create TX DMA map\n");
2971 /* We free all, it handles case where we are in the middle */
2972 ixgbe_free_transmit_structures(adapter);
2976 /*********************************************************************
2978 * Initialize a transmit ring.
2980 **********************************************************************/
2982 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2984 struct adapter *adapter = txr->adapter;
2985 struct ixgbe_tx_buf *txbuf;
2988 struct netmap_adapter *na = NA(adapter->ifp);
2989 struct netmap_slot *slot;
2990 #endif /* DEV_NETMAP */
2992 /* Clear the old ring contents */
2996 * (under lock): if in netmap mode, do some consistency
2997 * checks and set slot to entry 0 of the netmap ring.
2999 slot = netmap_reset(na, NR_TX, txr->me, 0);
3000 #endif /* DEV_NETMAP */
3001 bzero((void *)txr->tx_base,
3002 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3004 txr->next_avail_desc = 0;
3005 txr->next_to_clean = 0;
3007 /* Free any existing tx buffers. */
3008 txbuf = txr->tx_buffers;
3009 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3010 if (txbuf->m_head != NULL) {
3011 bus_dmamap_sync(txr->txtag, txbuf->map,
3012 BUS_DMASYNC_POSTWRITE);
3013 bus_dmamap_unload(txr->txtag, txbuf->map);
3014 m_freem(txbuf->m_head);
3015 txbuf->m_head = NULL;
3019 * In netmap mode, set the map for the packet buffer.
3020 * NOTE: Some drivers (not this one) also need to set
3021 * the physical buffer address in the NIC ring.
3022 * Slots in the netmap ring (indexed by "si") are
3023 * kring->nkr_hwofs positions "ahead" wrt the
3024 * corresponding slot in the NIC ring. In some drivers
3025 * (not here) nkr_hwofs can be negative. Function
3026 * netmap_idx_n2k() handles wraparounds properly.
3029 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3030 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3032 #endif /* DEV_NETMAP */
3033 /* Clear the EOP index */
3034 txbuf->eop_index = -1;
3038 /* Set the rate at which we sample packets */
3039 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3040 txr->atr_sample = atr_sample_rate;
3043 /* Set number of descriptors available */
3044 txr->tx_avail = adapter->num_tx_desc;
3046 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3047 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3048 IXGBE_TX_UNLOCK(txr);
3051 /*********************************************************************
3053 * Initialize all transmit rings.
3055 **********************************************************************/
3057 ixgbe_setup_transmit_structures(struct adapter *adapter)
3059 struct tx_ring *txr = adapter->tx_rings;
3061 for (int i = 0; i < adapter->num_queues; i++, txr++)
3062 ixgbe_setup_transmit_ring(txr);
3067 /*********************************************************************
3069 * Enable transmit unit.
3071 **********************************************************************/
3073 ixgbe_initialize_transmit_units(struct adapter *adapter)
3075 struct tx_ring *txr = adapter->tx_rings;
3076 struct ixgbe_hw *hw = &adapter->hw;
3078 /* Setup the Base and Length of the Tx Descriptor Ring */
3080 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3081 u64 tdba = txr->txdma.dma_paddr;
3084 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3085 (tdba & 0x00000000ffffffffULL));
3086 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3087 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3088 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3090 /* Setup the HW Tx Head and Tail descriptor pointers */
3091 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3092 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3094 /* Setup Transmit Descriptor Cmd Settings */
3095 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3096 txr->queue_status = IXGBE_QUEUE_IDLE;
3098 /* Disable Head Writeback */
3099 switch (hw->mac.type) {
3100 case ixgbe_mac_82598EB:
3101 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3103 case ixgbe_mac_82599EB:
3104 case ixgbe_mac_X540:
3106 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3109 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
3110 switch (hw->mac.type) {
3111 case ixgbe_mac_82598EB:
3112 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3114 case ixgbe_mac_82599EB:
3115 case ixgbe_mac_X540:
3117 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3123 if (hw->mac.type != ixgbe_mac_82598EB) {
3124 u32 dmatxctl, rttdcs;
3125 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3126 dmatxctl |= IXGBE_DMATXCTL_TE;
3127 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3128 /* Disable arbiter to set MTQC */
3129 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3130 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3131 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3132 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3133 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3134 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3140 /*********************************************************************
3142 * Free all transmit rings.
3144 **********************************************************************/
3146 ixgbe_free_transmit_structures(struct adapter *adapter)
3148 struct tx_ring *txr = adapter->tx_rings;
3150 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3152 ixgbe_free_transmit_buffers(txr);
3153 ixgbe_dma_free(adapter, &txr->txdma);
3154 IXGBE_TX_UNLOCK(txr);
3155 IXGBE_TX_LOCK_DESTROY(txr);
3157 kfree(adapter->tx_rings, M_DEVBUF);
3160 /*********************************************************************
3162 * Free transmit ring related data structures.
3164 **********************************************************************/
3166 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3168 struct adapter *adapter = txr->adapter;
3169 struct ixgbe_tx_buf *tx_buffer;
3172 INIT_DEBUGOUT("free_transmit_ring: begin");
3174 if (txr->tx_buffers == NULL)
3177 tx_buffer = txr->tx_buffers;
3178 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3179 if (tx_buffer->m_head != NULL) {
3180 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3181 BUS_DMASYNC_POSTWRITE);
3182 bus_dmamap_unload(txr->txtag,
3184 m_freem(tx_buffer->m_head);
3185 tx_buffer->m_head = NULL;
3186 if (tx_buffer->map != NULL) {
3187 bus_dmamap_destroy(txr->txtag,
3189 tx_buffer->map = NULL;
3191 } else if (tx_buffer->map != NULL) {
3192 bus_dmamap_unload(txr->txtag,
3194 bus_dmamap_destroy(txr->txtag,
3196 tx_buffer->map = NULL;
3199 #if 0 /* __FreeBSD_version >= 800000 */
3200 if (txr->br != NULL)
3201 buf_ring_free(txr->br, M_DEVBUF);
3203 if (txr->tx_buffers != NULL) {
3204 kfree(txr->tx_buffers, M_DEVBUF);
3205 txr->tx_buffers = NULL;
3207 if (txr->txtag != NULL) {
3208 bus_dma_tag_destroy(txr->txtag);
3214 /*********************************************************************
3216 * Advanced Context Descriptor setup for VLAN or CSUM
3218 **********************************************************************/
3221 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3223 struct adapter *adapter = txr->adapter;
3224 struct ixgbe_adv_tx_context_desc *TXD;
3225 struct ixgbe_tx_buf *tx_buffer;
3226 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3227 struct ether_vlan_header *eh;
3229 struct ip6_hdr *ip6;
3230 int ehdrlen, ip_hlen = 0;
3233 bool offload = TRUE;
3234 int ctxd = txr->next_avail_desc;
3240 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3243 tx_buffer = &txr->tx_buffers[ctxd];
3244 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3247 ** In advanced descriptors the vlan tag must
3248 ** be placed into the descriptor itself.
3251 if (mp->m_flags & M_VLANTAG) {
3252 vtag = htole16(mp->m_pkthdr.ether_vtag);
3253 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3254 } else if (offload == FALSE)
3259 * Determine where frame payload starts.
3260 * Jump over vlan headers if already present,
3261 * helpful for QinQ too.
3263 eh = mtod(mp, struct ether_vlan_header *);
3264 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3265 etype = ntohs(eh->evl_proto);
3266 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3268 etype = ntohs(eh->evl_encap_proto);
3269 ehdrlen = ETHER_HDR_LEN;
3272 /* Set the ether header length */
3273 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3277 ip = (struct ip *)(mp->m_data + ehdrlen);
3278 ip_hlen = ip->ip_hl << 2;
3280 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3282 case ETHERTYPE_IPV6:
3283 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3284 ip_hlen = sizeof(struct ip6_hdr);
3285 /* XXX-BZ this will go badly in case of ext hdrs. */
3286 ipproto = ip6->ip6_nxt;
3287 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3294 vlan_macip_lens |= ip_hlen;
3295 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3299 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3300 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3304 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3305 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3310 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3311 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3319 /* Now copy bits into descriptor */
3320 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3321 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3322 TXD->seqnum_seed = htole32(0);
3323 TXD->mss_l4len_idx = htole32(0);
3325 tx_buffer->m_head = NULL;
3326 tx_buffer->eop_index = -1;
3328 /* We've consumed the first desc, adjust counters */
3329 if (++ctxd == adapter->num_tx_desc)
3331 txr->next_avail_desc = ctxd;
3337 /**********************************************************************
3339 * Setup work for hardware segmentation offload (TSO) on
3340 * adapters using advanced tx descriptors
3342 **********************************************************************/
3345 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3348 struct adapter *adapter = txr->adapter;
3349 struct ixgbe_adv_tx_context_desc *TXD;
3350 struct ixgbe_tx_buf *tx_buffer;
3352 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3353 u16 vtag = 0, eh_type;
3356 u32 type_tucmd_mlhl = 0;
3358 u32 mss_l4len_idx = 0, len;
3359 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3360 struct ether_vlan_header *eh;
3362 struct ip6_hdr *ip6;
3371 * Determine where frame payload starts.
3372 * Jump over vlan headers if already present
3374 eh = mtod(mp, struct ether_vlan_header *);
3375 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3376 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3377 eh_type = eh->evl_proto;
3379 ehdrlen = ETHER_HDR_LEN;
3380 eh_type = eh->evl_encap_proto;
3383 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3384 len = ehdrlen + sizeof(struct tcphdr);
3385 switch (ntohs(eh_type)) {
3387 case ETHERTYPE_IPV6:
3388 if (mp->m_len < len + sizeof(struct ip6_hdr))
3390 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3391 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3392 if (ip6->ip6_nxt != IPPROTO_TCP)
3394 ip_hlen = sizeof(struct ip6_hdr);
3395 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3396 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3397 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3402 if (mp->m_len < len + sizeof(struct ip))
3404 ip = (struct ip *)(mp->m_data + ehdrlen);
3405 if (ip->ip_p != IPPROTO_TCP)
3408 ip_hlen = ip->ip_hl << 2;
3409 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3410 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3411 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3412 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3413 /* Tell transmit desc to also do IPv4 checksum. */
3414 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3418 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3419 __func__, ntohs(eh_type));
3423 ctxd = txr->next_avail_desc;
3424 tx_buffer = &txr->tx_buffers[ctxd];
3425 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3427 tcp_hlen = th->th_off << 2;
3429 /* This is used in the transmit desc in encap */
3430 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3432 /* VLAN MACLEN IPLEN */
3434 if (mp->m_flags & M_VLANTAG) {
3435 vtag = htole16(mp->m_pkthdr.ether_vtag);
3436 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3439 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3440 vlan_macip_lens |= ip_hlen;
3441 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3444 /* ADV DTYPE TUCMD */
3445 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3446 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3447 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3450 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3451 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3452 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3454 TXD->seqnum_seed = htole32(0);
3455 tx_buffer->m_head = NULL;
3456 tx_buffer->eop_index = -1;
3458 if (++ctxd == adapter->num_tx_desc)
3462 txr->next_avail_desc = ctxd;
3469 ** This routine parses packet headers so that Flow
3470 ** Director can make a hashed filter table entry
3471 ** allowing traffic flows to be identified and kept
3472 ** on the same cpu. This would be a performance
3473 ** hit, but we only do it at IXGBE_FDIR_RATE of
3477 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3479 struct adapter *adapter = txr->adapter;
3480 struct ix_queue *que;
3484 struct ether_vlan_header *eh;
3485 union ixgbe_atr_hash_dword input = {.dword = 0};
3486 union ixgbe_atr_hash_dword common = {.dword = 0};
3487 int ehdrlen, ip_hlen;
3490 eh = mtod(mp, struct ether_vlan_header *);
3491 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3492 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3493 etype = eh->evl_proto;
3495 ehdrlen = ETHER_HDR_LEN;
3496 etype = eh->evl_encap_proto;
3499 /* Only handling IPv4 */
3500 if (etype != htons(ETHERTYPE_IP))
3503 ip = (struct ip *)(mp->m_data + ehdrlen);
3504 ip_hlen = ip->ip_hl << 2;
3506 /* check if we're UDP or TCP */
3509 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3510 /* src and dst are inverted */
3511 common.port.dst ^= th->th_sport;
3512 common.port.src ^= th->th_dport;
3513 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3516 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3517 /* src and dst are inverted */
3518 common.port.dst ^= uh->uh_sport;
3519 common.port.src ^= uh->uh_dport;
3520 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3526 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3527 if (mp->m_pkthdr.ether_vtag)
3528 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3530 common.flex_bytes ^= etype;
3531 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3533 que = &adapter->queues[txr->me];
3535 ** This assumes the Rx queue and Tx
3536 ** queue are bound to the same CPU
3538 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3539 input, common, que->msix);
3541 #endif /* IXGBE_FDIR */
3543 /**********************************************************************
3545 * Examine each tx_buffer in the used queue. If the hardware is done
3546 * processing the packet then free associated resources. The
3547 * tx_buffer is put back on the free queue.
3549 **********************************************************************/
3551 ixgbe_txeof(struct tx_ring *txr)
3553 struct adapter *adapter = txr->adapter;
3554 struct ifnet *ifp = adapter->ifp;
3555 u32 first, last, done, processed;
3556 struct ixgbe_tx_buf *tx_buffer;
3557 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3559 KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3562 if (ifp->if_capenable & IFCAP_NETMAP) {
3563 struct netmap_adapter *na = NA(ifp);
3564 struct netmap_kring *kring = &na->tx_rings[txr->me];
3566 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3568 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3569 BUS_DMASYNC_POSTREAD);
3571 * In netmap mode, all the work is done in the context
3572 * of the client thread. Interrupt handlers only wake up
3573 * clients, which may be sleeping on individual rings
3574 * or on a global resource for all rings.
3575 * To implement tx interrupt mitigation, we wake up the client
3576 * thread roughly every half ring, even if the NIC interrupts
3577 * more frequently. This is implemented as follows:
3578 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3579 * the slot that should wake up the thread (nkr_num_slots
3580 * means the user thread should not be woken up);
3581 * - the driver ignores tx interrupts unless netmap_mitigate=0
3582 * or the slot has the DD bit set.
3584 * When the driver has separate locks, we need to
3585 * release and re-acquire txlock to avoid deadlocks.
3586 * XXX see if we can find a better way.
3588 if (!netmap_mitigate ||
3589 (kring->nr_kflags < kring->nkr_num_slots &&
3590 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3591 kring->nr_kflags = kring->nkr_num_slots;
3592 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3593 IXGBE_TX_UNLOCK(txr);
3594 IXGBE_CORE_LOCK(adapter);
3595 selwakeuppri(&na->tx_si, PI_NET);
3596 IXGBE_CORE_UNLOCK(adapter);
3601 #endif /* DEV_NETMAP */
3603 if (txr->tx_avail == adapter->num_tx_desc) {
3604 txr->queue_status = IXGBE_QUEUE_IDLE;
3609 first = txr->next_to_clean;
3610 tx_buffer = &txr->tx_buffers[first];
3611 /* For cleanup we just use legacy struct */
3612 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3613 last = tx_buffer->eop_index;
3616 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3619 ** Get the index of the first descriptor
3620 ** BEYOND the EOP and call that 'done'.
3621 ** I do this so the comparison in the
3622 ** inner while loop below can be simple
3624 if (++last == adapter->num_tx_desc) last = 0;
3627 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3628 BUS_DMASYNC_POSTREAD);
3630 ** Only the EOP descriptor of a packet now has the DD
3631 ** bit set, this is what we look for...
3633 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3634 /* We clean the range of the packet */
3635 while (first != done) {
3636 tx_desc->upper.data = 0;
3637 tx_desc->lower.data = 0;
3638 tx_desc->buffer_addr = 0;
3642 if (tx_buffer->m_head) {
3644 tx_buffer->m_head->m_pkthdr.len;
3645 bus_dmamap_sync(txr->txtag,
3647 BUS_DMASYNC_POSTWRITE);
3648 bus_dmamap_unload(txr->txtag,
3650 m_freem(tx_buffer->m_head);
3651 tx_buffer->m_head = NULL;
3652 tx_buffer->map = NULL;
3654 tx_buffer->eop_index = -1;
3655 txr->watchdog_time = ticks;
3657 if (++first == adapter->num_tx_desc)
3660 tx_buffer = &txr->tx_buffers[first];
3662 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3666 /* See if there is more work now */
3667 last = tx_buffer->eop_index;
3670 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3671 /* Get next done point */
3672 if (++last == adapter->num_tx_desc) last = 0;
3677 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3678 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3680 txr->next_to_clean = first;
3683 ** Watchdog calculation, we know there's
3684 ** work outstanding or the first return
3685 ** would have been taken, so none processed
3686 ** for too long indicates a hang.
3688 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3689 txr->queue_status = IXGBE_QUEUE_HUNG;
3691 /* With a minimum free clear the depleted state bit. */
3692 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3693 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3695 if (txr->tx_avail == adapter->num_tx_desc) {
3696 txr->queue_status = IXGBE_QUEUE_IDLE;
3703 /*********************************************************************
3705 * Refresh mbuf buffers for RX descriptor rings
3706 * - now keeps its own state so discards due to resource
3707 * exhaustion are unnecessary, if an mbuf cannot be obtained
3708 * it just returns, keeping its placeholder, thus it can simply
3709 * be recalled to try again.
3711 **********************************************************************/
3713 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3715 struct adapter *adapter = rxr->adapter;
3716 bus_dma_segment_t hseg[1];
3717 bus_dma_segment_t pseg[1];
3718 struct ixgbe_rx_buf *rxbuf;
3719 struct mbuf *mh, *mp;
3720 int i, j, nsegs, error;
3721 bool refreshed = FALSE;
3723 i = j = rxr->next_to_refresh;
3724 /* Control the loop with one beyond */
3725 if (++j == adapter->num_rx_desc)
3728 while (j != limit) {
3729 rxbuf = &rxr->rx_buffers[i];
3730 if (rxr->hdr_split == FALSE)
3733 if (rxbuf->m_head == NULL) {
3734 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3740 mh->m_pkthdr.len = mh->m_len = MHLEN;
3742 mh->m_flags |= M_PKTHDR;
3743 /* Get the memory mapping */
3744 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3745 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3747 kprintf("Refresh mbufs: hdr dmamap load"
3748 " failure - %d\n", error);
3750 rxbuf->m_head = NULL;
3754 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3755 BUS_DMASYNC_PREREAD);
3756 rxr->rx_base[i].read.hdr_addr =
3757 htole64(hseg[0].ds_addr);
3760 if (rxbuf->m_pack == NULL) {
3761 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3762 M_PKTHDR, adapter->rx_mbuf_sz);
3768 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3769 /* Get the memory mapping */
3770 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3771 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3773 kprintf("Refresh mbufs: payload dmamap load"
3774 " failure - %d\n", error);
3776 rxbuf->m_pack = NULL;
3780 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3781 BUS_DMASYNC_PREREAD);
3782 rxr->rx_base[i].read.pkt_addr =
3783 htole64(pseg[0].ds_addr);
3786 /* Next is precalculated */
3788 rxr->next_to_refresh = i;
3789 if (++j == adapter->num_rx_desc)
3793 if (refreshed) /* Update hardware tail index */
3794 IXGBE_WRITE_REG(&adapter->hw,
3795 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3799 /*********************************************************************
3801 * Allocate memory for rx_buffer structures. Since we use one
3802 * rx_buffer per received packet, the maximum number of rx_buffer's
3803 * that we'll need is equal to the number of receive descriptors
3804 * that we've allocated.
3806 **********************************************************************/
3808 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3810 struct adapter *adapter = rxr->adapter;
3811 device_t dev = adapter->dev;
3812 struct ixgbe_rx_buf *rxbuf;
3813 int i, bsize, error;
3815 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3816 if (!(rxr->rx_buffers =
3817 (struct ixgbe_rx_buf *) kmalloc(bsize,
3818 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3819 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3824 if ((error = bus_dma_tag_create(NULL, /* parent */
3825 1, 0, /* alignment, bounds */
3826 BUS_SPACE_MAXADDR, /* lowaddr */
3827 BUS_SPACE_MAXADDR, /* highaddr */
3828 NULL, NULL, /* filter, filterarg */
3829 MSIZE, /* maxsize */
3831 MSIZE, /* maxsegsize */
3834 device_printf(dev, "Unable to create RX DMA tag\n");
3838 if ((error = bus_dma_tag_create(NULL, /* parent */
3839 1, 0, /* alignment, bounds */
3840 BUS_SPACE_MAXADDR, /* lowaddr */
3841 BUS_SPACE_MAXADDR, /* highaddr */
3842 NULL, NULL, /* filter, filterarg */
3843 MJUM16BYTES, /* maxsize */
3845 MJUM16BYTES, /* maxsegsize */
3848 device_printf(dev, "Unable to create RX DMA tag\n");
3852 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3853 rxbuf = &rxr->rx_buffers[i];
3854 error = bus_dmamap_create(rxr->htag,
3855 BUS_DMA_NOWAIT, &rxbuf->hmap);
3857 device_printf(dev, "Unable to create RX head map\n");
3860 error = bus_dmamap_create(rxr->ptag,
3861 BUS_DMA_NOWAIT, &rxbuf->pmap);
3863 device_printf(dev, "Unable to create RX pkt map\n");
3871 /* Frees all, but can handle partial completion */
3872 ixgbe_free_receive_structures(adapter);
3877 ** Used to detect a descriptor that has
3878 ** been merged by Hardware RSC.
3881 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3883 return (le32toh(rx->wb.lower.lo_dword.data) &
3884 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3887 /*********************************************************************
3889 * Initialize Hardware RSC (LRO) feature on 82599
3890 * for an RX ring, this is toggled by the LRO capability
3891 * even though it is transparent to the stack.
3893 **********************************************************************/
3896 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3898 struct adapter *adapter = rxr->adapter;
3899 struct ixgbe_hw *hw = &adapter->hw;
3900 u32 rscctrl, rdrxctl;
3902 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3903 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3904 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3905 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3906 #endif /* DEV_NETMAP */
3907 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3908 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3909 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3911 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3912 rscctrl |= IXGBE_RSCCTL_RSCEN;
3914 ** Limit the total number of descriptors that
3915 ** can be combined, so it does not exceed 64K
3917 if (adapter->rx_mbuf_sz == MCLBYTES)
3918 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3919 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3920 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3921 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3922 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3923 else /* Using 16K cluster */
3924 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3926 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3928 /* Enable TCP header recognition */
3929 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3930 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3931 IXGBE_PSRTYPE_TCPHDR));
3933 /* Disable RSC for ACK packets */
3934 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3935 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3942 ixgbe_free_receive_ring(struct rx_ring *rxr)
3944 struct adapter *adapter;
3945 struct ixgbe_rx_buf *rxbuf;
3948 adapter = rxr->adapter;
3949 for (i = 0; i < adapter->num_rx_desc; i++) {
3950 rxbuf = &rxr->rx_buffers[i];
3951 if (rxbuf->m_head != NULL) {
3952 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3953 BUS_DMASYNC_POSTREAD);
3954 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3955 rxbuf->m_head->m_flags |= M_PKTHDR;
3956 m_freem(rxbuf->m_head);
3958 if (rxbuf->m_pack != NULL) {
3959 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3960 BUS_DMASYNC_POSTREAD);
3961 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3962 rxbuf->m_pack->m_flags |= M_PKTHDR;
3963 m_freem(rxbuf->m_pack);
3965 rxbuf->m_head = NULL;
3966 rxbuf->m_pack = NULL;
3971 /*********************************************************************
3973 * Initialize a receive ring and its buffers.
3975 **********************************************************************/
3977 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3979 struct adapter *adapter;
3982 struct ixgbe_rx_buf *rxbuf;
3983 bus_dma_segment_t pseg[1], hseg[1];
3985 struct lro_ctrl *lro = &rxr->lro;
3987 int rsize, nsegs, error = 0;
3989 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3990 struct netmap_slot *slot;
3991 #endif /* DEV_NETMAP */
3993 adapter = rxr->adapter;
3997 /* Clear the ring contents */
4000 /* same as in ixgbe_setup_transmit_ring() */
4001 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4002 #endif /* DEV_NETMAP */
4003 rsize = roundup2(adapter->num_rx_desc *
4004 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4005 bzero((void *)rxr->rx_base, rsize);
4007 /* Free current RX buffer structs and their mbufs */
4008 ixgbe_free_receive_ring(rxr);
4010 /* Configure header split? */
4011 if (ixgbe_header_split)
4012 rxr->hdr_split = TRUE;
4014 /* Now replenish the mbufs */
4015 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4016 struct mbuf *mh, *mp;
4018 rxbuf = &rxr->rx_buffers[j];
4021 * In netmap mode, fill the map and set the buffer
4022 * address in the NIC ring, considering the offset
4023 * between the netmap and NIC rings (see comment in
4024 * ixgbe_setup_transmit_ring() ). No need to allocate
4025 * an mbuf, so end the block with a continue;
4028 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4032 addr = PNMB(slot + sj, &paddr);
4033 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4034 /* Update descriptor */
4035 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4038 #endif /* DEV_NETMAP */
4040 ** Don't allocate mbufs if not
4041 ** doing header split, its wasteful
4043 if (rxr->hdr_split == FALSE)
4046 /* First the header */
4047 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4048 if (rxbuf->m_head == NULL) {
4052 m_adj(rxbuf->m_head, ETHER_ALIGN);
4054 mh->m_len = mh->m_pkthdr.len = MHLEN;
4055 mh->m_flags |= M_PKTHDR;
4056 /* Get the memory mapping */
4057 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4058 rxbuf->hmap, rxbuf->m_head, hseg, 1,
4059 &nsegs, BUS_DMA_NOWAIT);
4061 if (error != 0) /* Nothing elegant to do here */
4063 bus_dmamap_sync(rxr->htag,
4064 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4065 /* Update descriptor */
4066 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4069 /* Now the payload cluster */
4070 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4071 M_PKTHDR, adapter->rx_mbuf_sz);
4072 if (rxbuf->m_pack == NULL) {
4077 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4078 /* Get the memory mapping */
4079 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4080 rxbuf->pmap, mp, hseg, 1,
4081 &nsegs, BUS_DMA_NOWAIT);
4084 bus_dmamap_sync(rxr->ptag,
4085 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4086 /* Update descriptor */
4087 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4091 /* Setup our descriptor indices */
4092 rxr->next_to_check = 0;
4093 rxr->next_to_refresh = 0;
4094 rxr->lro_enabled = FALSE;
4095 rxr->rx_split_packets = 0;
4097 rxr->discard = FALSE;
4098 rxr->vtag_strip = FALSE;
4100 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4101 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4104 ** Now set up the LRO interface:
4105 ** 82598 uses software LRO, the
4106 ** 82599 and X540 use a hardware assist.
4109 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4110 (ifp->if_capenable & IFCAP_RXCSUM) &&
4111 (ifp->if_capenable & IFCAP_LRO))
4112 ixgbe_setup_hw_rsc(rxr);
4113 else if (ifp->if_capenable & IFCAP_LRO) {
4114 int err = tcp_lro_init(lro);
4116 device_printf(dev, "LRO Initialization failed!\n");
4119 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4120 rxr->lro_enabled = TRUE;
4121 lro->ifp = adapter->ifp;
4125 IXGBE_RX_UNLOCK(rxr);
4129 ixgbe_free_receive_ring(rxr);
4130 IXGBE_RX_UNLOCK(rxr);
4134 /*********************************************************************
4136 * Initialize all receive rings.
4138 **********************************************************************/
4140 ixgbe_setup_receive_structures(struct adapter *adapter)
4142 struct rx_ring *rxr = adapter->rx_rings;
4145 for (j = 0; j < adapter->num_queues; j++, rxr++)
4146 if (ixgbe_setup_receive_ring(rxr))
4152 * Free RX buffers allocated so far, we will only handle
4153 * the rings that completed, the failing case will have
4154 * cleaned up for itself. 'j' failed, so its the terminus.
4156 for (int i = 0; i < j; ++i) {
4157 rxr = &adapter->rx_rings[i];
4158 ixgbe_free_receive_ring(rxr);
4164 /*********************************************************************
4166 * Setup receive registers and features.
4168 **********************************************************************/
4169 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4171 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4174 ixgbe_initialize_receive_units(struct adapter *adapter)
4176 struct rx_ring *rxr = adapter->rx_rings;
4177 struct ixgbe_hw *hw = &adapter->hw;
4178 struct ifnet *ifp = adapter->ifp;
4179 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
4180 u32 reta, mrqc = 0, hlreg, random[10];
4184 * Make sure receives are disabled while
4185 * setting up the descriptor ring
4187 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4188 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4189 rxctrl & ~IXGBE_RXCTRL_RXEN);
4191 /* Enable broadcasts */
4192 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4193 fctrl |= IXGBE_FCTRL_BAM;
4194 fctrl |= IXGBE_FCTRL_DPF;
4195 fctrl |= IXGBE_FCTRL_PMCF;
4196 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4198 /* Set for Jumbo Frames? */
4199 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4200 if (ifp->if_mtu > ETHERMTU)
4201 hlreg |= IXGBE_HLREG0_JUMBOEN;
4203 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4205 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4206 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4207 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4209 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4210 #endif /* DEV_NETMAP */
4211 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4213 bufsz = (adapter->rx_mbuf_sz +
4214 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4216 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4217 u64 rdba = rxr->rxdma.dma_paddr;
4219 /* Setup the Base and Length of the Rx Descriptor Ring */
4220 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4221 (rdba & 0x00000000ffffffffULL));
4222 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4223 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4224 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4226 /* Set up the SRRCTL register */
4227 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4228 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4229 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4231 if (rxr->hdr_split) {
4232 /* Use a standard mbuf for the header */
4233 srrctl |= ((IXGBE_RX_HDR <<
4234 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4235 & IXGBE_SRRCTL_BSIZEHDR_MASK);
4236 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4238 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4239 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4241 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4242 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4243 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4246 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4247 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4248 IXGBE_PSRTYPE_UDPHDR |
4249 IXGBE_PSRTYPE_IPV4HDR |
4250 IXGBE_PSRTYPE_IPV6HDR;
4251 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4254 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4257 if (adapter->num_queues > 1) {
4261 /* set up random bits */
4262 karc4rand(&random, sizeof(random));
4264 /* Set up the redirection table */
4265 for (i = 0, j = 0; i < 128; i++, j++) {
4266 if (j == adapter->num_queues) j = 0;
4267 reta = (reta << 8) | (j * 0x11);
4269 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4272 /* Now fill our hash function seeds */
4273 for (int i = 0; i < 10; i++)
4274 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4276 /* Perform hash on these packet types */
4277 mrqc = IXGBE_MRQC_RSSEN
4278 | IXGBE_MRQC_RSS_FIELD_IPV4
4279 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4280 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4281 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4282 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4283 | IXGBE_MRQC_RSS_FIELD_IPV6
4284 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4285 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4286 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4287 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4289 /* RSS and RX IPP Checksum are mutually exclusive */
4290 rxcsum |= IXGBE_RXCSUM_PCSD;
4293 if (ifp->if_capenable & IFCAP_RXCSUM)
4294 rxcsum |= IXGBE_RXCSUM_PCSD;
4296 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4297 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4299 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4304 /*********************************************************************
4306 * Free all receive rings.
4308 **********************************************************************/
4310 ixgbe_free_receive_structures(struct adapter *adapter)
4312 struct rx_ring *rxr = adapter->rx_rings;
4314 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4316 struct lro_ctrl *lro = &rxr->lro;
4318 ixgbe_free_receive_buffers(rxr);
4319 /* Free LRO memory */
4323 /* Free the ring memory as well */
4324 ixgbe_dma_free(adapter, &rxr->rxdma);
4327 kfree(adapter->rx_rings, M_DEVBUF);
4331 /*********************************************************************
4333 * Free receive ring data structures
4335 **********************************************************************/
4337 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4339 struct adapter *adapter = rxr->adapter;
4340 struct ixgbe_rx_buf *rxbuf;
4342 INIT_DEBUGOUT("free_receive_structures: begin");
4344 /* Cleanup any existing buffers */
4345 if (rxr->rx_buffers != NULL) {
4346 for (int i = 0; i < adapter->num_rx_desc; i++) {
4347 rxbuf = &rxr->rx_buffers[i];
4348 if (rxbuf->m_head != NULL) {
4349 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4350 BUS_DMASYNC_POSTREAD);
4351 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4352 rxbuf->m_head->m_flags |= M_PKTHDR;
4353 m_freem(rxbuf->m_head);
4355 if (rxbuf->m_pack != NULL) {
4356 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4357 BUS_DMASYNC_POSTREAD);
4358 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4359 rxbuf->m_pack->m_flags |= M_PKTHDR;
4360 m_freem(rxbuf->m_pack);
4362 rxbuf->m_head = NULL;
4363 rxbuf->m_pack = NULL;
4364 if (rxbuf->hmap != NULL) {
4365 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4368 if (rxbuf->pmap != NULL) {
4369 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4373 if (rxr->rx_buffers != NULL) {
4374 kfree(rxr->rx_buffers, M_DEVBUF);
4375 rxr->rx_buffers = NULL;
4379 if (rxr->htag != NULL) {
4380 bus_dma_tag_destroy(rxr->htag);
4383 if (rxr->ptag != NULL) {
4384 bus_dma_tag_destroy(rxr->ptag);
4391 static __inline void
4392 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4396 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4397 * should be computed by hardware. Also it should not have VLAN tag in
4398 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4401 if (rxr->lro_enabled &&
4402 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4403 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4404 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4405 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4406 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4407 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4408 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4409 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4411 * Send to the stack if:
4412 ** - LRO not enabled, or
4413 ** - no LRO resources, or
4414 ** - lro enqueue fails
4416 if (rxr->lro.lro_cnt != 0)
4417 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4421 IXGBE_RX_UNLOCK(rxr);
4422 (*ifp->if_input)(ifp, m);
4426 static __inline void
4427 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4429 struct ixgbe_rx_buf *rbuf;
4431 rbuf = &rxr->rx_buffers[i];
4433 if (rbuf->fmp != NULL) {/* Partial chain ? */
4434 rbuf->fmp->m_flags |= M_PKTHDR;
4440 ** With advanced descriptors the writeback
4441 ** clobbers the buffer addrs, so its easier
4442 ** to just free the existing mbufs and take
4443 ** the normal refresh path to get new buffers
4447 m_free(rbuf->m_head);
4448 rbuf->m_head = NULL;
4452 m_free(rbuf->m_pack);
4453 rbuf->m_pack = NULL;
4460 /*********************************************************************
4462 * This routine executes in interrupt context. It replenishes
4463 * the mbufs in the descriptor and sends data which has been
4464 * dma'ed into host memory to upper layer.
4466 * We loop at most count times if count is > 0, or until done if
4469 * Return TRUE for more work, FALSE for all clean.
4470 *********************************************************************/
4472 ixgbe_rxeof(struct ix_queue *que, int count)
4474 struct adapter *adapter = que->adapter;
4475 struct rx_ring *rxr = que->rxr;
4476 struct ifnet *ifp = adapter->ifp;
4478 struct lro_ctrl *lro = &rxr->lro;
4479 struct lro_entry *queued;
4481 int i, nextp, processed = 0;
4483 union ixgbe_adv_rx_desc *cur;
4484 struct ixgbe_rx_buf *rbuf, *nbuf;
4489 if (ifp->if_capenable & IFCAP_NETMAP) {
4491 * Same as the txeof routine: only wakeup clients on intr.
4492 * NKR_PENDINTR in nr_kflags is used to implement interrupt
4493 * mitigation (ixgbe_rxsync() will not look for new packets
4494 * unless NKR_PENDINTR is set).
4496 struct netmap_adapter *na = NA(ifp);
4498 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4499 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4500 IXGBE_RX_UNLOCK(rxr);
4501 IXGBE_CORE_LOCK(adapter);
4502 selwakeuppri(&na->rx_si, PI_NET);
4503 IXGBE_CORE_UNLOCK(adapter);
4506 #endif /* DEV_NETMAP */
4507 for (i = rxr->next_to_check; count != 0;) {
4508 struct mbuf *sendmp, *mh, *mp;
4510 u16 hlen, plen, hdr;
4516 /* Sync the ring. */
4517 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4518 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4520 cur = &rxr->rx_base[i];
4521 staterr = le32toh(cur->wb.upper.status_error);
4523 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4525 if ((ifp->if_flags & IFF_RUNNING) == 0)
4532 cur->wb.upper.status_error = 0;
4533 rbuf = &rxr->rx_buffers[i];
4537 plen = le16toh(cur->wb.upper.length);
4538 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4539 IXGBE_RXDADV_PKTTYPE_MASK;
4540 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4541 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4543 /* Process vlan info */
4545 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4546 vtag = le16toh(cur->wb.upper.vlan);
4549 /* Make sure bad packets are discarded */
4550 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4553 rxr->rx_discarded++;
4555 rxr->discard = FALSE;
4557 rxr->discard = TRUE;
4558 ixgbe_rx_discard(rxr, i);
4563 ** On 82599 which supports a hardware
4564 ** LRO (called HW RSC), packets need
4565 ** not be fragmented across sequential
4566 ** descriptors, rather the next descriptor
4567 ** is indicated in bits of the descriptor.
4568 ** This also means that we might proceses
4569 ** more than one packet at a time, something
4570 ** that has never been true before, it
4571 ** required eliminating global chain pointers
4572 ** in favor of what we are doing here. -jfv
4576 ** Figure out the next descriptor
4579 if (rxr->hw_rsc == TRUE) {
4580 rsc = ixgbe_rsc_count(cur);
4581 rxr->rsc_num += (rsc - 1);
4583 if (rsc) { /* Get hardware index */
4585 IXGBE_RXDADV_NEXTP_MASK) >>
4586 IXGBE_RXDADV_NEXTP_SHIFT);
4587 } else { /* Just sequential */
4589 if (nextp == adapter->num_rx_desc)
4592 nbuf = &rxr->rx_buffers[nextp];
4596 ** The header mbuf is ONLY used when header
4597 ** split is enabled, otherwise we get normal
4598 ** behavior, ie, both header and payload
4599 ** are DMA'd into the payload buffer.
4601 ** Rather than using the fmp/lmp global pointers
4602 ** we now keep the head of a packet chain in the
4603 ** buffer struct and pass this along from one
4604 ** descriptor to the next, until we get EOP.
4606 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4607 /* This must be an initial descriptor */
4608 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4609 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4610 if (hlen > IXGBE_RX_HDR)
4611 hlen = IXGBE_RX_HDR;
4613 mh->m_flags |= M_PKTHDR;
4615 mh->m_pkthdr.len = mh->m_len;
4616 /* Null buf pointer so it is refreshed */
4617 rbuf->m_head = NULL;
4619 ** Check the payload length, this
4620 ** could be zero if its a small
4626 mp->m_flags &= ~M_PKTHDR;
4628 mh->m_pkthdr.len += mp->m_len;
4629 /* Null buf pointer so it is refreshed */
4630 rbuf->m_pack = NULL;
4631 rxr->rx_split_packets++;
4634 ** Now create the forward
4635 ** chain so when complete
4639 /* stash the chain head */
4641 /* Make forward chain */
4643 mp->m_next = nbuf->m_pack;
4645 mh->m_next = nbuf->m_pack;
4647 /* Singlet, prepare to send */
4649 /* If hardware handled vtag */
4652 sendmp->m_pkthdr.ether_vtag = vtag;
4653 sendmp->m_flags |= M_VLANTAG;
4659 ** Either no header split, or a
4660 ** secondary piece of a fragmented
4665 ** See if there is a stored head
4666 ** that determines what we are
4669 rbuf->m_pack = rbuf->fmp = NULL;
4671 if (sendmp != NULL) { /* secondary frag */
4672 mp->m_flags &= ~M_PKTHDR;
4673 sendmp->m_pkthdr.len += mp->m_len;
4675 /* first desc of a non-ps chain */
4677 sendmp->m_flags |= M_PKTHDR;
4678 sendmp->m_pkthdr.len = mp->m_len;
4680 if (staterr & IXGBE_RXD_STAT_VP) {
4681 sendmp->m_pkthdr.ether_vtag = vtag;
4682 sendmp->m_flags |= M_VLANTAG;
4686 /* Pass the head pointer on */
4690 mp->m_next = nbuf->m_pack;
4694 /* Sending this frame? */
4696 sendmp->m_pkthdr.rcvif = ifp;
4699 /* capture data for AIM */
4700 rxr->bytes += sendmp->m_pkthdr.len;
4701 rxr->rx_bytes += sendmp->m_pkthdr.len;
4702 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4703 ixgbe_rx_checksum(staterr, sendmp, ptype);
4704 #if 0 /* __FreeBSD_version >= 800000 */
4705 sendmp->m_pkthdr.flowid = que->msix;
4706 sendmp->m_flags |= M_FLOWID;
4710 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4711 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4713 /* Advance our pointers to the next descriptor. */
4714 if (++i == adapter->num_rx_desc)
4717 /* Now send to the stack or do LRO */
4718 if (sendmp != NULL) {
4719 rxr->next_to_check = i;
4720 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4721 i = rxr->next_to_check;
4724 /* Every 8 descriptors we go to refresh mbufs */
4725 if (processed == 8) {
4726 ixgbe_refresh_mbufs(rxr, i);
4731 /* Refresh any remaining buf structs */
4732 if (ixgbe_rx_unrefreshed(rxr))
4733 ixgbe_refresh_mbufs(rxr, i);
4735 rxr->next_to_check = i;
4738 * Flush any outstanding LRO work
4741 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4742 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4743 tcp_lro_flush(lro, queued);
4747 IXGBE_RX_UNLOCK(rxr);
4750 ** We still have cleaning to do?
4751 ** Schedule another interrupt if so.
4753 if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4754 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4762 /*********************************************************************
4764 * Verify that the hardware indicated that the checksum is valid.
4765 * Inform the stack about the status of checksum so that stack
4766 * doesn't spend time verifying the checksum.
4768 *********************************************************************/
4770 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4772 u16 status = (u16) staterr;
4773 u8 errors = (u8) (staterr >> 24);
4776 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4777 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4780 if (status & IXGBE_RXD_STAT_IPCS) {
4781 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4782 /* IP Checksum Good */
4783 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4784 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4787 mp->m_pkthdr.csum_flags = 0;
4789 if (status & IXGBE_RXD_STAT_L4CS) {
4790 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4793 type = CSUM_SCTP_VALID;
4795 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4796 mp->m_pkthdr.csum_flags |= type;
4798 mp->m_pkthdr.csum_data = htons(0xffff);
4806 ** This routine is run via an vlan config EVENT,
4807 ** it enables us to use the HW Filter table since
4808 ** we can get the vlan id. This just creates the
4809 ** entry in the soft version of the VFTA, init will
4810 ** repopulate the real table.
4813 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4815 struct adapter *adapter = ifp->if_softc;
4818 if (ifp->if_softc != arg) /* Not our event */
4821 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4824 IXGBE_CORE_LOCK(adapter);
4825 index = (vtag >> 5) & 0x7F;
4827 adapter->shadow_vfta[index] |= (1 << bit);
4828 ++adapter->num_vlans;
4829 ixgbe_init_locked(adapter);
4830 IXGBE_CORE_UNLOCK(adapter);
4834 ** This routine is run via an vlan
4835 ** unconfig EVENT, remove our entry
4836 ** in the soft vfta.
4839 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4841 struct adapter *adapter = ifp->if_softc;
4844 if (ifp->if_softc != arg)
4847 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4850 IXGBE_CORE_LOCK(adapter);
4851 index = (vtag >> 5) & 0x7F;
4853 adapter->shadow_vfta[index] &= ~(1 << bit);
4854 --adapter->num_vlans;
4855 /* Re-init to load the changes */
4856 ixgbe_init_locked(adapter);
4857 IXGBE_CORE_UNLOCK(adapter);
4861 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4864 struct ifnet *ifp = adapter->ifp;
4865 struct ixgbe_hw *hw = &adapter->hw;
4866 struct rx_ring *rxr;
4870 ** We get here thru init_locked, meaning
4871 ** a soft reset, this has already cleared
4872 ** the VFTA and other state, so if there
4873 ** have been no vlan's registered do nothing.
4875 if (adapter->num_vlans == 0)
4879 ** A soft reset zero's out the VFTA, so
4880 ** we need to repopulate it now.
4882 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4883 if (adapter->shadow_vfta[i] != 0)
4884 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4885 adapter->shadow_vfta[i]);
4887 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4888 /* Enable the Filter Table if enabled */
4889 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4890 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4891 ctrl |= IXGBE_VLNCTRL_VFE;
4893 if (hw->mac.type == ixgbe_mac_82598EB)
4894 ctrl |= IXGBE_VLNCTRL_VME;
4895 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4897 /* Setup the queues for vlans */
4898 for (int i = 0; i < adapter->num_queues; i++) {
4899 rxr = &adapter->rx_rings[i];
4900 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4901 if (hw->mac.type != ixgbe_mac_82598EB) {
4902 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4903 ctrl |= IXGBE_RXDCTL_VME;
4904 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4906 rxr->vtag_strip = TRUE;
4912 ixgbe_enable_intr(struct adapter *adapter)
4914 struct ixgbe_hw *hw = &adapter->hw;
4915 struct ix_queue *que = adapter->queues;
4916 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4919 /* Enable Fan Failure detection */
4920 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4921 mask |= IXGBE_EIMS_GPI_SDP1;
4923 mask |= IXGBE_EIMS_ECC;
4924 mask |= IXGBE_EIMS_GPI_SDP0;
4925 mask |= IXGBE_EIMS_GPI_SDP1;
4926 mask |= IXGBE_EIMS_GPI_SDP2;
4928 mask |= IXGBE_EIMS_FLOW_DIR;
4932 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4934 /* With RSS we use auto clear */
4935 if (adapter->msix_mem) {
4936 mask = IXGBE_EIMS_ENABLE_MASK;
4937 /* Don't autoclear Link */
4938 mask &= ~IXGBE_EIMS_OTHER;
4939 mask &= ~IXGBE_EIMS_LSC;
4940 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4944 ** Now enable all queues, this is done separately to
4945 ** allow for handling the extended (beyond 32) MSIX
4946 ** vectors that can be used by 82599
4948 for (int i = 0; i < adapter->num_queues; i++, que++)
4949 ixgbe_enable_queue(adapter, que->msix);
4951 IXGBE_WRITE_FLUSH(hw);
4957 ixgbe_disable_intr(struct adapter *adapter)
4959 if (adapter->msix_mem)
4960 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4961 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4962 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4964 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4965 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4966 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4968 IXGBE_WRITE_FLUSH(&adapter->hw);
4973 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4977 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4984 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4986 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4993 ** Setup the correct IVAR register for a particular MSIX interrupt
4994 ** (yes this is all very magic and confusing :)
4995 ** - entry is the register array entry
4996 ** - vector is the MSIX vector for this queue
4997 ** - type is RX/TX/MISC
5000 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5002 struct ixgbe_hw *hw = &adapter->hw;
5005 vector |= IXGBE_IVAR_ALLOC_VAL;
5007 switch (hw->mac.type) {
5009 case ixgbe_mac_82598EB:
5011 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5013 entry += (type * 64);
5014 index = (entry >> 2) & 0x1F;
5015 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5016 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5017 ivar |= (vector << (8 * (entry & 0x3)));
5018 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5021 case ixgbe_mac_82599EB:
5022 case ixgbe_mac_X540:
5023 if (type == -1) { /* MISC IVAR */
5024 index = (entry & 1) * 8;
5025 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5026 ivar &= ~(0xFF << index);
5027 ivar |= (vector << index);
5028 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5029 } else { /* RX/TX IVARS */
5030 index = (16 * (entry & 1)) + (8 * type);
5031 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5032 ivar &= ~(0xFF << index);
5033 ivar |= (vector << index);
5034 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5043 ixgbe_configure_ivars(struct adapter *adapter)
5045 struct ix_queue *que = adapter->queues;
5048 if (ixgbe_max_interrupt_rate > 0)
5049 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5053 for (int i = 0; i < adapter->num_queues; i++, que++) {
5054 /* First the RX queue entry */
5055 ixgbe_set_ivar(adapter, i, que->msix, 0);
5056 /* ... and the TX */
5057 ixgbe_set_ivar(adapter, i, que->msix, 1);
5058 /* Set an Initial EITR value */
5059 IXGBE_WRITE_REG(&adapter->hw,
5060 IXGBE_EITR(que->msix), newitr);
5063 /* For the Link interrupt */
5064 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5068 ** ixgbe_sfp_probe - called in the local timer to
5069 ** determine if a port had optics inserted.
5071 static bool ixgbe_sfp_probe(struct adapter *adapter)
5073 struct ixgbe_hw *hw = &adapter->hw;
5074 device_t dev = adapter->dev;
5075 bool result = FALSE;
5077 if ((hw->phy.type == ixgbe_phy_nl) &&
5078 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5079 s32 ret = hw->phy.ops.identify_sfp(hw);
5082 ret = hw->phy.ops.reset(hw);
5083 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5084 device_printf(dev,"Unsupported SFP+ module detected!");
5085 kprintf(" Reload driver with supported module.\n");
5086 adapter->sfp_probe = FALSE;
5089 device_printf(dev,"SFP+ module detected!\n");
5090 /* We now have supported optics */
5091 adapter->sfp_probe = FALSE;
5092 /* Set the optics type so system reports correctly */
5093 ixgbe_setup_optics(adapter);
5101 ** Tasklet handler for MSIX Link interrupts
5102 ** - do outside interrupt since it might sleep
5105 ixgbe_handle_link(void *context, int pending)
5107 struct adapter *adapter = context;
5109 ixgbe_check_link(&adapter->hw,
5110 &adapter->link_speed, &adapter->link_up, 0);
5111 ixgbe_update_link_status(adapter);
5115 ** Tasklet for handling SFP module interrupts
5118 ixgbe_handle_mod(void *context, int pending)
5120 struct adapter *adapter = context;
5121 struct ixgbe_hw *hw = &adapter->hw;
5122 device_t dev = adapter->dev;
5125 err = hw->phy.ops.identify_sfp(hw);
5126 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5128 "Unsupported SFP+ module type was detected.\n");
5131 err = hw->mac.ops.setup_sfp(hw);
5132 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5134 "Setup failure - unsupported SFP+ module type.\n");
5137 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5143 ** Tasklet for handling MSF (multispeed fiber) interrupts
5146 ixgbe_handle_msf(void *context, int pending)
5148 struct adapter *adapter = context;
5149 struct ixgbe_hw *hw = &adapter->hw;
5153 autoneg = hw->phy.autoneg_advertised;
5154 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5155 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5156 if (hw->mac.ops.setup_link)
5157 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5163 ** Tasklet for reinitializing the Flow Director filter table
5166 ixgbe_reinit_fdir(void *context, int pending)
5168 struct adapter *adapter = context;
5169 struct ifnet *ifp = adapter->ifp;
5171 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5173 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5174 adapter->fdir_reinit = 0;
5175 /* Restart the interface */
5176 ifp->if_drv_flags |= IFF_DRV_RUNNING;
5181 /**********************************************************************
5183 * Update the board statistics counters.
5185 **********************************************************************/
5187 ixgbe_update_stats_counters(struct adapter *adapter)
5189 struct ifnet *ifp = adapter->ifp;
5190 struct ixgbe_hw *hw = &adapter->hw;
5191 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5192 u64 total_missed_rx = 0;
5194 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5195 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5196 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5197 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5199 for (int i = 0; i < 8; i++) {
5201 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5202 /* missed_rx tallies misses for the gprc workaround */
5204 /* global total per queue */
5205 adapter->stats.mpc[i] += mp;
5206 /* Running comprehensive total for stats display */
5207 total_missed_rx += adapter->stats.mpc[i];
5208 if (hw->mac.type == ixgbe_mac_82598EB)
5209 adapter->stats.rnbc[i] +=
5210 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5211 adapter->stats.pxontxc[i] +=
5212 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5213 adapter->stats.pxonrxc[i] +=
5214 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5215 adapter->stats.pxofftxc[i] +=
5216 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5217 adapter->stats.pxoffrxc[i] +=
5218 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5219 adapter->stats.pxon2offc[i] +=
5220 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5222 for (int i = 0; i < 16; i++) {
5223 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5224 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5225 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5226 adapter->stats.qbrc[i] +=
5227 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5228 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5229 adapter->stats.qbtc[i] +=
5230 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5231 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5233 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5234 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5235 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5237 /* Hardware workaround, gprc counts missed packets */
5238 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5239 adapter->stats.gprc -= missed_rx;
5241 if (hw->mac.type != ixgbe_mac_82598EB) {
5242 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5243 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5244 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5245 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5246 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5247 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5248 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5249 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5251 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5252 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5253 /* 82598 only has a counter in the high register */
5254 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5255 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5256 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5260 * Workaround: mprc hardware is incorrectly counting
5261 * broadcasts, so for now we subtract those.
5263 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5264 adapter->stats.bprc += bprc;
5265 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5266 if (hw->mac.type == ixgbe_mac_82598EB)
5267 adapter->stats.mprc -= bprc;
5269 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5270 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5271 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5272 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5273 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5274 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5276 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5277 adapter->stats.lxontxc += lxon;
5278 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5279 adapter->stats.lxofftxc += lxoff;
5280 total = lxon + lxoff;
5282 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5283 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5284 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5285 adapter->stats.gptc -= total;
5286 adapter->stats.mptc -= total;
5287 adapter->stats.ptc64 -= total;
5288 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5290 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5291 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5292 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5293 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5294 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5295 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5296 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5297 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5298 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5299 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5300 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5301 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5302 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5303 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5304 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5305 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5306 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5307 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5308 /* Only read FCOE on 82599 */
5309 if (hw->mac.type != ixgbe_mac_82598EB) {
5310 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5311 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5312 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5313 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5314 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5317 /* Fill out the OS statistics structure */
5318 ifp->if_ipackets = adapter->stats.gprc;
5319 ifp->if_opackets = adapter->stats.gptc;
5320 ifp->if_ibytes = adapter->stats.gorc;
5321 ifp->if_obytes = adapter->stats.gotc;
5322 ifp->if_imcasts = adapter->stats.mprc;
5323 ifp->if_collisions = 0;
5326 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5327 adapter->stats.rlec;
5330 /** ixgbe_sysctl_tdh_handler - Handler function
5331 * Retrieves the TDH value from the hardware
5334 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5338 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5341 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5342 error = sysctl_handle_int(oidp, &val, 0, req);
5343 if (error || !req->newptr)
5348 /** ixgbe_sysctl_tdt_handler - Handler function
5349 * Retrieves the TDT value from the hardware
5352 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5356 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5359 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5360 error = sysctl_handle_int(oidp, &val, 0, req);
5361 if (error || !req->newptr)
5366 /** ixgbe_sysctl_rdh_handler - Handler function
5367 * Retrieves the RDH value from the hardware
5370 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5374 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5377 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5378 error = sysctl_handle_int(oidp, &val, 0, req);
5379 if (error || !req->newptr)
5384 /** ixgbe_sysctl_rdt_handler - Handler function
5385 * Retrieves the RDT value from the hardware
5388 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5392 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5395 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5396 error = sysctl_handle_int(oidp, &val, 0, req);
5397 if (error || !req->newptr)
5403 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5406 struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5407 unsigned int reg, usec, rate;
5409 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5410 usec = ((reg & 0x0FF8) >> 3);
5412 rate = 500000 / usec;
5415 error = sysctl_handle_int(oidp, &rate, 0, req);
5416 if (error || !req->newptr)
5418 reg &= ~0xfff; /* default, no limitation */
5419 ixgbe_max_interrupt_rate = 0;
5420 if (rate > 0 && rate < 500000) {
5423 ixgbe_max_interrupt_rate = rate;
5424 reg |= ((4000000/rate) & 0xff8 );
5426 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5431 * Add sysctl variables, one per statistic, to the system.
5434 ixgbe_add_hw_stats(struct adapter *adapter)
5436 struct tx_ring *txr = adapter->tx_rings;
5437 struct rx_ring *rxr = adapter->rx_rings;
5439 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5440 struct sysctl_oid *tree = adapter->sysctl_tree;
5441 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5442 struct ixgbe_hw_stats *stats = &adapter->stats;
5444 struct sysctl_oid *stat_node, *queue_node;
5445 struct sysctl_oid_list *stat_list, *queue_list;
5447 #define QUEUE_NAME_LEN 32
5448 char namebuf[QUEUE_NAME_LEN];
5450 /* Driver Statistics */
5451 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5452 CTLFLAG_RD, &adapter->dropped_pkts,
5453 "Driver dropped packets");
5454 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5455 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5456 "m_defrag() failed");
5457 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5458 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5459 "Driver tx dma failure in xmit");
5460 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5461 CTLFLAG_RD, &adapter->watchdog_events,
5462 "Watchdog timeouts");
5464 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5465 CTLFLAG_RD, &adapter->tso_tx,
5468 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5469 CTLFLAG_RD, &adapter->link_irq,
5470 "Link MSIX IRQ Handled");
5472 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5473 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5474 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5475 CTLFLAG_RD, NULL, "Queue Name");
5476 queue_list = SYSCTL_CHILDREN(queue_node);
5478 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5479 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5480 sizeof(&adapter->queues[i]),
5481 ixgbe_sysctl_interrupt_rate_handler, "IU",
5483 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5484 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5485 "irqs on this queue");
5486 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5487 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5488 ixgbe_sysctl_tdh_handler, "IU",
5489 "Transmit Descriptor Head");
5490 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5491 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5492 ixgbe_sysctl_tdt_handler, "IU",
5493 "Transmit Descriptor Tail");
5494 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5495 CTLFLAG_RD, &txr->no_desc_avail, 0,
5496 "Queue No Descriptor Available");
5497 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5498 CTLFLAG_RD, &txr->total_packets, 0,
5499 "Queue Packets Transmitted");
5502 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5503 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5504 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5505 CTLFLAG_RD, NULL, "Queue Name");
5506 queue_list = SYSCTL_CHILDREN(queue_node);
5509 struct lro_ctrl *lro = &rxr->lro;
5512 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5513 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5514 CTLFLAG_RD, NULL, "Queue Name");
5515 queue_list = SYSCTL_CHILDREN(queue_node);
5517 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5518 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5519 ixgbe_sysctl_rdh_handler, "IU",
5520 "Receive Descriptor Head");
5521 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5522 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5523 ixgbe_sysctl_rdt_handler, "IU",
5524 "Receive Descriptor Tail");
5525 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5526 CTLFLAG_RD, &rxr->rx_packets, 0,
5527 "Queue Packets Received");
5528 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5529 CTLFLAG_RD, &rxr->rx_bytes, 0,
5530 "Queue Bytes Received");
5532 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5533 CTLFLAG_RD, &lro->lro_queued, 0,
5535 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5536 CTLFLAG_RD, &lro->lro_flushed, 0,
5541 /* MAC stats get the own sub node */
5543 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5544 CTLFLAG_RD, NULL, "MAC Statistics");
5545 stat_list = SYSCTL_CHILDREN(stat_node);
5547 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5548 CTLFLAG_RD, &stats->crcerrs, 0,
5550 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5551 CTLFLAG_RD, &stats->illerrc, 0,
5552 "Illegal Byte Errors");
5553 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5554 CTLFLAG_RD, &stats->errbc, 0,
5556 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5557 CTLFLAG_RD, &stats->mspdc, 0,
5558 "MAC Short Packets Discarded");
5559 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5560 CTLFLAG_RD, &stats->mlfc, 0,
5561 "MAC Local Faults");
5562 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5563 CTLFLAG_RD, &stats->mrfc, 0,
5564 "MAC Remote Faults");
5565 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5566 CTLFLAG_RD, &stats->rlec, 0,
5567 "Receive Length Errors");
5568 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5569 CTLFLAG_RD, &stats->lxontxc, 0,
5570 "Link XON Transmitted");
5571 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5572 CTLFLAG_RD, &stats->lxonrxc, 0,
5573 "Link XON Received");
5574 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5575 CTLFLAG_RD, &stats->lxofftxc, 0,
5576 "Link XOFF Transmitted");
5577 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5578 CTLFLAG_RD, &stats->lxoffrxc, 0,
5579 "Link XOFF Received");
5581 /* Packet Reception Stats */
5582 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5583 CTLFLAG_RD, &stats->tor, 0,
5584 "Total Octets Received");
5585 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5586 CTLFLAG_RD, &stats->gorc, 0,
5587 "Good Octets Received");
5588 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5589 CTLFLAG_RD, &stats->tpr, 0,
5590 "Total Packets Received");
5591 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5592 CTLFLAG_RD, &stats->gprc, 0,
5593 "Good Packets Received");
5594 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5595 CTLFLAG_RD, &stats->mprc, 0,
5596 "Multicast Packets Received");
5597 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5598 CTLFLAG_RD, &stats->bprc, 0,
5599 "Broadcast Packets Received");
5600 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5601 CTLFLAG_RD, &stats->prc64, 0,
5602 "64 byte frames received ");
5603 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5604 CTLFLAG_RD, &stats->prc127, 0,
5605 "65-127 byte frames received");
5606 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5607 CTLFLAG_RD, &stats->prc255, 0,
5608 "128-255 byte frames received");
5609 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5610 CTLFLAG_RD, &stats->prc511, 0,
5611 "256-511 byte frames received");
5612 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5613 CTLFLAG_RD, &stats->prc1023, 0,
5614 "512-1023 byte frames received");
5615 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5616 CTLFLAG_RD, &stats->prc1522, 0,
5617 "1023-1522 byte frames received");
5618 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5619 CTLFLAG_RD, &stats->ruc, 0,
5620 "Receive Undersized");
5621 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5622 CTLFLAG_RD, &stats->rfc, 0,
5623 "Fragmented Packets Received ");
5624 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5625 CTLFLAG_RD, &stats->roc, 0,
5626 "Oversized Packets Received");
5627 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5628 CTLFLAG_RD, &stats->rjc, 0,
5630 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5631 CTLFLAG_RD, &stats->mngprc, 0,
5632 "Management Packets Received");
5633 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5634 CTLFLAG_RD, &stats->mngptc, 0,
5635 "Management Packets Dropped");
5636 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5637 CTLFLAG_RD, &stats->xec, 0,
5640 /* Packet Transmission Stats */
5641 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5642 CTLFLAG_RD, &stats->gotc, 0,
5643 "Good Octets Transmitted");
5644 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5645 CTLFLAG_RD, &stats->tpt, 0,
5646 "Total Packets Transmitted");
5647 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5648 CTLFLAG_RD, &stats->gptc, 0,
5649 "Good Packets Transmitted");
5650 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5651 CTLFLAG_RD, &stats->bptc, 0,
5652 "Broadcast Packets Transmitted");
5653 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5654 CTLFLAG_RD, &stats->mptc, 0,
5655 "Multicast Packets Transmitted");
5656 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5657 CTLFLAG_RD, &stats->mngptc, 0,
5658 "Management Packets Transmitted");
5659 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5660 CTLFLAG_RD, &stats->ptc64, 0,
5661 "64 byte frames transmitted ");
5662 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5663 CTLFLAG_RD, &stats->ptc127, 0,
5664 "65-127 byte frames transmitted");
5665 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5666 CTLFLAG_RD, &stats->ptc255, 0,
5667 "128-255 byte frames transmitted");
5668 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5669 CTLFLAG_RD, &stats->ptc511, 0,
5670 "256-511 byte frames transmitted");
5671 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5672 CTLFLAG_RD, &stats->ptc1023, 0,
5673 "512-1023 byte frames transmitted");
5674 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5675 CTLFLAG_RD, &stats->ptc1522, 0,
5676 "1024-1522 byte frames transmitted");
5679 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5680 CTLFLAG_RD, &stats->fccrc, 0,
5682 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5683 CTLFLAG_RD, &stats->fclast, 0,
5685 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5686 CTLFLAG_RD, &stats->fcoerpdc, 0,
5687 "FCoE Packets Dropped");
5688 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5689 CTLFLAG_RD, &stats->fcoeprc, 0,
5690 "FCoE Packets Received");
5691 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5692 CTLFLAG_RD, &stats->fcoeptc, 0,
5693 "FCoE Packets Transmitted");
5694 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5695 CTLFLAG_RD, &stats->fcoedwrc, 0,
5696 "FCoE DWords Received");
5697 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5698 CTLFLAG_RD, &stats->fcoedwtc, 0,
5699 "FCoE DWords Transmitted");
5703 ** Set flow control using sysctl:
5704 ** Flow control values:
5711 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5714 struct adapter *adapter = (struct adapter *) arg1;
5717 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5718 if ((error) || (req->newptr == NULL))
5721 /* Don't bother if it's not changed */
5722 if (adapter->fc == last)
5725 switch (adapter->fc) {
5726 case ixgbe_fc_rx_pause:
5727 case ixgbe_fc_tx_pause:
5729 adapter->hw.fc.requested_mode = adapter->fc;
5733 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5736 ixgbe_fc_enable(&adapter->hw, 0);
5741 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5742 const char *description, int *limit, int value)
5745 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5746 SYSCTL_CHILDREN(adapter->sysctl_tree),
5747 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5751 ** Control link advertise speed:
5753 ** 1 - advertise only 1G
5754 ** 2 - advertise 100Mb
5757 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5760 struct adapter *adapter;
5762 struct ixgbe_hw *hw;
5763 ixgbe_link_speed speed, last;
5765 adapter = (struct adapter *) arg1;
5768 last = hw->phy.autoneg_advertised;
5770 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5772 if ((error) || (adapter->advertise == -1))
5775 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5776 (hw->phy.multispeed_fiber)))
5779 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5780 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5784 if (adapter->advertise == 1)
5785 speed = IXGBE_LINK_SPEED_1GB_FULL;
5786 else if (adapter->advertise == 2)
5787 speed = IXGBE_LINK_SPEED_100_FULL;
5789 speed = IXGBE_LINK_SPEED_1GB_FULL |
5790 IXGBE_LINK_SPEED_10GB_FULL;
5792 if (speed == last) /* no change */
5795 hw->mac.autotry_restart = TRUE;
5796 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5802 ** Thermal Shutdown Trigger
5803 ** - cause a Thermal Overtemp IRQ
5806 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5808 int error, fire = 0;
5809 struct adapter *adapter = (struct adapter *) arg1;
5810 struct ixgbe_hw *hw = &adapter->hw;
5813 if (hw->mac.type != ixgbe_mac_X540)
5816 error = sysctl_handle_int(oidp, &fire, 0, req);
5817 if ((error) || (req->newptr == NULL))
5821 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5822 reg |= IXGBE_EICR_TS;
5823 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);