1 /******************************************************************************
3 Copyright (c) 2001-2012, Intel Corporation
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
36 #include "opt_inet6.h"
40 /*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43 int ixgbe_display_debug_stats = 0;
45 /*********************************************************************
47 *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
50 /*********************************************************************
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
90 /*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
94 static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
98 /*********************************************************************
100 *********************************************************************/
101 static int ixgbe_probe(device_t);
102 static int ixgbe_attach(device_t);
103 static int ixgbe_detach(device_t);
104 static int ixgbe_shutdown(device_t);
105 static void ixgbe_start(struct ifnet *);
106 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111 static void ixgbe_qflush(struct ifnet *);
113 static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void ixgbe_init(void *);
115 static void ixgbe_init_locked(struct adapter *);
116 static void ixgbe_stop(void *);
117 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int ixgbe_media_change(struct ifnet *);
119 static void ixgbe_identify_hardware(struct adapter *);
120 static int ixgbe_allocate_pci_resources(struct adapter *);
121 static int ixgbe_allocate_msix(struct adapter *);
122 static int ixgbe_allocate_legacy(struct adapter *);
123 static int ixgbe_allocate_queues(struct adapter *);
124 #if 0 /* HAVE_MSIX */
125 static int ixgbe_setup_msix(struct adapter *);
127 static void ixgbe_free_pci_resources(struct adapter *);
128 static void ixgbe_local_timer(void *);
129 static int ixgbe_setup_interface(device_t, struct adapter *);
130 static void ixgbe_config_link(struct adapter *);
132 static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133 static int ixgbe_setup_transmit_structures(struct adapter *);
134 static void ixgbe_setup_transmit_ring(struct tx_ring *);
135 static void ixgbe_initialize_transmit_units(struct adapter *);
136 static void ixgbe_free_transmit_structures(struct adapter *);
137 static void ixgbe_free_transmit_buffers(struct tx_ring *);
139 static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140 static int ixgbe_setup_receive_structures(struct adapter *);
141 static int ixgbe_setup_receive_ring(struct rx_ring *);
142 static void ixgbe_initialize_receive_units(struct adapter *);
143 static void ixgbe_free_receive_structures(struct adapter *);
144 static void ixgbe_free_receive_buffers(struct rx_ring *);
146 static void ixgbe_setup_hw_rsc(struct rx_ring *);
149 static void ixgbe_enable_intr(struct adapter *);
150 static void ixgbe_disable_intr(struct adapter *);
151 static void ixgbe_update_stats_counters(struct adapter *);
152 static bool ixgbe_txeof(struct tx_ring *);
153 static bool ixgbe_rxeof(struct ix_queue *, int);
154 static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
155 static void ixgbe_set_promisc(struct adapter *);
156 static void ixgbe_set_multi(struct adapter *);
157 static void ixgbe_update_link_status(struct adapter *);
158 static void ixgbe_refresh_mbufs(struct rx_ring *, int);
159 static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
160 static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
161 static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
162 static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
163 static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
164 struct ixgbe_dma_alloc *, int);
165 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
166 static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
167 const char *, int *, int);
168 static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
170 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
172 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void ixgbe_configure_ivars(struct adapter *);
174 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
176 static void ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
180 static void ixgbe_add_hw_stats(struct adapter *adapter);
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
186 /* Support for pluggable optic modules */
187 static bool ixgbe_sfp_probe(struct adapter *);
188 static void ixgbe_setup_optics(struct adapter *);
190 /* Legacy (single vector interrupt handler */
191 static void ixgbe_legacy_irq(void *);
193 /* The MSI/X Interrupt handlers */
194 static void ixgbe_msix_que(void *);
195 static void ixgbe_msix_link(void *);
197 /* Deferred interrupt tasklets */
198 static void ixgbe_handle_que(void *, int);
199 static void ixgbe_handle_link(void *, int);
200 static void ixgbe_handle_msf(void *, int);
201 static void ixgbe_handle_mod(void *, int);
204 static void ixgbe_atr(struct tx_ring *, struct mbuf *);
205 static void ixgbe_reinit_fdir(void *, int);
208 /*********************************************************************
209 * FreeBSD Device Interface Entry Points
210 *********************************************************************/
212 static device_method_t ixgbe_methods[] = {
213 /* Device interface */
214 DEVMETHOD(device_probe, ixgbe_probe),
215 DEVMETHOD(device_attach, ixgbe_attach),
216 DEVMETHOD(device_detach, ixgbe_detach),
217 DEVMETHOD(device_shutdown, ixgbe_shutdown),
221 static driver_t ixgbe_driver = {
222 "ix", ixgbe_methods, sizeof(struct adapter),
225 devclass_t ixgbe_devclass;
226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
232 ** TUNEABLE PARAMETERS:
236 ** AIM: Adaptive Interrupt Moderation
237 ** which means that the interrupt rate
238 ** is varied over time based on the
239 ** traffic for that interrupt vector
241 static int ixgbe_enable_aim = TRUE;
242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
247 /* How many packets rxeof tries to clean at a time */
248 static int ixgbe_rx_process_limit = 128;
249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
252 ** Smart speed setting, default to on
253 ** this only works as a compile option
254 ** right now as its during attach, set
255 ** this to 'ixgbe_smart_speed_off' to
258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
260 static int ixgbe_msi_enable = 1;
261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
264 * MSIX should be the default for best performance,
265 * but this allows it to be forced off for testing.
267 static int ixgbe_enable_msix = 1;
268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
271 * Header split: this causes the hardware to DMA
272 * the header into a separate mbuf from the payload,
273 * it can be a performance win in some workloads, but
274 * in others it actually hurts, its off by default.
276 static int ixgbe_header_split = FALSE;
277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
280 * Number of Queues, can be set to 0,
281 * it then autoconfigures based on the
282 * number of cpus with a max of 8. This
283 * can be overriden manually here.
285 static int ixgbe_num_queues = 0;
286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
289 ** Number of TX descriptors per ring,
290 ** setting higher than RX as this seems
291 ** the better performing choice.
293 static int ixgbe_txd = PERFORM_TXD;
294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
296 /* Number of RX descriptors per ring */
297 static int ixgbe_rxd = PERFORM_RXD;
298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
300 /* Keep running tab on them for sanity check */
301 static int ixgbe_total_ports;
305 ** For Flow Director: this is the
306 ** number of TX packets we sample
307 ** for the filter pool, this means
308 ** every 20th packet will be probed.
310 ** This feature can be disabled by
311 ** setting this to 0.
313 static int atr_sample_rate = 20;
315 ** Flow Director actually 'steals'
316 ** part of the packet buffer as its
317 ** filter pool, this variable controls
319 ** 0 = 64K, 1 = 128K, 2 = 256K
321 static int fdir_pballoc = 1;
326 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
327 * be a reference on how to implement netmap support in a driver.
328 * Additional comments are in ixgbe_netmap.h .
330 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
331 * that extend the standard driver.
333 #include <dev/netmap/ixgbe_netmap.h>
334 #endif /* DEV_NETMAP */
336 /*********************************************************************
337 * Device identification routine
339 * ixgbe_probe determines if the driver should be loaded on
340 * adapter based on PCI vendor/device id of the adapter.
342 * return BUS_PROBE_DEFAULT on success, positive on failure
343 *********************************************************************/
346 ixgbe_probe(device_t dev)
348 ixgbe_vendor_info_t *ent;
350 u16 pci_vendor_id = 0;
351 u16 pci_device_id = 0;
352 u16 pci_subvendor_id = 0;
353 u16 pci_subdevice_id = 0;
354 char adapter_name[256];
356 INIT_DEBUGOUT("ixgbe_probe: begin");
358 pci_vendor_id = pci_get_vendor(dev);
359 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
362 pci_device_id = pci_get_device(dev);
363 pci_subvendor_id = pci_get_subvendor(dev);
364 pci_subdevice_id = pci_get_subdevice(dev);
366 ent = ixgbe_vendor_info_array;
367 while (ent->vendor_id != 0) {
368 if ((pci_vendor_id == ent->vendor_id) &&
369 (pci_device_id == ent->device_id) &&
371 ((pci_subvendor_id == ent->subvendor_id) ||
372 (ent->subvendor_id == 0)) &&
374 ((pci_subdevice_id == ent->subdevice_id) ||
375 (ent->subdevice_id == 0))) {
376 ksprintf(adapter_name, "%s, Version - %s",
377 ixgbe_strings[ent->index],
378 ixgbe_driver_version);
379 device_set_desc_copy(dev, adapter_name);
381 return (BUS_PROBE_DEFAULT);
388 /*********************************************************************
389 * Device initialization routine
391 * The attach entry point is called when the driver is being loaded.
392 * This routine identifies the type of hardware, allocates all resources
393 * and initializes the hardware.
395 * return 0 on success, positive on failure
396 *********************************************************************/
399 ixgbe_attach(device_t dev)
401 struct adapter *adapter;
407 INIT_DEBUGOUT("ixgbe_attach: begin");
409 if (resource_disabled("ixgbe", device_get_unit(dev))) {
410 device_printf(dev, "Disabled by device hint\n");
414 /* Allocate, clear, and link in our adapter structure */
415 adapter = device_get_softc(dev);
416 adapter->dev = adapter->osdep.dev = dev;
420 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
421 spin_init(&adapter->mcast_spin);
425 sysctl_ctx_init(&adapter->sysctl_ctx);
426 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
427 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
428 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
429 if (adapter->sysctl_tree == NULL) {
430 device_printf(adapter->dev, "can't add sysctl node\n");
433 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
434 SYSCTL_CHILDREN(adapter->sysctl_tree),
435 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
436 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
438 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
439 SYSCTL_CHILDREN(adapter->sysctl_tree),
440 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
441 &ixgbe_enable_aim, 1, "Interrupt Moderation");
444 ** Allow a kind of speed control by forcing the autoneg
445 ** advertised speed list to only a certain value, this
446 ** supports 1G on 82599 devices, and 100Mb on x540.
448 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
449 SYSCTL_CHILDREN(adapter->sysctl_tree),
450 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
451 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
453 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
454 SYSCTL_CHILDREN(adapter->sysctl_tree),
455 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
456 0, ixgbe_set_thermal_test, "I", "Thermal Test");
458 /* Set up the timer callout */
459 /* XXX: shouldn't this be a spin lock ? */
460 lockinit(&adapter->core_lock, "ixgbe core lock", 0, LK_CANRECURSE);
461 callout_init(&adapter->timer);
463 /* Determine hardware revision */
464 ixgbe_identify_hardware(adapter);
466 /* Do base PCI setup - map BAR0 */
467 if (ixgbe_allocate_pci_resources(adapter)) {
468 device_printf(dev, "Allocation of PCI resources failed\n");
473 /* Do descriptor calc and sanity checks */
474 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
475 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
476 device_printf(dev, "TXD config issue, using default!\n");
477 adapter->num_tx_desc = DEFAULT_TXD;
479 adapter->num_tx_desc = ixgbe_txd;
482 ** With many RX rings it is easy to exceed the
483 ** system mbuf allocation. Tuning nmbclusters
484 ** can alleviate this.
486 if (nmbclusters > 0 ) {
488 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
489 if (s > nmbclusters) {
490 device_printf(dev, "RX Descriptors exceed "
491 "system mbuf max, using default instead!\n");
492 ixgbe_rxd = DEFAULT_RXD;
496 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
497 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
498 device_printf(dev, "RXD config issue, using default!\n");
499 adapter->num_rx_desc = DEFAULT_RXD;
501 adapter->num_rx_desc = ixgbe_rxd;
503 /* Allocate our TX/RX Queues */
504 if (ixgbe_allocate_queues(adapter)) {
509 /* Allocate multicast array memory. */
510 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
511 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
512 if (adapter->mta == NULL) {
513 device_printf(dev, "Can not allocate multicast setup array\n");
518 /* Initialize the shared code */
519 error = ixgbe_init_shared_code(hw);
520 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
522 ** No optics in this port, set up
523 ** so the timer routine will probe
524 ** for later insertion.
526 adapter->sfp_probe = TRUE;
528 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
529 device_printf(dev,"Unsupported SFP+ module detected!\n");
533 device_printf(dev,"Unable to initialize the shared code\n");
538 /* Make sure we have a good EEPROM before we read from it */
539 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
540 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
545 error = ixgbe_init_hw(hw);
547 case IXGBE_ERR_EEPROM_VERSION:
548 device_printf(dev, "This device is a pre-production adapter/"
549 "LOM. Please be aware there may be issues associated "
550 "with your hardware.\n If you are experiencing problems "
551 "please contact your Intel or hardware representative "
552 "who provided you with this hardware.\n");
554 case IXGBE_ERR_SFP_NOT_SUPPORTED:
555 device_printf(dev,"Unsupported SFP+ Module\n");
557 device_printf(dev,"Hardware Initialization Failure\n");
559 case IXGBE_ERR_SFP_NOT_PRESENT:
560 device_printf(dev,"No SFP+ Module found\n");
566 /* Detect and set physical type */
567 ixgbe_setup_optics(adapter);
569 if ((adapter->msix > 1) && (ixgbe_enable_msix))
570 error = ixgbe_allocate_msix(adapter);
572 error = ixgbe_allocate_legacy(adapter);
576 /* Setup OS specific network interface */
577 if (ixgbe_setup_interface(dev, adapter) != 0)
580 /* Sysctl for limiting the amount of work done in the taskqueue */
581 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
582 "max number of rx packets to process", &adapter->rx_process_limit,
583 ixgbe_rx_process_limit);
585 /* Initialize statistics */
586 ixgbe_update_stats_counters(adapter);
588 /* Register for VLAN events */
589 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
590 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
591 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
592 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
594 /* Print PCIE bus type/speed/width info */
595 ixgbe_get_bus_info(hw);
596 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
597 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
598 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
599 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
600 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
601 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
604 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
605 (hw->bus.speed == ixgbe_bus_speed_2500)) {
606 device_printf(dev, "PCI-Express bandwidth available"
607 " for this card\n is not sufficient for"
608 " optimal performance.\n");
609 device_printf(dev, "For optimal performance a x8 "
610 "PCIE, or x4 PCIE 2 slot is required.\n");
613 /* let hardware know driver is loaded */
614 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
615 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
616 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
618 ixgbe_add_hw_stats(adapter);
621 ixgbe_netmap_attach(adapter);
622 #endif /* DEV_NETMAP */
623 INIT_DEBUGOUT("ixgbe_attach: end");
626 ixgbe_free_transmit_structures(adapter);
627 ixgbe_free_receive_structures(adapter);
629 if (adapter->ifp != NULL)
630 if_free(adapter->ifp);
631 ixgbe_free_pci_resources(adapter);
632 kfree(adapter->mta, M_DEVBUF);
637 /*********************************************************************
638 * Device removal routine
640 * The detach entry point is called when the driver is being removed.
641 * This routine stops the adapter and deallocates all the resources
642 * that were allocated for driver operation.
644 * return 0 on success, positive on failure
645 *********************************************************************/
648 ixgbe_detach(device_t dev)
650 struct adapter *adapter = device_get_softc(dev);
651 struct ix_queue *que = adapter->queues;
654 INIT_DEBUGOUT("ixgbe_detach: begin");
657 /* Make sure VLANS are not using driver */
658 if (adapter->ifp->if_vlantrunk != NULL) {
659 device_printf(dev,"Vlan in use, detach first\n");
664 IXGBE_CORE_LOCK(adapter);
666 IXGBE_CORE_UNLOCK(adapter);
668 for (int i = 0; i < adapter->num_queues; i++, que++) {
670 taskqueue_drain(que->tq, &que->que_task);
671 taskqueue_free(que->tq);
675 /* Drain the Link queue */
677 taskqueue_drain(adapter->tq, &adapter->link_task);
678 taskqueue_drain(adapter->tq, &adapter->mod_task);
679 taskqueue_drain(adapter->tq, &adapter->msf_task);
681 taskqueue_drain(adapter->tq, &adapter->fdir_task);
683 taskqueue_free(adapter->tq);
686 /* let hardware know driver is unloading */
687 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
688 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
689 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
691 /* Unregister VLAN events */
693 if (adapter->vlan_attach != NULL)
694 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
695 if (adapter->vlan_detach != NULL)
696 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
699 ether_ifdetach(adapter->ifp);
700 callout_stop(&adapter->timer);
701 lockuninit(&adapter->core_lock);
703 netmap_detach(adapter->ifp);
704 #endif /* DEV_NETMAP */
705 ixgbe_free_pci_resources(adapter);
706 bus_generic_detach(dev);
707 if_free(adapter->ifp);
709 ixgbe_free_transmit_structures(adapter);
710 ixgbe_free_receive_structures(adapter);
711 kfree(adapter->mta, M_DEVBUF);
712 sysctl_ctx_free(&adapter->sysctl_ctx);
714 spin_uninit(&adapter->mcast_spin);
715 IXGBE_CORE_LOCK_DESTROY(adapter);
719 /*********************************************************************
721 * Shutdown entry point
723 **********************************************************************/
726 ixgbe_shutdown(device_t dev)
728 struct adapter *adapter = device_get_softc(dev);
729 IXGBE_CORE_LOCK(adapter);
731 IXGBE_CORE_UNLOCK(adapter);
736 /*********************************************************************
737 * Transmit entry point
739 * ixgbe_start is called by the stack to initiate a transmit.
740 * The driver will remain in this routine as long as there are
741 * packets to transmit and transmit resources are available.
742 * In case resources are not available stack is notified and
743 * the packet is requeued.
744 **********************************************************************/
747 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
750 struct adapter *adapter = txr->adapter;
752 IXGBE_TX_LOCK_ASSERT(txr);
754 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
756 if (!adapter->link_active)
759 while (!ifq_is_empty(&ifp->if_snd)) {
760 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
761 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
765 m_head = ifq_dequeue(&ifp->if_snd, NULL);
769 if (ixgbe_xmit(txr, &m_head)) {
770 #if 0 /* XXX: prepend to an ALTQ queue ? */
772 IF_PREPEND(&ifp->if_snd, m_head);
774 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
775 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
778 /* Send a copy of the frame to the BPF listener */
779 ETHER_BPF_MTAP(ifp, m_head);
781 /* Set watchdog on */
782 txr->watchdog_time = ticks;
783 txr->queue_status = IXGBE_QUEUE_WORKING;
790 * Legacy TX start - called by the stack, this
791 * always uses the first tx ring, and should
792 * not be used with multiqueue tx enabled.
795 ixgbe_start(struct ifnet *ifp)
797 struct adapter *adapter = ifp->if_softc;
798 struct tx_ring *txr = adapter->tx_rings;
800 if (ifp->if_flags & IFF_RUNNING) {
802 ixgbe_start_locked(txr, ifp);
803 IXGBE_TX_UNLOCK(txr);
808 #if 0 /* __FreeBSD_version >= 800000 */
810 ** Multiqueue Transmit driver
814 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
816 struct adapter *adapter = ifp->if_softc;
817 struct ix_queue *que;
821 /* Which queue to use */
822 if ((m->m_flags & M_FLOWID) != 0)
823 i = m->m_pkthdr.flowid % adapter->num_queues;
825 i = curcpu % adapter->num_queues;
827 txr = &adapter->tx_rings[i];
828 que = &adapter->queues[i];
830 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
831 IXGBE_TX_TRYLOCK(txr)) {
832 err = ixgbe_mq_start_locked(ifp, txr, m);
833 IXGBE_TX_UNLOCK(txr);
835 err = drbr_enqueue(ifp, txr->br, m);
836 taskqueue_enqueue(que->tq, &que->que_task);
843 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
845 struct adapter *adapter = txr->adapter;
847 int enqueued, err = 0;
849 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
850 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
851 adapter->link_active == 0) {
853 err = drbr_enqueue(ifp, txr->br, m);
859 next = drbr_dequeue(ifp, txr->br);
860 } else if (drbr_needs_enqueue(ifp, txr->br)) {
861 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
863 next = drbr_dequeue(ifp, txr->br);
867 /* Process the queue */
868 while (next != NULL) {
869 if ((err = ixgbe_xmit(txr, &next)) != 0) {
871 err = drbr_enqueue(ifp, txr->br, next);
875 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
876 /* Send a copy of the frame to the BPF listener */
877 ETHER_BPF_MTAP(ifp, next);
878 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
880 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
882 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
883 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
886 next = drbr_dequeue(ifp, txr->br);
890 /* Set watchdog on */
891 txr->queue_status |= IXGBE_QUEUE_WORKING;
892 txr->watchdog_time = ticks;
895 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
902 ** Flush all ring buffers
905 ixgbe_qflush(struct ifnet *ifp)
907 struct adapter *adapter = ifp->if_softc;
908 struct tx_ring *txr = adapter->tx_rings;
911 for (int i = 0; i < adapter->num_queues; i++, txr++) {
913 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
915 IXGBE_TX_UNLOCK(txr);
919 #endif /* __FreeBSD_version >= 800000 */
921 /*********************************************************************
924 * ixgbe_ioctl is called when the user wants to configure the
927 * return 0 on success, positive on failure
928 **********************************************************************/
931 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
933 struct adapter *adapter = ifp->if_softc;
934 struct ifreq *ifr = (struct ifreq *) data;
935 #if defined(INET) || defined(INET6)
936 struct ifaddr *ifa = (struct ifaddr *)data;
937 bool avoid_reset = FALSE;
945 if (ifa->ifa_addr->sa_family == AF_INET)
949 if (ifa->ifa_addr->sa_family == AF_INET6)
952 #if defined(INET) || defined(INET6)
954 ** Calling init results in link renegotiation,
955 ** so we avoid doing it when possible.
958 ifp->if_flags |= IFF_UP;
959 if (!(ifp->if_flags & IFF_RUNNING))
961 if (!(ifp->if_flags & IFF_NOARP))
962 arp_ifinit(ifp, ifa);
964 error = ether_ioctl(ifp, command, data);
968 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
969 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
972 IXGBE_CORE_LOCK(adapter);
973 ifp->if_mtu = ifr->ifr_mtu;
974 adapter->max_frame_size =
975 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
976 ixgbe_init_locked(adapter);
977 IXGBE_CORE_UNLOCK(adapter);
981 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
982 IXGBE_CORE_LOCK(adapter);
983 if (ifp->if_flags & IFF_UP) {
984 if ((ifp->if_flags & IFF_RUNNING)) {
985 if ((ifp->if_flags ^ adapter->if_flags) &
986 (IFF_PROMISC | IFF_ALLMULTI)) {
987 ixgbe_set_promisc(adapter);
990 ixgbe_init_locked(adapter);
992 if (ifp->if_flags & IFF_RUNNING)
994 adapter->if_flags = ifp->if_flags;
995 IXGBE_CORE_UNLOCK(adapter);
999 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1000 if (ifp->if_flags & IFF_RUNNING) {
1001 IXGBE_CORE_LOCK(adapter);
1002 ixgbe_disable_intr(adapter);
1003 ixgbe_set_multi(adapter);
1004 ixgbe_enable_intr(adapter);
1005 IXGBE_CORE_UNLOCK(adapter);
1010 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1011 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1015 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1016 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1017 if (mask & IFCAP_HWCSUM)
1018 ifp->if_capenable ^= IFCAP_HWCSUM;
1020 if (mask & IFCAP_TSO4)
1021 ifp->if_capenable ^= IFCAP_TSO4;
1022 if (mask & IFCAP_TSO6)
1023 ifp->if_capenable ^= IFCAP_TSO6;
1026 if (mask & IFCAP_LRO)
1027 ifp->if_capenable ^= IFCAP_LRO;
1029 if (mask & IFCAP_VLAN_HWTAGGING)
1030 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1031 if (mask & IFCAP_VLAN_HWFILTER)
1032 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1034 if (mask & IFCAP_VLAN_HWTSO)
1035 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1037 if (ifp->if_flags & IFF_RUNNING) {
1038 IXGBE_CORE_LOCK(adapter);
1039 ixgbe_init_locked(adapter);
1040 IXGBE_CORE_UNLOCK(adapter);
1043 VLAN_CAPABILITIES(ifp);
1049 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1050 error = ether_ioctl(ifp, command, data);
1057 /*********************************************************************
1060 * This routine is used in two ways. It is used by the stack as
1061 * init entry point in network interface structure. It is also used
1062 * by the driver as a hw/sw initialization routine to get to a
1065 * return 0 on success, positive on failure
1066 **********************************************************************/
1067 #define IXGBE_MHADD_MFS_SHIFT 16
1070 ixgbe_init_locked(struct adapter *adapter)
1072 struct ifnet *ifp = adapter->ifp;
1073 device_t dev = adapter->dev;
1074 struct ixgbe_hw *hw = &adapter->hw;
1075 u32 k, txdctl, mhadd, gpie;
1078 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1079 INIT_DEBUGOUT("ixgbe_init: begin");
1080 hw->adapter_stopped = FALSE;
1081 ixgbe_stop_adapter(hw);
1082 callout_stop(&adapter->timer);
1084 /* reprogram the RAR[0] in case user changed it. */
1085 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1087 /* Get the latest mac address, User can use a LAA */
1088 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1089 IXGBE_ETH_LENGTH_OF_ADDRESS);
1090 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1091 hw->addr_ctrl.rar_used_count = 1;
1093 /* Set the various hardware offload abilities */
1094 ifp->if_hwassist = 0;
1096 if (ifp->if_capenable & IFCAP_TSO)
1097 ifp->if_hwassist |= CSUM_TSO;
1099 if (ifp->if_capenable & IFCAP_TXCSUM) {
1100 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1102 if (hw->mac.type != ixgbe_mac_82598EB)
1103 ifp->if_hwassist |= CSUM_SCTP;
1107 /* Prepare transmit descriptors and buffers */
1108 if (ixgbe_setup_transmit_structures(adapter)) {
1109 device_printf(dev,"Could not setup transmit structures\n");
1110 ixgbe_stop(adapter);
1115 ixgbe_initialize_transmit_units(adapter);
1117 /* Setup Multicast table */
1118 ixgbe_set_multi(adapter);
1121 ** Determine the correct mbuf pool
1122 ** for doing jumbo/headersplit
1124 if (adapter->max_frame_size <= 2048)
1125 adapter->rx_mbuf_sz = MCLBYTES;
1126 else if (adapter->max_frame_size <= 4096)
1127 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1128 else if (adapter->max_frame_size <= 9216)
1129 adapter->rx_mbuf_sz = MJUM9BYTES;
1131 adapter->rx_mbuf_sz = MJUM16BYTES;
1133 /* Prepare receive descriptors and buffers */
1134 if (ixgbe_setup_receive_structures(adapter)) {
1135 device_printf(dev,"Could not setup receive structures\n");
1136 ixgbe_stop(adapter);
1140 /* Configure RX settings */
1141 ixgbe_initialize_receive_units(adapter);
1143 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1145 /* Enable Fan Failure Interrupt */
1146 gpie |= IXGBE_SDP1_GPIEN;
1148 /* Add for Module detection */
1149 if (hw->mac.type == ixgbe_mac_82599EB)
1150 gpie |= IXGBE_SDP2_GPIEN;
1152 /* Thermal Failure Detection */
1153 if (hw->mac.type == ixgbe_mac_X540)
1154 gpie |= IXGBE_SDP0_GPIEN;
1156 if (adapter->msix > 1) {
1157 /* Enable Enhanced MSIX mode */
1158 gpie |= IXGBE_GPIE_MSIX_MODE;
1159 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1162 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1165 if (ifp->if_mtu > ETHERMTU) {
1166 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1167 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1168 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1169 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1172 /* Now enable all the queues */
1174 for (int i = 0; i < adapter->num_queues; i++) {
1175 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1176 txdctl |= IXGBE_TXDCTL_ENABLE;
1177 /* Set WTHRESH to 8, burst writeback */
1178 txdctl |= (8 << 16);
1180 * When the internal queue falls below PTHRESH (32),
1181 * start prefetching as long as there are at least
1182 * HTHRESH (1) buffers ready. The values are taken
1183 * from the Intel linux driver 3.8.21.
1184 * Prefetching enables tx line rate even with 1 queue.
1186 txdctl |= (32 << 0) | (1 << 8);
1187 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1190 for (int i = 0; i < adapter->num_queues; i++) {
1191 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1192 if (hw->mac.type == ixgbe_mac_82598EB) {
1198 rxdctl &= ~0x3FFFFF;
1201 rxdctl |= IXGBE_RXDCTL_ENABLE;
1202 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1203 for (k = 0; k < 10; k++) {
1204 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1205 IXGBE_RXDCTL_ENABLE)
1213 * In netmap mode, we must preserve the buffers made
1214 * available to userspace before the if_init()
1215 * (this is true by default on the TX side, because
1216 * init makes all buffers available to userspace).
1218 * netmap_reset() and the device specific routines
1219 * (e.g. ixgbe_setup_receive_rings()) map these
1220 * buffers at the end of the NIC ring, so here we
1221 * must set the RDT (tail) register to make sure
1222 * they are not overwritten.
1224 * In this driver the NIC ring starts at RDH = 0,
1225 * RDT points to the last slot available for reception (?),
1226 * so RDT = num_rx_desc - 1 means the whole ring is available.
1228 if (ifp->if_capenable & IFCAP_NETMAP) {
1229 struct netmap_adapter *na = NA(adapter->ifp);
1230 struct netmap_kring *kring = &na->rx_rings[i];
1231 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1233 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1235 #endif /* DEV_NETMAP */
1236 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1239 /* Set up VLAN support and filter */
1240 ixgbe_setup_vlan_hw_support(adapter);
1242 /* Enable Receive engine */
1243 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1244 if (hw->mac.type == ixgbe_mac_82598EB)
1245 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1246 rxctrl |= IXGBE_RXCTRL_RXEN;
1247 ixgbe_enable_rx_dma(hw, rxctrl);
1249 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1251 /* Set up MSI/X routing */
1252 if (ixgbe_enable_msix) {
1253 ixgbe_configure_ivars(adapter);
1254 /* Set up auto-mask */
1255 if (hw->mac.type == ixgbe_mac_82598EB)
1256 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1258 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1259 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1261 } else { /* Simple settings for Legacy/MSI */
1262 ixgbe_set_ivar(adapter, 0, 0, 0);
1263 ixgbe_set_ivar(adapter, 0, 0, 1);
1264 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1268 /* Init Flow director */
1269 if (hw->mac.type != ixgbe_mac_82598EB) {
1270 u32 hdrm = 32 << fdir_pballoc;
1272 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1273 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1278 ** Check on any SFP devices that
1279 ** need to be kick-started
1281 if (hw->phy.type == ixgbe_phy_none) {
1282 int err = hw->phy.ops.identify(hw);
1283 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1285 "Unsupported SFP+ module type was detected.\n");
1290 /* Set moderation on the Link interrupt */
1291 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1293 /* Config/Enable Link */
1294 ixgbe_config_link(adapter);
1296 /* Hardware Packet Buffer & Flow Control setup */
1298 u32 rxpb, frame, size, tmp;
1300 frame = adapter->max_frame_size;
1302 /* Calculate High Water */
1303 if (hw->mac.type == ixgbe_mac_X540)
1304 tmp = IXGBE_DV_X540(frame, frame);
1306 tmp = IXGBE_DV(frame, frame);
1307 size = IXGBE_BT2KB(tmp);
1308 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1309 hw->fc.high_water[0] = rxpb - size;
1311 /* Now calculate Low Water */
1312 if (hw->mac.type == ixgbe_mac_X540)
1313 tmp = IXGBE_LOW_DV_X540(frame);
1315 tmp = IXGBE_LOW_DV(frame);
1316 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1318 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1319 hw->fc.pause_time = IXGBE_FC_PAUSE;
1320 hw->fc.send_xon = TRUE;
1322 /* Initialize the FC settings */
1325 /* And now turn on interrupts */
1326 ixgbe_enable_intr(adapter);
1328 /* Now inform the stack we're ready */
1329 ifp->if_flags |= IFF_RUNNING;
1330 ifp->if_flags &= ~IFF_OACTIVE;
1336 ixgbe_init(void *arg)
1338 struct adapter *adapter = arg;
1340 IXGBE_CORE_LOCK(adapter);
1341 ixgbe_init_locked(adapter);
1342 IXGBE_CORE_UNLOCK(adapter);
1349 ** MSIX Interrupt Handlers and Tasklets
1354 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1356 struct ixgbe_hw *hw = &adapter->hw;
1357 u64 queue = (u64)(1 << vector);
1360 if (hw->mac.type == ixgbe_mac_82598EB) {
1361 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1362 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1364 mask = (queue & 0xFFFFFFFF);
1366 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1367 mask = (queue >> 32);
1369 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1374 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1376 struct ixgbe_hw *hw = &adapter->hw;
1377 u64 queue = (u64)(1 << vector);
1380 if (hw->mac.type == ixgbe_mac_82598EB) {
1381 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1382 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1384 mask = (queue & 0xFFFFFFFF);
1386 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1387 mask = (queue >> 32);
1389 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1394 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1398 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1399 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1400 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1402 mask = (queues & 0xFFFFFFFF);
1403 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1404 mask = (queues >> 32);
1405 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1411 ixgbe_handle_que(void *context, int pending)
1413 struct ix_queue *que = context;
1414 struct adapter *adapter = que->adapter;
1415 struct tx_ring *txr = que->txr;
1416 struct ifnet *ifp = adapter->ifp;
1419 if (ifp->if_flags & IFF_RUNNING) {
1420 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1423 #if 0 /*__FreeBSD_version >= 800000*/
1424 if (!drbr_empty(ifp, txr->br))
1425 ixgbe_mq_start_locked(ifp, txr, NULL);
1427 if (!ifq_is_empty(&ifp->if_snd))
1428 ixgbe_start_locked(txr, ifp);
1430 IXGBE_TX_UNLOCK(txr);
1432 taskqueue_enqueue(que->tq, &que->que_task);
1437 /* Reenable this interrupt */
1438 ixgbe_enable_queue(adapter, que->msix);
1443 /*********************************************************************
1445 * Legacy Interrupt Service routine
1447 **********************************************************************/
1450 ixgbe_legacy_irq(void *arg)
1452 struct ix_queue *que = arg;
1453 struct adapter *adapter = que->adapter;
1454 struct ixgbe_hw *hw = &adapter->hw;
1455 struct tx_ring *txr = adapter->tx_rings;
1456 bool more_tx, more_rx;
1457 u32 reg_eicr, loop = MAX_LOOP;
1460 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1463 if (reg_eicr == 0) {
1464 ixgbe_enable_intr(adapter);
1468 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1472 more_tx = ixgbe_txeof(txr);
1473 } while (loop-- && more_tx);
1474 IXGBE_TX_UNLOCK(txr);
1476 if (more_rx || more_tx)
1477 taskqueue_enqueue(que->tq, &que->que_task);
1479 /* Check for fan failure */
1480 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1481 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1482 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1483 "REPLACE IMMEDIATELY!!\n");
1484 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1487 /* Link status change */
1488 if (reg_eicr & IXGBE_EICR_LSC)
1489 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1491 ixgbe_enable_intr(adapter);
1496 /*********************************************************************
1498 * MSIX Queue Interrupt Service routine
1500 **********************************************************************/
1502 ixgbe_msix_que(void *arg)
1504 struct ix_queue *que = arg;
1505 struct adapter *adapter = que->adapter;
1506 struct tx_ring *txr = que->txr;
1507 struct rx_ring *rxr = que->rxr;
1508 bool more_tx, more_rx;
1511 ixgbe_disable_queue(adapter, que->msix);
1514 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1517 more_tx = ixgbe_txeof(txr);
1519 ** Make certain that if the stack
1520 ** has anything queued the task gets
1521 ** scheduled to handle it.
1524 #if __FreeBSD_version < 800000
1525 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1527 if (!drbr_empty(adapter->ifp, txr->br))
1530 if (!ifq_is_empty(&adapter->ifp->if_snd))
1532 IXGBE_TX_UNLOCK(txr);
1536 if (ixgbe_enable_aim == FALSE)
1539 ** Do Adaptive Interrupt Moderation:
1540 ** - Write out last calculated setting
1541 ** - Calculate based on average size over
1542 ** the last interval.
1544 if (que->eitr_setting)
1545 IXGBE_WRITE_REG(&adapter->hw,
1546 IXGBE_EITR(que->msix), que->eitr_setting);
1548 que->eitr_setting = 0;
1550 /* Idle, do nothing */
1551 if ((txr->bytes == 0) && (rxr->bytes == 0))
1554 if ((txr->bytes) && (txr->packets))
1555 newitr = txr->bytes/txr->packets;
1556 if ((rxr->bytes) && (rxr->packets))
1557 newitr = max(newitr,
1558 (rxr->bytes / rxr->packets));
1559 newitr += 24; /* account for hardware frame, crc */
1561 /* set an upper boundary */
1562 newitr = min(newitr, 3000);
1564 /* Be nice to the mid range */
1565 if ((newitr > 300) && (newitr < 1200))
1566 newitr = (newitr / 3);
1568 newitr = (newitr / 2);
1570 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1571 newitr |= newitr << 16;
1573 newitr |= IXGBE_EITR_CNT_WDIS;
1575 /* save for next interrupt */
1576 que->eitr_setting = newitr;
1585 if (more_tx || more_rx)
1586 taskqueue_enqueue(que->tq, &que->que_task);
1587 else /* Reenable this interrupt */
1588 ixgbe_enable_queue(adapter, que->msix);
1594 ixgbe_msix_link(void *arg)
1596 struct adapter *adapter = arg;
1597 struct ixgbe_hw *hw = &adapter->hw;
1600 ++adapter->link_irq;
1602 /* First get the cause */
1603 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1604 /* Clear interrupt with write */
1605 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1607 /* Link status change */
1608 if (reg_eicr & IXGBE_EICR_LSC)
1609 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1611 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1613 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1614 /* This is probably overkill :) */
1615 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1617 /* Disable the interrupt */
1618 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1619 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1622 if (reg_eicr & IXGBE_EICR_ECC) {
1623 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1624 "Please Reboot!!\n");
1625 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1628 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1629 /* Clear the interrupt */
1630 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1631 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1632 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1633 /* Clear the interrupt */
1634 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1635 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1639 /* Check for fan failure */
1640 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1641 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1642 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1643 "REPLACE IMMEDIATELY!!\n");
1644 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1647 /* Check for over temp condition */
1648 if ((hw->mac.type == ixgbe_mac_X540) &&
1649 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1650 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1651 "PHY IS SHUT DOWN!!\n");
1652 device_printf(adapter->dev, "System shutdown required\n");
1653 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1656 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1660 /*********************************************************************
1662 * Media Ioctl callback
1664 * This routine is called whenever the user queries the status of
1665 * the interface using ifconfig.
1667 **********************************************************************/
1669 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1671 struct adapter *adapter = ifp->if_softc;
1673 INIT_DEBUGOUT("ixgbe_media_status: begin");
1674 IXGBE_CORE_LOCK(adapter);
1675 ixgbe_update_link_status(adapter);
1677 ifmr->ifm_status = IFM_AVALID;
1678 ifmr->ifm_active = IFM_ETHER;
1680 if (!adapter->link_active) {
1681 IXGBE_CORE_UNLOCK(adapter);
1685 ifmr->ifm_status |= IFM_ACTIVE;
1687 switch (adapter->link_speed) {
1688 case IXGBE_LINK_SPEED_100_FULL:
1689 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1691 case IXGBE_LINK_SPEED_1GB_FULL:
1692 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1694 case IXGBE_LINK_SPEED_10GB_FULL:
1695 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1699 IXGBE_CORE_UNLOCK(adapter);
1704 /*********************************************************************
1706 * Media Ioctl callback
1708 * This routine is called when the user changes speed/duplex using
1709 * media/mediopt option with ifconfig.
1711 **********************************************************************/
1713 ixgbe_media_change(struct ifnet * ifp)
1715 struct adapter *adapter = ifp->if_softc;
1716 struct ifmedia *ifm = &adapter->media;
1718 INIT_DEBUGOUT("ixgbe_media_change: begin");
1720 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1723 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1725 adapter->hw.phy.autoneg_advertised =
1726 IXGBE_LINK_SPEED_100_FULL |
1727 IXGBE_LINK_SPEED_1GB_FULL |
1728 IXGBE_LINK_SPEED_10GB_FULL;
1731 device_printf(adapter->dev, "Only auto media type\n");
1738 /*********************************************************************
1740 * This routine maps the mbufs to tx descriptors, allowing the
1741 * TX engine to transmit the packets.
1742 * - return 0 on success, positive on failure
1744 **********************************************************************/
1747 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1749 struct adapter *adapter = txr->adapter;
1750 u32 olinfo_status = 0, cmd_type_len;
1752 int i, j, error, nsegs;
1753 int first, last = 0;
1754 struct mbuf *m_head;
1755 bus_dma_segment_t segs[1];
1757 struct ixgbe_tx_buf *txbuf;
1758 union ixgbe_adv_tx_desc *txd = NULL;
1762 /* Basic descriptor defines */
1763 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1764 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1766 if (m_head->m_flags & M_VLANTAG)
1767 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1770 * Important to capture the first descriptor
1771 * used because it will contain the index of
1772 * the one we tell the hardware to report back
1774 first = txr->next_avail_desc;
1775 txbuf = &txr->tx_buffers[first];
1779 * Map the packet for DMA.
1781 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1782 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1784 if (error == EFBIG) {
1787 m = m_defrag(*m_headp, MB_DONTWAIT);
1789 adapter->mbuf_defrag_failed++;
1797 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1798 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1800 if (error == ENOMEM) {
1801 adapter->no_tx_dma_setup++;
1803 } else if (error != 0) {
1804 adapter->no_tx_dma_setup++;
1809 } else if (error == ENOMEM) {
1810 adapter->no_tx_dma_setup++;
1812 } else if (error != 0) {
1813 adapter->no_tx_dma_setup++;
1819 /* Make certain there are enough descriptors */
1820 if (nsegs > txr->tx_avail - 2) {
1821 txr->no_desc_avail++;
1828 ** Set up the appropriate offload context
1829 ** this becomes the first descriptor of
1833 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1834 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1835 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1836 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1837 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1841 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1843 if (ixgbe_tx_ctx_setup(txr, m_head))
1844 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1846 #ifdef IXGBE_IEEE1588
1847 /* This is changing soon to an mtag detection */
1848 if (we detect this mbuf has a TSTAMP mtag)
1849 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1853 /* Do the flow director magic */
1854 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1856 if (txr->atr_count >= atr_sample_rate) {
1857 ixgbe_atr(txr, m_head);
1862 /* Record payload length */
1864 olinfo_status |= m_head->m_pkthdr.len <<
1865 IXGBE_ADVTXD_PAYLEN_SHIFT;
1867 i = txr->next_avail_desc;
1868 for (j = 0; j < nsegs; j++) {
1872 txbuf = &txr->tx_buffers[i];
1873 txd = &txr->tx_base[i];
1874 seglen = segs[j].ds_len;
1875 segaddr = htole64(segs[j].ds_addr);
1877 txd->read.buffer_addr = segaddr;
1878 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1879 cmd_type_len |seglen);
1880 txd->read.olinfo_status = htole32(olinfo_status);
1881 last = i; /* descriptor that will get completion IRQ */
1883 if (++i == adapter->num_tx_desc)
1886 txbuf->m_head = NULL;
1887 txbuf->eop_index = -1;
1890 txd->read.cmd_type_len |=
1891 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1892 txr->tx_avail -= nsegs;
1893 txr->next_avail_desc = i;
1895 txbuf->m_head = m_head;
1896 /* Swap the dma map between the first and last descriptor */
1897 txr->tx_buffers[first].map = txbuf->map;
1899 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1901 /* Set the index of the descriptor that will be marked done */
1902 txbuf = &txr->tx_buffers[first];
1903 txbuf->eop_index = last;
1905 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1906 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1908 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1909 * hardware that this frame is available to transmit.
1911 ++txr->total_packets;
1912 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1917 bus_dmamap_unload(txr->txtag, txbuf->map);
1923 ixgbe_set_promisc(struct adapter *adapter)
1926 struct ifnet *ifp = adapter->ifp;
1928 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1929 reg_rctl &= (~IXGBE_FCTRL_UPE);
1930 reg_rctl &= (~IXGBE_FCTRL_MPE);
1931 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1933 if (ifp->if_flags & IFF_PROMISC) {
1934 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1935 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1936 } else if (ifp->if_flags & IFF_ALLMULTI) {
1937 reg_rctl |= IXGBE_FCTRL_MPE;
1938 reg_rctl &= ~IXGBE_FCTRL_UPE;
1939 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1945 /*********************************************************************
1948 * This routine is called whenever multicast address list is updated.
1950 **********************************************************************/
1951 #define IXGBE_RAR_ENTRIES 16
1954 ixgbe_set_multi(struct adapter *adapter)
1959 struct ifmultiaddr *ifma;
1961 struct ifnet *ifp = adapter->ifp;
1963 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1966 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1967 MAX_NUM_MULTICAST_ADDRESSES);
1969 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1970 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1971 if (ifp->if_flags & IFF_PROMISC)
1972 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1973 else if (ifp->if_flags & IFF_ALLMULTI) {
1974 fctrl |= IXGBE_FCTRL_MPE;
1975 fctrl &= ~IXGBE_FCTRL_UPE;
1977 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1979 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1981 spin_lock(&adapter->mcast_spin);
1982 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1983 if (ifma->ifma_addr->sa_family != AF_LINK)
1985 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1986 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1987 IXGBE_ETH_LENGTH_OF_ADDRESS);
1990 spin_unlock(&adapter->mcast_spin);
1993 ixgbe_update_mc_addr_list(&adapter->hw,
1994 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
2000 * This is an iterator function now needed by the multicast
2001 * shared code. It simply feeds the shared code routine the
2002 * addresses in the array of ixgbe_set_multi() one by one.
2005 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2007 u8 *addr = *update_ptr;
2011 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2012 *update_ptr = newptr;
2017 /*********************************************************************
2020 * This routine checks for link status,updates statistics,
2021 * and runs the watchdog check.
2023 **********************************************************************/
2026 ixgbe_local_timer(void *arg)
2028 struct adapter *adapter = arg;
2029 device_t dev = adapter->dev;
2030 struct ifnet *ifp = adapter->ifp;
2031 struct ix_queue *que = adapter->queues;
2032 struct tx_ring *txr = adapter->tx_rings;
2033 int hung, busy, paused;
2035 lockmgr(&adapter->core_lock, LK_EXCLUSIVE);
2036 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2037 hung = busy = paused = 0;
2039 /* Check for pluggable optics */
2040 if (adapter->sfp_probe)
2041 if (!ixgbe_sfp_probe(adapter))
2042 goto out; /* Nothing to do */
2044 ixgbe_update_link_status(adapter);
2045 ixgbe_update_stats_counters(adapter);
2048 * If the interface has been paused
2049 * then don't do the watchdog check
2051 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2055 ** Check the TX queues status
2056 ** - central locked handling of OACTIVE
2057 ** - watchdog only if all queues show hung
2059 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2060 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2063 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2065 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2066 taskqueue_enqueue(que->tq, &que->que_task);
2068 /* Only truely watchdog if all queues show hung */
2069 if (hung == adapter->num_queues)
2071 /* Only turn off the stack flow when ALL are depleted */
2072 if (busy == adapter->num_queues)
2073 ifp->if_flags |= IFF_OACTIVE;
2074 else if ((ifp->if_flags & IFF_OACTIVE) &&
2075 (busy < adapter->num_queues))
2076 ifp->if_flags &= ~IFF_OACTIVE;
2079 ixgbe_rearm_queues(adapter, adapter->que_mask);
2080 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2081 lockmgr(&adapter->core_lock, LK_RELEASE);
2085 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2086 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2087 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2088 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2089 device_printf(dev,"TX(%d) desc avail = %d,"
2090 "Next TX to Clean = %d\n",
2091 txr->me, txr->tx_avail, txr->next_to_clean);
2092 adapter->ifp->if_flags &= ~IFF_RUNNING;
2093 adapter->watchdog_events++;
2094 ixgbe_init_locked(adapter);
2096 lockmgr(&adapter->core_lock, LK_RELEASE);
2100 ** Note: this routine updates the OS on the link state
2101 ** the real check of the hardware only happens with
2102 ** a link interrupt.
2105 ixgbe_update_link_status(struct adapter *adapter)
2107 struct ifnet *ifp = adapter->ifp;
2108 struct tx_ring *txr = adapter->tx_rings;
2109 device_t dev = adapter->dev;
2112 if (adapter->link_up){
2113 if (adapter->link_active == FALSE) {
2115 device_printf(dev,"Link is up %d Gbps %s \n",
2116 ((adapter->link_speed == 128)? 10:1),
2118 adapter->link_active = TRUE;
2119 /* Update any Flow Control changes */
2120 ixgbe_fc_enable(&adapter->hw);
2121 ifp->if_link_state = LINK_STATE_UP;
2122 if_link_state_change(ifp);
2124 } else { /* Link down */
2125 if (adapter->link_active == TRUE) {
2127 device_printf(dev,"Link is Down\n");
2128 ifp->if_link_state = LINK_STATE_DOWN;
2129 if_link_state_change(ifp);
2130 adapter->link_active = FALSE;
2131 for (int i = 0; i < adapter->num_queues;
2133 txr->queue_status = IXGBE_QUEUE_IDLE;
2141 /*********************************************************************
2143 * This routine disables all traffic on the adapter by issuing a
2144 * global reset on the MAC and deallocates TX/RX buffers.
2146 **********************************************************************/
2149 ixgbe_stop(void *arg)
2152 struct adapter *adapter = arg;
2153 struct ixgbe_hw *hw = &adapter->hw;
2156 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2158 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2159 ixgbe_disable_intr(adapter);
2160 callout_stop(&adapter->timer);
2162 /* Let the stack know...*/
2163 ifp->if_flags &= ~IFF_RUNNING;
2164 ifp->if_flags |= IFF_OACTIVE;
2167 hw->adapter_stopped = FALSE;
2168 ixgbe_stop_adapter(hw);
2169 /* Turn off the laser */
2170 if (hw->phy.multispeed_fiber)
2171 ixgbe_disable_tx_laser(hw);
2173 /* reprogram the RAR[0] in case user changed it. */
2174 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2180 /*********************************************************************
2182 * Determine hardware revision.
2184 **********************************************************************/
2186 ixgbe_identify_hardware(struct adapter *adapter)
2188 device_t dev = adapter->dev;
2189 struct ixgbe_hw *hw = &adapter->hw;
2191 /* Save off the information about this board */
2192 hw->vendor_id = pci_get_vendor(dev);
2193 hw->device_id = pci_get_device(dev);
2194 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2195 hw->subsystem_vendor_id =
2196 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2197 hw->subsystem_device_id =
2198 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2200 /* We need this here to set the num_segs below */
2201 ixgbe_set_mac_type(hw);
2203 /* Pick up the 82599 and VF settings */
2204 if (hw->mac.type != ixgbe_mac_82598EB) {
2205 hw->phy.smart_speed = ixgbe_smart_speed;
2206 adapter->num_segs = IXGBE_82599_SCATTER;
2208 adapter->num_segs = IXGBE_82598_SCATTER;
2213 /*********************************************************************
2215 * Determine optic type
2217 **********************************************************************/
2219 ixgbe_setup_optics(struct adapter *adapter)
2221 struct ixgbe_hw *hw = &adapter->hw;
2224 layer = ixgbe_get_supported_physical_layer(hw);
2226 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2227 adapter->optics = IFM_10G_T;
2231 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2232 adapter->optics = IFM_1000_T;
2236 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2237 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2238 adapter->optics = IFM_10G_LR;
2242 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2243 adapter->optics = IFM_10G_SR;
2247 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2248 adapter->optics = IFM_10G_TWINAX;
2252 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2253 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2254 adapter->optics = IFM_10G_CX4;
2258 /* If we get here just set the default */
2259 adapter->optics = IFM_ETHER | IFM_AUTO;
2263 /*********************************************************************
2265 * Setup the Legacy or MSI Interrupt handler
2267 **********************************************************************/
2269 ixgbe_allocate_legacy(struct adapter *adapter)
2271 device_t dev = adapter->dev;
2272 struct ix_queue *que = adapter->queues;
2274 unsigned int intr_flags;
2277 if (adapter->msix == 1)
2280 /* Try allocating a MSI interrupt first */
2281 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2284 /* We allocate a single interrupt resource */
2285 adapter->res = bus_alloc_resource_any(dev,
2286 SYS_RES_IRQ, &rid, intr_flags);
2287 if (adapter->res == NULL) {
2288 device_printf(dev, "Unable to allocate bus resource: "
2294 * Try allocating a fast interrupt and the associated deferred
2295 * processing contexts.
2297 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2298 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2299 taskqueue_thread_enqueue, &que->tq);
2300 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2301 device_get_nameunit(adapter->dev));
2303 /* Tasklets for Link, SFP and Multispeed Fiber */
2304 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2305 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2306 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2308 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2310 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2311 taskqueue_thread_enqueue, &adapter->tq);
2312 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2313 device_get_nameunit(adapter->dev));
2315 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2316 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2317 device_printf(dev, "Failed to register fast interrupt "
2318 "handler: %d\n", error);
2319 taskqueue_free(que->tq);
2320 taskqueue_free(adapter->tq);
2325 /* For simplicity in the handlers */
2326 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2332 /*********************************************************************
2334 * Setup MSIX Interrupt resources and handlers
2336 **********************************************************************/
2338 ixgbe_allocate_msix(struct adapter *adapter)
2340 device_t dev = adapter->dev;
2341 struct ix_queue *que = adapter->queues;
2342 int error, rid, vector = 0;
2344 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2346 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2347 RF_SHAREABLE | RF_ACTIVE);
2348 if (que->res == NULL) {
2349 device_printf(dev,"Unable to allocate"
2350 " bus resource: que interrupt [%d]\n", vector);
2353 /* Set the handler function */
2354 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2355 ixgbe_msix_que, que, &que->tag, &que->serializer);
2358 device_printf(dev, "Failed to register QUE handler");
2361 #if 0 /* __FreeBSD_version >= 800504 */
2362 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2365 adapter->que_mask |= (u64)(1 << que->msix);
2367 ** Bind the msix vector, and thus the
2368 ** ring to the corresponding cpu.
2371 if (adapter->num_queues > 1)
2372 bus_bind_intr(dev, que->res, i);
2375 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2376 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2377 taskqueue_thread_enqueue, &que->tq);
2378 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2379 device_get_nameunit(adapter->dev));
2384 adapter->res = bus_alloc_resource_any(dev,
2385 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2386 if (!adapter->res) {
2387 device_printf(dev,"Unable to allocate"
2388 " bus resource: Link interrupt [%d]\n", rid);
2391 /* Set the link handler function */
2392 error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2393 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2395 adapter->res = NULL;
2396 device_printf(dev, "Failed to register LINK handler");
2399 #if 0 /* __FreeBSD_version >= 800504 */
2400 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2402 adapter->linkvec = vector;
2403 /* Tasklets for Link, SFP and Multispeed Fiber */
2404 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2405 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2406 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2408 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2410 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2411 taskqueue_thread_enqueue, &adapter->tq);
2412 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2413 device_get_nameunit(adapter->dev));
2418 #if 0 /* HAVE_MSIX */
2420 * Setup Either MSI/X or MSI
2423 ixgbe_setup_msix(struct adapter *adapter)
2425 device_t dev = adapter->dev;
2426 int rid, want, queues, msgs;
2428 /* Override by tuneable */
2429 if (ixgbe_enable_msix == 0)
2432 /* First try MSI/X */
2433 rid = PCIR_BAR(MSIX_82598_BAR);
2434 adapter->msix_mem = bus_alloc_resource_any(dev,
2435 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2436 if (!adapter->msix_mem) {
2437 rid += 4; /* 82599 maps in higher BAR */
2438 adapter->msix_mem = bus_alloc_resource_any(dev,
2439 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2441 if (!adapter->msix_mem) {
2442 /* May not be enabled */
2443 device_printf(adapter->dev,
2444 "Unable to map MSIX table \n");
2448 msgs = pci_msix_count(dev);
2449 if (msgs == 0) { /* system has msix disabled */
2450 bus_release_resource(dev, SYS_RES_MEMORY,
2451 rid, adapter->msix_mem);
2452 adapter->msix_mem = NULL;
2456 /* Figure out a reasonable auto config value */
2457 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2459 if (ixgbe_num_queues != 0)
2460 queues = ixgbe_num_queues;
2461 /* Set max queues to 8 when autoconfiguring */
2462 else if ((ixgbe_num_queues == 0) && (queues > 8))
2466 ** Want one vector (RX/TX pair) per queue
2467 ** plus an additional for Link.
2473 device_printf(adapter->dev,
2474 "MSIX Configuration Problem, "
2475 "%d vectors but %d queues wanted!\n",
2477 return (0); /* Will go to Legacy setup */
2479 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2480 device_printf(adapter->dev,
2481 "Using MSIX interrupts with %d vectors\n", msgs);
2482 adapter->num_queues = queues;
2486 msgs = pci_msi_count(dev);
2487 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2488 device_printf(adapter->dev,"Using an MSI interrupt\n");
2490 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2497 ixgbe_allocate_pci_resources(struct adapter *adapter)
2500 device_t dev = adapter->dev;
2503 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2506 if (!(adapter->pci_mem)) {
2507 device_printf(dev,"Unable to allocate bus resource: memory\n");
2511 adapter->osdep.mem_bus_space_tag =
2512 rman_get_bustag(adapter->pci_mem);
2513 adapter->osdep.mem_bus_space_handle =
2514 rman_get_bushandle(adapter->pci_mem);
2515 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2517 /* Legacy defaults */
2518 adapter->num_queues = 1;
2519 adapter->hw.back = &adapter->osdep;
2522 ** Now setup MSI or MSI/X, should
2523 ** return us the number of supported
2524 ** vectors. (Will be 1 for MSI)
2526 #if 0 /* HAVE_MSIX */
2527 adapter->msix = ixgbe_setup_msix(adapter);
2533 ixgbe_free_pci_resources(struct adapter * adapter)
2535 struct ix_queue *que = adapter->queues;
2536 device_t dev = adapter->dev;
2539 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2540 memrid = PCIR_BAR(MSIX_82598_BAR);
2542 memrid = PCIR_BAR(MSIX_82599_BAR);
2545 ** There is a slight possibility of a failure mode
2546 ** in attach that will result in entering this function
2547 ** before interrupt resources have been initialized, and
2548 ** in that case we do not want to execute the loops below
2549 ** We can detect this reliably by the state of the adapter
2552 if (adapter->res == NULL)
2556 ** Release all msix queue resources:
2558 for (int i = 0; i < adapter->num_queues; i++, que++) {
2559 rid = que->msix + 1;
2560 if (que->tag != NULL) {
2561 bus_teardown_intr(dev, que->res, que->tag);
2564 if (que->res != NULL)
2565 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2569 /* Clean the Legacy or Link interrupt last */
2570 if (adapter->linkvec) /* we are doing MSIX */
2571 rid = adapter->linkvec + 1;
2573 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2575 if (adapter->tag != NULL) {
2576 bus_teardown_intr(dev, adapter->res, adapter->tag);
2577 adapter->tag = NULL;
2579 if (adapter->res != NULL)
2580 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2581 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2582 pci_release_msi(adapter->dev);
2586 pci_release_msi(dev);
2588 if (adapter->msix_mem != NULL)
2589 bus_release_resource(dev, SYS_RES_MEMORY,
2590 memrid, adapter->msix_mem);
2592 if (adapter->pci_mem != NULL)
2593 bus_release_resource(dev, SYS_RES_MEMORY,
2594 PCIR_BAR(0), adapter->pci_mem);
2599 /*********************************************************************
2601 * Setup networking device structure and register an interface.
2603 **********************************************************************/
2605 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2607 struct ixgbe_hw *hw = &adapter->hw;
2610 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2612 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2614 device_printf(dev, "can not allocate ifnet structure\n");
2617 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2618 ifp->if_baudrate = 1000000000;
2619 ifp->if_init = ixgbe_init;
2620 ifp->if_softc = adapter;
2621 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2622 ifp->if_ioctl = ixgbe_ioctl;
2623 ifp->if_start = ixgbe_start;
2624 #if 0 /* __FreeBSD_version >= 800000 */
2625 ifp->if_transmit = ixgbe_mq_start;
2626 ifp->if_qflush = ixgbe_qflush;
2628 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2630 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2632 adapter->max_frame_size =
2633 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2636 * Tell the upper layer(s) we support long frames.
2638 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2641 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2643 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2644 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2645 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2650 ifp->if_capenable = ifp->if_capabilities;
2652 /* Don't enable LRO by default */
2654 ifp->if_capabilities |= IFCAP_LRO;
2658 ** Don't turn this on by default, if vlans are
2659 ** created on another pseudo device (eg. lagg)
2660 ** then vlan events are not passed thru, breaking
2661 ** operation, but with HW FILTER off it works. If
2662 ** using vlans directly on the ixgbe driver you can
2663 ** enable this and get full hardware tag filtering.
2665 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2668 * Specify the media types supported by this adapter and register
2669 * callbacks to update media and link information
2671 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2672 ixgbe_media_status);
2673 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2674 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2675 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2676 ifmedia_add(&adapter->media,
2677 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2678 ifmedia_add(&adapter->media,
2679 IFM_ETHER | IFM_1000_T, 0, NULL);
2681 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2682 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2688 ixgbe_config_link(struct adapter *adapter)
2690 struct ixgbe_hw *hw = &adapter->hw;
2691 u32 autoneg, err = 0;
2692 bool sfp, negotiate;
2694 sfp = ixgbe_is_sfp(hw);
2697 if (hw->phy.multispeed_fiber) {
2698 hw->mac.ops.setup_sfp(hw);
2699 ixgbe_enable_tx_laser(hw);
2700 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2702 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2704 if (hw->mac.ops.check_link)
2705 err = ixgbe_check_link(hw, &autoneg,
2706 &adapter->link_up, FALSE);
2709 autoneg = hw->phy.autoneg_advertised;
2710 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2711 err = hw->mac.ops.get_link_capabilities(hw,
2712 &autoneg, &negotiate);
2715 if (hw->mac.ops.setup_link)
2716 err = hw->mac.ops.setup_link(hw, autoneg,
2717 negotiate, adapter->link_up);
2723 /********************************************************************
2724 * Manage DMA'able memory.
2725 *******************************************************************/
2727 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2731 *(bus_addr_t *) arg = segs->ds_addr;
2736 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2737 struct ixgbe_dma_alloc *dma, int mapflags)
2739 device_t dev = adapter->dev;
2742 r = bus_dma_tag_create(NULL, /* parent */
2743 DBA_ALIGN, 0, /* alignment, bounds */
2744 BUS_SPACE_MAXADDR, /* lowaddr */
2745 BUS_SPACE_MAXADDR, /* highaddr */
2746 NULL, NULL, /* filter, filterarg */
2749 size, /* maxsegsize */
2750 BUS_DMA_ALLOCNOW, /* flags */
2753 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2757 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2758 BUS_DMA_NOWAIT, &dma->dma_map);
2760 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2764 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2768 mapflags | BUS_DMA_NOWAIT);
2770 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2774 dma->dma_size = size;
2777 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2779 bus_dma_tag_destroy(dma->dma_tag);
2781 dma->dma_map = NULL;
2782 dma->dma_tag = NULL;
2787 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2789 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2790 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2791 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2792 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2793 bus_dma_tag_destroy(dma->dma_tag);
2797 /*********************************************************************
2799 * Allocate memory for the transmit and receive rings, and then
2800 * the descriptors associated with each, called only once at attach.
2802 **********************************************************************/
2804 ixgbe_allocate_queues(struct adapter *adapter)
2806 device_t dev = adapter->dev;
2807 struct ix_queue *que;
2808 struct tx_ring *txr;
2809 struct rx_ring *rxr;
2810 int rsize, tsize, error = IXGBE_SUCCESS;
2811 int txconf = 0, rxconf = 0;
2813 /* First allocate the top level queue structs */
2814 if (!(adapter->queues =
2815 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2816 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2817 device_printf(dev, "Unable to allocate queue memory\n");
2822 /* First allocate the TX ring struct memory */
2823 if (!(adapter->tx_rings =
2824 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2825 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2826 device_printf(dev, "Unable to allocate TX ring memory\n");
2831 /* Next allocate the RX */
2832 if (!(adapter->rx_rings =
2833 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2834 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2835 device_printf(dev, "Unable to allocate RX ring memory\n");
2840 /* For the ring itself */
2841 tsize = roundup2(adapter->num_tx_desc *
2842 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2845 * Now set up the TX queues, txconf is needed to handle the
2846 * possibility that things fail midcourse and we need to
2847 * undo memory gracefully
2849 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2850 /* Set up some basics */
2851 txr = &adapter->tx_rings[i];
2852 txr->adapter = adapter;
2855 /* Initialize the TX side lock */
2856 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2857 device_get_nameunit(dev), txr->me);
2858 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2860 if (ixgbe_dma_malloc(adapter, tsize,
2861 &txr->txdma, BUS_DMA_NOWAIT)) {
2863 "Unable to allocate TX Descriptor memory\n");
2867 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2868 bzero((void *)txr->tx_base, tsize);
2870 /* Now allocate transmit buffers for the ring */
2871 if (ixgbe_allocate_transmit_buffers(txr)) {
2873 "Critical Failure setting up transmit buffers\n");
2877 #if 0 /* __FreeBSD_version >= 800000 */
2878 /* Allocate a buf ring */
2879 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2880 M_WAITOK, &txr->tx_mtx);
2881 if (txr->br == NULL) {
2883 "Critical Failure setting up buf ring\n");
2891 * Next the RX queues...
2893 rsize = roundup2(adapter->num_rx_desc *
2894 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2895 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2896 rxr = &adapter->rx_rings[i];
2897 /* Set up some basics */
2898 rxr->adapter = adapter;
2901 /* Initialize the RX side lock */
2902 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2903 device_get_nameunit(dev), rxr->me);
2904 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2906 if (ixgbe_dma_malloc(adapter, rsize,
2907 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2909 "Unable to allocate RxDescriptor memory\n");
2913 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2914 bzero((void *)rxr->rx_base, rsize);
2916 /* Allocate receive buffers for the ring*/
2917 if (ixgbe_allocate_receive_buffers(rxr)) {
2919 "Critical Failure setting up receive buffers\n");
2926 ** Finally set up the queue holding structs
2928 for (int i = 0; i < adapter->num_queues; i++) {
2929 que = &adapter->queues[i];
2930 que->adapter = adapter;
2931 que->txr = &adapter->tx_rings[i];
2932 que->rxr = &adapter->rx_rings[i];
2938 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2939 ixgbe_dma_free(adapter, &rxr->rxdma);
2941 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2942 ixgbe_dma_free(adapter, &txr->txdma);
2943 kfree(adapter->rx_rings, M_DEVBUF);
2945 kfree(adapter->tx_rings, M_DEVBUF);
2947 kfree(adapter->queues, M_DEVBUF);
2952 /*********************************************************************
2954 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2955 * the information needed to transmit a packet on the wire. This is
2956 * called only once at attach, setup is done every reset.
2958 **********************************************************************/
2960 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2962 struct adapter *adapter = txr->adapter;
2963 device_t dev = adapter->dev;
2964 struct ixgbe_tx_buf *txbuf;
2968 * Setup DMA descriptor areas.
2970 if ((error = bus_dma_tag_create(
2972 1, 0, /* alignment, bounds */
2973 BUS_SPACE_MAXADDR, /* lowaddr */
2974 BUS_SPACE_MAXADDR, /* highaddr */
2975 NULL, NULL, /* filter, filterarg */
2976 IXGBE_TSO_SIZE, /* maxsize */
2977 adapter->num_segs, /* nsegments */
2978 PAGE_SIZE, /* maxsegsize */
2981 device_printf(dev,"Unable to allocate TX DMA tag\n");
2985 if (!(txr->tx_buffers =
2986 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2987 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2988 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2993 /* Create the descriptor buffer dma maps */
2994 txbuf = txr->tx_buffers;
2995 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2996 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2998 device_printf(dev, "Unable to create TX DMA map\n");
3005 /* We free all, it handles case where we are in the middle */
3006 ixgbe_free_transmit_structures(adapter);
3010 /*********************************************************************
3012 * Initialize a transmit ring.
3014 **********************************************************************/
3016 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3018 struct adapter *adapter = txr->adapter;
3019 struct ixgbe_tx_buf *txbuf;
3022 struct netmap_adapter *na = NA(adapter->ifp);
3023 struct netmap_slot *slot;
3024 #endif /* DEV_NETMAP */
3026 /* Clear the old ring contents */
3030 * (under lock): if in netmap mode, do some consistency
3031 * checks and set slot to entry 0 of the netmap ring.
3033 slot = netmap_reset(na, NR_TX, txr->me, 0);
3034 #endif /* DEV_NETMAP */
3035 bzero((void *)txr->tx_base,
3036 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3038 txr->next_avail_desc = 0;
3039 txr->next_to_clean = 0;
3041 /* Free any existing tx buffers. */
3042 txbuf = txr->tx_buffers;
3043 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3044 if (txbuf->m_head != NULL) {
3045 bus_dmamap_sync(txr->txtag, txbuf->map,
3046 BUS_DMASYNC_POSTWRITE);
3047 bus_dmamap_unload(txr->txtag, txbuf->map);
3048 m_freem(txbuf->m_head);
3049 txbuf->m_head = NULL;
3053 * In netmap mode, set the map for the packet buffer.
3054 * NOTE: Some drivers (not this one) also need to set
3055 * the physical buffer address in the NIC ring.
3056 * Slots in the netmap ring (indexed by "si") are
3057 * kring->nkr_hwofs positions "ahead" wrt the
3058 * corresponding slot in the NIC ring. In some drivers
3059 * (not here) nkr_hwofs can be negative. Function
3060 * netmap_idx_n2k() handles wraparounds properly.
3063 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3064 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3066 #endif /* DEV_NETMAP */
3067 /* Clear the EOP index */
3068 txbuf->eop_index = -1;
3072 /* Set the rate at which we sample packets */
3073 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3074 txr->atr_sample = atr_sample_rate;
3077 /* Set number of descriptors available */
3078 txr->tx_avail = adapter->num_tx_desc;
3080 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3081 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3082 IXGBE_TX_UNLOCK(txr);
3085 /*********************************************************************
3087 * Initialize all transmit rings.
3089 **********************************************************************/
3091 ixgbe_setup_transmit_structures(struct adapter *adapter)
3093 struct tx_ring *txr = adapter->tx_rings;
3095 for (int i = 0; i < adapter->num_queues; i++, txr++)
3096 ixgbe_setup_transmit_ring(txr);
3101 /*********************************************************************
3103 * Enable transmit unit.
3105 **********************************************************************/
3107 ixgbe_initialize_transmit_units(struct adapter *adapter)
3109 struct tx_ring *txr = adapter->tx_rings;
3110 struct ixgbe_hw *hw = &adapter->hw;
3112 /* Setup the Base and Length of the Tx Descriptor Ring */
3114 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3115 u64 tdba = txr->txdma.dma_paddr;
3118 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3119 (tdba & 0x00000000ffffffffULL));
3120 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3121 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3122 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3124 /* Setup the HW Tx Head and Tail descriptor pointers */
3125 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3126 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3128 /* Setup Transmit Descriptor Cmd Settings */
3129 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3130 txr->queue_status = IXGBE_QUEUE_IDLE;
3132 /* Disable Head Writeback */
3133 switch (hw->mac.type) {
3134 case ixgbe_mac_82598EB:
3135 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3137 case ixgbe_mac_82599EB:
3138 case ixgbe_mac_X540:
3140 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3143 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3144 switch (hw->mac.type) {
3145 case ixgbe_mac_82598EB:
3146 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3148 case ixgbe_mac_82599EB:
3149 case ixgbe_mac_X540:
3151 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3157 if (hw->mac.type != ixgbe_mac_82598EB) {
3158 u32 dmatxctl, rttdcs;
3159 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3160 dmatxctl |= IXGBE_DMATXCTL_TE;
3161 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3162 /* Disable arbiter to set MTQC */
3163 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3164 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3165 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3166 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3167 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3168 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3174 /*********************************************************************
3176 * Free all transmit rings.
3178 **********************************************************************/
3180 ixgbe_free_transmit_structures(struct adapter *adapter)
3182 struct tx_ring *txr = adapter->tx_rings;
3184 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3186 ixgbe_free_transmit_buffers(txr);
3187 ixgbe_dma_free(adapter, &txr->txdma);
3188 IXGBE_TX_UNLOCK(txr);
3189 IXGBE_TX_LOCK_DESTROY(txr);
3191 kfree(adapter->tx_rings, M_DEVBUF);
3194 /*********************************************************************
3196 * Free transmit ring related data structures.
3198 **********************************************************************/
3200 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3202 struct adapter *adapter = txr->adapter;
3203 struct ixgbe_tx_buf *tx_buffer;
3206 INIT_DEBUGOUT("free_transmit_ring: begin");
3208 if (txr->tx_buffers == NULL)
3211 tx_buffer = txr->tx_buffers;
3212 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3213 if (tx_buffer->m_head != NULL) {
3214 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3215 BUS_DMASYNC_POSTWRITE);
3216 bus_dmamap_unload(txr->txtag,
3218 m_freem(tx_buffer->m_head);
3219 tx_buffer->m_head = NULL;
3220 if (tx_buffer->map != NULL) {
3221 bus_dmamap_destroy(txr->txtag,
3223 tx_buffer->map = NULL;
3225 } else if (tx_buffer->map != NULL) {
3226 bus_dmamap_unload(txr->txtag,
3228 bus_dmamap_destroy(txr->txtag,
3230 tx_buffer->map = NULL;
3233 #if 0 /* __FreeBSD_version >= 800000 */
3234 if (txr->br != NULL)
3235 buf_ring_free(txr->br, M_DEVBUF);
3237 if (txr->tx_buffers != NULL) {
3238 kfree(txr->tx_buffers, M_DEVBUF);
3239 txr->tx_buffers = NULL;
3241 if (txr->txtag != NULL) {
3242 bus_dma_tag_destroy(txr->txtag);
3248 /*********************************************************************
3250 * Advanced Context Descriptor setup for VLAN or CSUM
3252 **********************************************************************/
3255 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3257 struct adapter *adapter = txr->adapter;
3258 struct ixgbe_adv_tx_context_desc *TXD;
3259 struct ixgbe_tx_buf *tx_buffer;
3260 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3261 struct ether_vlan_header *eh;
3263 struct ip6_hdr *ip6;
3264 int ehdrlen, ip_hlen = 0;
3267 bool offload = TRUE;
3268 int ctxd = txr->next_avail_desc;
3274 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3277 tx_buffer = &txr->tx_buffers[ctxd];
3278 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3281 ** In advanced descriptors the vlan tag must
3282 ** be placed into the descriptor itself.
3285 if (mp->m_flags & M_VLANTAG) {
3286 vtag = htole16(mp->m_pkthdr.ether_vtag);
3287 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3288 } else if (offload == FALSE)
3293 * Determine where frame payload starts.
3294 * Jump over vlan headers if already present,
3295 * helpful for QinQ too.
3297 eh = mtod(mp, struct ether_vlan_header *);
3298 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3299 etype = ntohs(eh->evl_proto);
3300 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3302 etype = ntohs(eh->evl_encap_proto);
3303 ehdrlen = ETHER_HDR_LEN;
3306 /* Set the ether header length */
3307 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3311 ip = (struct ip *)(mp->m_data + ehdrlen);
3312 ip_hlen = ip->ip_hl << 2;
3314 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3316 case ETHERTYPE_IPV6:
3317 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3318 ip_hlen = sizeof(struct ip6_hdr);
3319 /* XXX-BZ this will go badly in case of ext hdrs. */
3320 ipproto = ip6->ip6_nxt;
3321 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3328 vlan_macip_lens |= ip_hlen;
3329 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3333 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3334 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3338 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3339 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3344 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3345 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3353 /* Now copy bits into descriptor */
3354 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3355 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3356 TXD->seqnum_seed = htole32(0);
3357 TXD->mss_l4len_idx = htole32(0);
3359 tx_buffer->m_head = NULL;
3360 tx_buffer->eop_index = -1;
3362 /* We've consumed the first desc, adjust counters */
3363 if (++ctxd == adapter->num_tx_desc)
3365 txr->next_avail_desc = ctxd;
3371 /**********************************************************************
3373 * Setup work for hardware segmentation offload (TSO) on
3374 * adapters using advanced tx descriptors
3376 **********************************************************************/
3379 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3382 struct adapter *adapter = txr->adapter;
3383 struct ixgbe_adv_tx_context_desc *TXD;
3384 struct ixgbe_tx_buf *tx_buffer;
3386 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3387 u16 vtag = 0, eh_type;
3390 u32 type_tucmd_mlhl = 0;
3392 u32 mss_l4len_idx = 0, len;
3393 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3394 struct ether_vlan_header *eh;
3396 struct ip6_hdr *ip6;
3405 * Determine where frame payload starts.
3406 * Jump over vlan headers if already present
3408 eh = mtod(mp, struct ether_vlan_header *);
3409 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3410 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3411 eh_type = eh->evl_proto;
3413 ehdrlen = ETHER_HDR_LEN;
3414 eh_type = eh->evl_encap_proto;
3417 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3418 len = ehdrlen + sizeof(struct tcphdr);
3419 switch (ntohs(eh_type)) {
3421 case ETHERTYPE_IPV6:
3422 if (mp->m_len < len + sizeof(struct ip6_hdr))
3424 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3425 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3426 if (ip6->ip6_nxt != IPPROTO_TCP)
3428 ip_hlen = sizeof(struct ip6_hdr);
3429 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3430 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3431 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3436 if (mp->m_len < len + sizeof(struct ip))
3438 ip = (struct ip *)(mp->m_data + ehdrlen);
3439 if (ip->ip_p != IPPROTO_TCP)
3442 ip_hlen = ip->ip_hl << 2;
3443 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3444 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3445 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3446 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3447 /* Tell transmit desc to also do IPv4 checksum. */
3448 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3452 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3453 __func__, ntohs(eh_type));
3457 ctxd = txr->next_avail_desc;
3458 tx_buffer = &txr->tx_buffers[ctxd];
3459 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3461 tcp_hlen = th->th_off << 2;
3463 /* This is used in the transmit desc in encap */
3464 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3466 /* VLAN MACLEN IPLEN */
3468 if (mp->m_flags & M_VLANTAG) {
3469 vtag = htole16(mp->m_pkthdr.ether_vtag);
3470 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3473 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3474 vlan_macip_lens |= ip_hlen;
3475 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3478 /* ADV DTYPE TUCMD */
3479 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3480 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3481 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3484 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3485 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3486 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3488 TXD->seqnum_seed = htole32(0);
3489 tx_buffer->m_head = NULL;
3490 tx_buffer->eop_index = -1;
3492 if (++ctxd == adapter->num_tx_desc)
3496 txr->next_avail_desc = ctxd;
3503 ** This routine parses packet headers so that Flow
3504 ** Director can make a hashed filter table entry
3505 ** allowing traffic flows to be identified and kept
3506 ** on the same cpu. This would be a performance
3507 ** hit, but we only do it at IXGBE_FDIR_RATE of
3511 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3513 struct adapter *adapter = txr->adapter;
3514 struct ix_queue *que;
3518 struct ether_vlan_header *eh;
3519 union ixgbe_atr_hash_dword input = {.dword = 0};
3520 union ixgbe_atr_hash_dword common = {.dword = 0};
3521 int ehdrlen, ip_hlen;
3524 eh = mtod(mp, struct ether_vlan_header *);
3525 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3526 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3527 etype = eh->evl_proto;
3529 ehdrlen = ETHER_HDR_LEN;
3530 etype = eh->evl_encap_proto;
3533 /* Only handling IPv4 */
3534 if (etype != htons(ETHERTYPE_IP))
3537 ip = (struct ip *)(mp->m_data + ehdrlen);
3538 ip_hlen = ip->ip_hl << 2;
3540 /* check if we're UDP or TCP */
3543 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3544 /* src and dst are inverted */
3545 common.port.dst ^= th->th_sport;
3546 common.port.src ^= th->th_dport;
3547 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3550 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3551 /* src and dst are inverted */
3552 common.port.dst ^= uh->uh_sport;
3553 common.port.src ^= uh->uh_dport;
3554 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3560 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3561 if (mp->m_pkthdr.ether_vtag)
3562 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3564 common.flex_bytes ^= etype;
3565 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3567 que = &adapter->queues[txr->me];
3569 ** This assumes the Rx queue and Tx
3570 ** queue are bound to the same CPU
3572 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3573 input, common, que->msix);
3575 #endif /* IXGBE_FDIR */
3577 /**********************************************************************
3579 * Examine each tx_buffer in the used queue. If the hardware is done
3580 * processing the packet then free associated resources. The
3581 * tx_buffer is put back on the free queue.
3583 **********************************************************************/
3585 ixgbe_txeof(struct tx_ring *txr)
3587 struct adapter *adapter = txr->adapter;
3588 struct ifnet *ifp = adapter->ifp;
3589 u32 first, last, done, processed;
3590 struct ixgbe_tx_buf *tx_buffer;
3591 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3593 KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3596 if (ifp->if_capenable & IFCAP_NETMAP) {
3597 struct netmap_adapter *na = NA(ifp);
3598 struct netmap_kring *kring = &na->tx_rings[txr->me];
3600 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3602 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3603 BUS_DMASYNC_POSTREAD);
3605 * In netmap mode, all the work is done in the context
3606 * of the client thread. Interrupt handlers only wake up
3607 * clients, which may be sleeping on individual rings
3608 * or on a global resource for all rings.
3609 * To implement tx interrupt mitigation, we wake up the client
3610 * thread roughly every half ring, even if the NIC interrupts
3611 * more frequently. This is implemented as follows:
3612 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3613 * the slot that should wake up the thread (nkr_num_slots
3614 * means the user thread should not be woken up);
3615 * - the driver ignores tx interrupts unless netmap_mitigate=0
3616 * or the slot has the DD bit set.
3618 * When the driver has separate locks, we need to
3619 * release and re-acquire txlock to avoid deadlocks.
3620 * XXX see if we can find a better way.
3622 if (!netmap_mitigate ||
3623 (kring->nr_kflags < kring->nkr_num_slots &&
3624 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3625 kring->nr_kflags = kring->nkr_num_slots;
3626 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3627 IXGBE_TX_UNLOCK(txr);
3628 IXGBE_CORE_LOCK(adapter);
3629 selwakeuppri(&na->tx_si, PI_NET);
3630 IXGBE_CORE_UNLOCK(adapter);
3635 #endif /* DEV_NETMAP */
3637 if (txr->tx_avail == adapter->num_tx_desc) {
3638 txr->queue_status = IXGBE_QUEUE_IDLE;
3643 first = txr->next_to_clean;
3644 tx_buffer = &txr->tx_buffers[first];
3645 /* For cleanup we just use legacy struct */
3646 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3647 last = tx_buffer->eop_index;
3650 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3653 ** Get the index of the first descriptor
3654 ** BEYOND the EOP and call that 'done'.
3655 ** I do this so the comparison in the
3656 ** inner while loop below can be simple
3658 if (++last == adapter->num_tx_desc) last = 0;
3661 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3662 BUS_DMASYNC_POSTREAD);
3664 ** Only the EOP descriptor of a packet now has the DD
3665 ** bit set, this is what we look for...
3667 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3668 /* We clean the range of the packet */
3669 while (first != done) {
3670 tx_desc->upper.data = 0;
3671 tx_desc->lower.data = 0;
3672 tx_desc->buffer_addr = 0;
3676 if (tx_buffer->m_head) {
3678 tx_buffer->m_head->m_pkthdr.len;
3679 bus_dmamap_sync(txr->txtag,
3681 BUS_DMASYNC_POSTWRITE);
3682 bus_dmamap_unload(txr->txtag,
3684 m_freem(tx_buffer->m_head);
3685 tx_buffer->m_head = NULL;
3686 tx_buffer->map = NULL;
3688 tx_buffer->eop_index = -1;
3689 txr->watchdog_time = ticks;
3691 if (++first == adapter->num_tx_desc)
3694 tx_buffer = &txr->tx_buffers[first];
3696 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3700 /* See if there is more work now */
3701 last = tx_buffer->eop_index;
3704 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3705 /* Get next done point */
3706 if (++last == adapter->num_tx_desc) last = 0;
3711 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3712 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3714 txr->next_to_clean = first;
3717 ** Watchdog calculation, we know there's
3718 ** work outstanding or the first return
3719 ** would have been taken, so none processed
3720 ** for too long indicates a hang.
3722 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3723 txr->queue_status = IXGBE_QUEUE_HUNG;
3725 /* With a minimum free clear the depleted state bit. */
3726 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3727 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3729 if (txr->tx_avail == adapter->num_tx_desc) {
3730 txr->queue_status = IXGBE_QUEUE_IDLE;
3737 /*********************************************************************
3739 * Refresh mbuf buffers for RX descriptor rings
3740 * - now keeps its own state so discards due to resource
3741 * exhaustion are unnecessary, if an mbuf cannot be obtained
3742 * it just returns, keeping its placeholder, thus it can simply
3743 * be recalled to try again.
3745 **********************************************************************/
3747 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3749 struct adapter *adapter = rxr->adapter;
3750 bus_dma_segment_t hseg[1];
3751 bus_dma_segment_t pseg[1];
3752 struct ixgbe_rx_buf *rxbuf;
3753 struct mbuf *mh, *mp;
3754 int i, j, nsegs, error;
3755 bool refreshed = FALSE;
3757 i = j = rxr->next_to_refresh;
3758 /* Control the loop with one beyond */
3759 if (++j == adapter->num_rx_desc)
3762 while (j != limit) {
3763 rxbuf = &rxr->rx_buffers[i];
3764 if (rxr->hdr_split == FALSE)
3767 if (rxbuf->m_head == NULL) {
3768 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3774 mh->m_pkthdr.len = mh->m_len = MHLEN;
3776 mh->m_flags |= M_PKTHDR;
3777 /* Get the memory mapping */
3778 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3779 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3781 kprintf("Refresh mbufs: hdr dmamap load"
3782 " failure - %d\n", error);
3784 rxbuf->m_head = NULL;
3788 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3789 BUS_DMASYNC_PREREAD);
3790 rxr->rx_base[i].read.hdr_addr =
3791 htole64(hseg[0].ds_addr);
3794 if (rxbuf->m_pack == NULL) {
3795 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3796 M_PKTHDR, adapter->rx_mbuf_sz);
3802 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3803 /* Get the memory mapping */
3804 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3805 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3807 kprintf("Refresh mbufs: payload dmamap load"
3808 " failure - %d\n", error);
3810 rxbuf->m_pack = NULL;
3814 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3815 BUS_DMASYNC_PREREAD);
3816 rxr->rx_base[i].read.pkt_addr =
3817 htole64(pseg[0].ds_addr);
3820 /* Next is precalculated */
3822 rxr->next_to_refresh = i;
3823 if (++j == adapter->num_rx_desc)
3827 if (refreshed) /* Update hardware tail index */
3828 IXGBE_WRITE_REG(&adapter->hw,
3829 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3833 /*********************************************************************
3835 * Allocate memory for rx_buffer structures. Since we use one
3836 * rx_buffer per received packet, the maximum number of rx_buffer's
3837 * that we'll need is equal to the number of receive descriptors
3838 * that we've allocated.
3840 **********************************************************************/
3842 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3844 struct adapter *adapter = rxr->adapter;
3845 device_t dev = adapter->dev;
3846 struct ixgbe_rx_buf *rxbuf;
3847 int i, bsize, error;
3849 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3850 if (!(rxr->rx_buffers =
3851 (struct ixgbe_rx_buf *) kmalloc(bsize,
3852 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3853 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3858 if ((error = bus_dma_tag_create(NULL, /* parent */
3859 1, 0, /* alignment, bounds */
3860 BUS_SPACE_MAXADDR, /* lowaddr */
3861 BUS_SPACE_MAXADDR, /* highaddr */
3862 NULL, NULL, /* filter, filterarg */
3863 MSIZE, /* maxsize */
3865 MSIZE, /* maxsegsize */
3868 device_printf(dev, "Unable to create RX DMA tag\n");
3872 if ((error = bus_dma_tag_create(NULL, /* parent */
3873 1, 0, /* alignment, bounds */
3874 BUS_SPACE_MAXADDR, /* lowaddr */
3875 BUS_SPACE_MAXADDR, /* highaddr */
3876 NULL, NULL, /* filter, filterarg */
3877 MJUM16BYTES, /* maxsize */
3879 MJUM16BYTES, /* maxsegsize */
3882 device_printf(dev, "Unable to create RX DMA tag\n");
3886 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3887 rxbuf = &rxr->rx_buffers[i];
3888 error = bus_dmamap_create(rxr->htag,
3889 BUS_DMA_NOWAIT, &rxbuf->hmap);
3891 device_printf(dev, "Unable to create RX head map\n");
3894 error = bus_dmamap_create(rxr->ptag,
3895 BUS_DMA_NOWAIT, &rxbuf->pmap);
3897 device_printf(dev, "Unable to create RX pkt map\n");
3905 /* Frees all, but can handle partial completion */
3906 ixgbe_free_receive_structures(adapter);
3911 ** Used to detect a descriptor that has
3912 ** been merged by Hardware RSC.
3915 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3917 return (le32toh(rx->wb.lower.lo_dword.data) &
3918 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3921 /*********************************************************************
3923 * Initialize Hardware RSC (LRO) feature on 82599
3924 * for an RX ring, this is toggled by the LRO capability
3925 * even though it is transparent to the stack.
3927 **********************************************************************/
3930 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3932 struct adapter *adapter = rxr->adapter;
3933 struct ixgbe_hw *hw = &adapter->hw;
3934 u32 rscctrl, rdrxctl;
3936 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3937 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3938 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3939 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3940 #endif /* DEV_NETMAP */
3941 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3942 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3943 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3945 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3946 rscctrl |= IXGBE_RSCCTL_RSCEN;
3948 ** Limit the total number of descriptors that
3949 ** can be combined, so it does not exceed 64K
3951 if (adapter->rx_mbuf_sz == MCLBYTES)
3952 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3953 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3954 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3955 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3956 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3957 else /* Using 16K cluster */
3958 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3960 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3962 /* Enable TCP header recognition */
3963 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3964 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3965 IXGBE_PSRTYPE_TCPHDR));
3967 /* Disable RSC for ACK packets */
3968 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3969 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3976 ixgbe_free_receive_ring(struct rx_ring *rxr)
3978 struct adapter *adapter;
3979 struct ixgbe_rx_buf *rxbuf;
3982 adapter = rxr->adapter;
3983 for (i = 0; i < adapter->num_rx_desc; i++) {
3984 rxbuf = &rxr->rx_buffers[i];
3985 if (rxbuf->m_head != NULL) {
3986 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3987 BUS_DMASYNC_POSTREAD);
3988 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3989 rxbuf->m_head->m_flags |= M_PKTHDR;
3990 m_freem(rxbuf->m_head);
3992 if (rxbuf->m_pack != NULL) {
3993 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3994 BUS_DMASYNC_POSTREAD);
3995 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3996 rxbuf->m_pack->m_flags |= M_PKTHDR;
3997 m_freem(rxbuf->m_pack);
3999 rxbuf->m_head = NULL;
4000 rxbuf->m_pack = NULL;
4005 /*********************************************************************
4007 * Initialize a receive ring and its buffers.
4009 **********************************************************************/
4011 ixgbe_setup_receive_ring(struct rx_ring *rxr)
4013 struct adapter *adapter;
4016 struct ixgbe_rx_buf *rxbuf;
4017 bus_dma_segment_t pseg[1], hseg[1];
4019 struct lro_ctrl *lro = &rxr->lro;
4021 int rsize, nsegs, error = 0;
4023 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4024 struct netmap_slot *slot;
4025 #endif /* DEV_NETMAP */
4027 adapter = rxr->adapter;
4031 /* Clear the ring contents */
4034 /* same as in ixgbe_setup_transmit_ring() */
4035 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4036 #endif /* DEV_NETMAP */
4037 rsize = roundup2(adapter->num_rx_desc *
4038 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4039 bzero((void *)rxr->rx_base, rsize);
4041 /* Free current RX buffer structs and their mbufs */
4042 ixgbe_free_receive_ring(rxr);
4044 /* Configure header split? */
4045 if (ixgbe_header_split)
4046 rxr->hdr_split = TRUE;
4048 /* Now replenish the mbufs */
4049 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4050 struct mbuf *mh, *mp;
4052 rxbuf = &rxr->rx_buffers[j];
4055 * In netmap mode, fill the map and set the buffer
4056 * address in the NIC ring, considering the offset
4057 * between the netmap and NIC rings (see comment in
4058 * ixgbe_setup_transmit_ring() ). No need to allocate
4059 * an mbuf, so end the block with a continue;
4062 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4066 addr = PNMB(slot + sj, &paddr);
4067 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4068 /* Update descriptor */
4069 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4072 #endif /* DEV_NETMAP */
4074 ** Don't allocate mbufs if not
4075 ** doing header split, its wasteful
4077 if (rxr->hdr_split == FALSE)
4080 /* First the header */
4081 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4082 if (rxbuf->m_head == NULL) {
4086 m_adj(rxbuf->m_head, ETHER_ALIGN);
4088 mh->m_len = mh->m_pkthdr.len = MHLEN;
4089 mh->m_flags |= M_PKTHDR;
4090 /* Get the memory mapping */
4091 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4092 rxbuf->hmap, rxbuf->m_head, hseg, 1,
4093 &nsegs, BUS_DMA_NOWAIT);
4095 if (error != 0) /* Nothing elegant to do here */
4097 bus_dmamap_sync(rxr->htag,
4098 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4099 /* Update descriptor */
4100 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4103 /* Now the payload cluster */
4104 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4105 M_PKTHDR, adapter->rx_mbuf_sz);
4106 if (rxbuf->m_pack == NULL) {
4111 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4112 /* Get the memory mapping */
4113 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4114 rxbuf->pmap, mp, hseg, 1,
4115 &nsegs, BUS_DMA_NOWAIT);
4118 bus_dmamap_sync(rxr->ptag,
4119 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4120 /* Update descriptor */
4121 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4125 /* Setup our descriptor indices */
4126 rxr->next_to_check = 0;
4127 rxr->next_to_refresh = 0;
4128 rxr->lro_enabled = FALSE;
4129 rxr->rx_split_packets = 0;
4131 rxr->discard = FALSE;
4132 rxr->vtag_strip = FALSE;
4134 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4135 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4138 ** Now set up the LRO interface:
4139 ** 82598 uses software LRO, the
4140 ** 82599 and X540 use a hardware assist.
4143 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4144 (ifp->if_capenable & IFCAP_RXCSUM) &&
4145 (ifp->if_capenable & IFCAP_LRO))
4146 ixgbe_setup_hw_rsc(rxr);
4147 else if (ifp->if_capenable & IFCAP_LRO) {
4148 int err = tcp_lro_init(lro);
4150 device_printf(dev, "LRO Initialization failed!\n");
4153 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4154 rxr->lro_enabled = TRUE;
4155 lro->ifp = adapter->ifp;
4159 IXGBE_RX_UNLOCK(rxr);
4163 ixgbe_free_receive_ring(rxr);
4164 IXGBE_RX_UNLOCK(rxr);
4168 /*********************************************************************
4170 * Initialize all receive rings.
4172 **********************************************************************/
4174 ixgbe_setup_receive_structures(struct adapter *adapter)
4176 struct rx_ring *rxr = adapter->rx_rings;
4179 for (j = 0; j < adapter->num_queues; j++, rxr++)
4180 if (ixgbe_setup_receive_ring(rxr))
4186 * Free RX buffers allocated so far, we will only handle
4187 * the rings that completed, the failing case will have
4188 * cleaned up for itself. 'j' failed, so its the terminus.
4190 for (int i = 0; i < j; ++i) {
4191 rxr = &adapter->rx_rings[i];
4192 ixgbe_free_receive_ring(rxr);
4198 /*********************************************************************
4200 * Setup receive registers and features.
4202 **********************************************************************/
4203 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4205 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4208 ixgbe_initialize_receive_units(struct adapter *adapter)
4210 struct rx_ring *rxr = adapter->rx_rings;
4211 struct ixgbe_hw *hw = &adapter->hw;
4212 struct ifnet *ifp = adapter->ifp;
4213 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum;
4214 u32 reta, mrqc = 0, hlreg, random[10];
4218 * Make sure receives are disabled while
4219 * setting up the descriptor ring
4221 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4222 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4223 rxctrl & ~IXGBE_RXCTRL_RXEN);
4225 /* Enable broadcasts */
4226 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4227 fctrl |= IXGBE_FCTRL_BAM;
4228 fctrl |= IXGBE_FCTRL_DPF;
4229 fctrl |= IXGBE_FCTRL_PMCF;
4230 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4232 /* Set for Jumbo Frames? */
4233 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4234 if (ifp->if_mtu > ETHERMTU)
4235 hlreg |= IXGBE_HLREG0_JUMBOEN;
4237 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4239 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4240 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4241 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4243 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4244 #endif /* DEV_NETMAP */
4245 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4247 bufsz = (adapter->rx_mbuf_sz +
4248 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4250 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4251 u64 rdba = rxr->rxdma.dma_paddr;
4253 /* Setup the Base and Length of the Rx Descriptor Ring */
4254 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4255 (rdba & 0x00000000ffffffffULL));
4256 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4257 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4258 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4260 /* Set up the SRRCTL register */
4261 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4262 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4263 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4265 if (rxr->hdr_split) {
4266 /* Use a standard mbuf for the header */
4267 srrctl |= ((IXGBE_RX_HDR <<
4268 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4269 & IXGBE_SRRCTL_BSIZEHDR_MASK);
4270 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4272 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4273 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4275 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4276 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4277 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4280 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4281 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4282 IXGBE_PSRTYPE_UDPHDR |
4283 IXGBE_PSRTYPE_IPV4HDR |
4284 IXGBE_PSRTYPE_IPV6HDR;
4285 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4288 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4291 if (adapter->num_queues > 1) {
4295 /* set up random bits */
4296 karc4rand(&random, sizeof(random));
4298 /* Set up the redirection table */
4299 for (i = 0, j = 0; i < 128; i++, j++) {
4300 if (j == adapter->num_queues) j = 0;
4301 reta = (reta << 8) | (j * 0x11);
4303 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4306 /* Now fill our hash function seeds */
4307 for (int i = 0; i < 10; i++)
4308 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4310 /* Perform hash on these packet types */
4311 mrqc = IXGBE_MRQC_RSSEN
4312 | IXGBE_MRQC_RSS_FIELD_IPV4
4313 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4314 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4315 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4316 | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4317 | IXGBE_MRQC_RSS_FIELD_IPV6
4318 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4319 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4320 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4321 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4323 /* RSS and RX IPP Checksum are mutually exclusive */
4324 rxcsum |= IXGBE_RXCSUM_PCSD;
4327 if (ifp->if_capenable & IFCAP_RXCSUM)
4328 rxcsum |= IXGBE_RXCSUM_PCSD;
4330 if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4331 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4333 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4338 /*********************************************************************
4340 * Free all receive rings.
4342 **********************************************************************/
4344 ixgbe_free_receive_structures(struct adapter *adapter)
4346 struct rx_ring *rxr = adapter->rx_rings;
4348 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4350 struct lro_ctrl *lro = &rxr->lro;
4352 ixgbe_free_receive_buffers(rxr);
4353 /* Free LRO memory */
4357 /* Free the ring memory as well */
4358 ixgbe_dma_free(adapter, &rxr->rxdma);
4361 kfree(adapter->rx_rings, M_DEVBUF);
4365 /*********************************************************************
4367 * Free receive ring data structures
4369 **********************************************************************/
4371 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4373 struct adapter *adapter = rxr->adapter;
4374 struct ixgbe_rx_buf *rxbuf;
4376 INIT_DEBUGOUT("free_receive_structures: begin");
4378 /* Cleanup any existing buffers */
4379 if (rxr->rx_buffers != NULL) {
4380 for (int i = 0; i < adapter->num_rx_desc; i++) {
4381 rxbuf = &rxr->rx_buffers[i];
4382 if (rxbuf->m_head != NULL) {
4383 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4384 BUS_DMASYNC_POSTREAD);
4385 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4386 rxbuf->m_head->m_flags |= M_PKTHDR;
4387 m_freem(rxbuf->m_head);
4389 if (rxbuf->m_pack != NULL) {
4390 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4391 BUS_DMASYNC_POSTREAD);
4392 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4393 rxbuf->m_pack->m_flags |= M_PKTHDR;
4394 m_freem(rxbuf->m_pack);
4396 rxbuf->m_head = NULL;
4397 rxbuf->m_pack = NULL;
4398 if (rxbuf->hmap != NULL) {
4399 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4402 if (rxbuf->pmap != NULL) {
4403 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4407 if (rxr->rx_buffers != NULL) {
4408 kfree(rxr->rx_buffers, M_DEVBUF);
4409 rxr->rx_buffers = NULL;
4413 if (rxr->htag != NULL) {
4414 bus_dma_tag_destroy(rxr->htag);
4417 if (rxr->ptag != NULL) {
4418 bus_dma_tag_destroy(rxr->ptag);
4425 static __inline void
4426 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4430 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4431 * should be computed by hardware. Also it should not have VLAN tag in
4432 * ethernet header. In case of IPv6 we do not yet support ext. hdrs.
4435 if (rxr->lro_enabled &&
4436 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4437 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4438 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4439 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4440 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4441 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4442 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4443 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4445 * Send to the stack if:
4446 ** - LRO not enabled, or
4447 ** - no LRO resources, or
4448 ** - lro enqueue fails
4450 if (rxr->lro.lro_cnt != 0)
4451 if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4455 IXGBE_RX_UNLOCK(rxr);
4456 (*ifp->if_input)(ifp, m);
4460 static __inline void
4461 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4463 struct ixgbe_rx_buf *rbuf;
4465 rbuf = &rxr->rx_buffers[i];
4467 if (rbuf->fmp != NULL) {/* Partial chain ? */
4468 rbuf->fmp->m_flags |= M_PKTHDR;
4474 ** With advanced descriptors the writeback
4475 ** clobbers the buffer addrs, so its easier
4476 ** to just free the existing mbufs and take
4477 ** the normal refresh path to get new buffers
4481 m_free(rbuf->m_head);
4482 rbuf->m_head = NULL;
4486 m_free(rbuf->m_pack);
4487 rbuf->m_pack = NULL;
4494 /*********************************************************************
4496 * This routine executes in interrupt context. It replenishes
4497 * the mbufs in the descriptor and sends data which has been
4498 * dma'ed into host memory to upper layer.
4500 * We loop at most count times if count is > 0, or until done if
4503 * Return TRUE for more work, FALSE for all clean.
4504 *********************************************************************/
4506 ixgbe_rxeof(struct ix_queue *que, int count)
4508 struct adapter *adapter = que->adapter;
4509 struct rx_ring *rxr = que->rxr;
4510 struct ifnet *ifp = adapter->ifp;
4512 struct lro_ctrl *lro = &rxr->lro;
4513 struct lro_entry *queued;
4515 int i, nextp, processed = 0;
4517 union ixgbe_adv_rx_desc *cur;
4518 struct ixgbe_rx_buf *rbuf, *nbuf;
4523 if (ifp->if_capenable & IFCAP_NETMAP) {
4525 * Same as the txeof routine: only wakeup clients on intr.
4526 * NKR_PENDINTR in nr_kflags is used to implement interrupt
4527 * mitigation (ixgbe_rxsync() will not look for new packets
4528 * unless NKR_PENDINTR is set).
4530 struct netmap_adapter *na = NA(ifp);
4532 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4533 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4534 IXGBE_RX_UNLOCK(rxr);
4535 IXGBE_CORE_LOCK(adapter);
4536 selwakeuppri(&na->rx_si, PI_NET);
4537 IXGBE_CORE_UNLOCK(adapter);
4540 #endif /* DEV_NETMAP */
4541 for (i = rxr->next_to_check; count != 0;) {
4542 struct mbuf *sendmp, *mh, *mp;
4544 u16 hlen, plen, hdr;
4550 /* Sync the ring. */
4551 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4552 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4554 cur = &rxr->rx_base[i];
4555 staterr = le32toh(cur->wb.upper.status_error);
4557 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4559 if ((ifp->if_flags & IFF_RUNNING) == 0)
4566 cur->wb.upper.status_error = 0;
4567 rbuf = &rxr->rx_buffers[i];
4571 plen = le16toh(cur->wb.upper.length);
4572 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4573 IXGBE_RXDADV_PKTTYPE_MASK;
4574 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4575 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4577 /* Process vlan info */
4579 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4580 vtag = le16toh(cur->wb.upper.vlan);
4583 /* Make sure bad packets are discarded */
4584 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4587 rxr->rx_discarded++;
4589 rxr->discard = FALSE;
4591 rxr->discard = TRUE;
4592 ixgbe_rx_discard(rxr, i);
4597 ** On 82599 which supports a hardware
4598 ** LRO (called HW RSC), packets need
4599 ** not be fragmented across sequential
4600 ** descriptors, rather the next descriptor
4601 ** is indicated in bits of the descriptor.
4602 ** This also means that we might proceses
4603 ** more than one packet at a time, something
4604 ** that has never been true before, it
4605 ** required eliminating global chain pointers
4606 ** in favor of what we are doing here. -jfv
4610 ** Figure out the next descriptor
4613 if (rxr->hw_rsc == TRUE) {
4614 rsc = ixgbe_rsc_count(cur);
4615 rxr->rsc_num += (rsc - 1);
4617 if (rsc) { /* Get hardware index */
4619 IXGBE_RXDADV_NEXTP_MASK) >>
4620 IXGBE_RXDADV_NEXTP_SHIFT);
4621 } else { /* Just sequential */
4623 if (nextp == adapter->num_rx_desc)
4626 nbuf = &rxr->rx_buffers[nextp];
4630 ** The header mbuf is ONLY used when header
4631 ** split is enabled, otherwise we get normal
4632 ** behavior, ie, both header and payload
4633 ** are DMA'd into the payload buffer.
4635 ** Rather than using the fmp/lmp global pointers
4636 ** we now keep the head of a packet chain in the
4637 ** buffer struct and pass this along from one
4638 ** descriptor to the next, until we get EOP.
4640 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4641 /* This must be an initial descriptor */
4642 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4643 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4644 if (hlen > IXGBE_RX_HDR)
4645 hlen = IXGBE_RX_HDR;
4647 mh->m_flags |= M_PKTHDR;
4649 mh->m_pkthdr.len = mh->m_len;
4650 /* Null buf pointer so it is refreshed */
4651 rbuf->m_head = NULL;
4653 ** Check the payload length, this
4654 ** could be zero if its a small
4660 mp->m_flags &= ~M_PKTHDR;
4662 mh->m_pkthdr.len += mp->m_len;
4663 /* Null buf pointer so it is refreshed */
4664 rbuf->m_pack = NULL;
4665 rxr->rx_split_packets++;
4668 ** Now create the forward
4669 ** chain so when complete
4673 /* stash the chain head */
4675 /* Make forward chain */
4677 mp->m_next = nbuf->m_pack;
4679 mh->m_next = nbuf->m_pack;
4681 /* Singlet, prepare to send */
4683 /* If hardware handled vtag */
4686 sendmp->m_pkthdr.ether_vtag = vtag;
4687 sendmp->m_flags |= M_VLANTAG;
4693 ** Either no header split, or a
4694 ** secondary piece of a fragmented
4699 ** See if there is a stored head
4700 ** that determines what we are
4703 rbuf->m_pack = rbuf->fmp = NULL;
4705 if (sendmp != NULL) { /* secondary frag */
4706 mp->m_flags &= ~M_PKTHDR;
4707 sendmp->m_pkthdr.len += mp->m_len;
4709 /* first desc of a non-ps chain */
4711 sendmp->m_flags |= M_PKTHDR;
4712 sendmp->m_pkthdr.len = mp->m_len;
4714 if (staterr & IXGBE_RXD_STAT_VP) {
4715 sendmp->m_pkthdr.ether_vtag = vtag;
4716 sendmp->m_flags |= M_VLANTAG;
4720 /* Pass the head pointer on */
4724 mp->m_next = nbuf->m_pack;
4728 /* Sending this frame? */
4730 sendmp->m_pkthdr.rcvif = ifp;
4733 /* capture data for AIM */
4734 rxr->bytes += sendmp->m_pkthdr.len;
4735 rxr->rx_bytes += sendmp->m_pkthdr.len;
4736 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4737 ixgbe_rx_checksum(staterr, sendmp, ptype);
4738 #if 0 /* __FreeBSD_version >= 800000 */
4739 sendmp->m_pkthdr.flowid = que->msix;
4740 sendmp->m_flags |= M_FLOWID;
4744 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4745 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4747 /* Advance our pointers to the next descriptor. */
4748 if (++i == adapter->num_rx_desc)
4751 /* Now send to the stack or do LRO */
4752 if (sendmp != NULL) {
4753 rxr->next_to_check = i;
4754 ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4755 i = rxr->next_to_check;
4758 /* Every 8 descriptors we go to refresh mbufs */
4759 if (processed == 8) {
4760 ixgbe_refresh_mbufs(rxr, i);
4765 /* Refresh any remaining buf structs */
4766 if (ixgbe_rx_unrefreshed(rxr))
4767 ixgbe_refresh_mbufs(rxr, i);
4769 rxr->next_to_check = i;
4772 * Flush any outstanding LRO work
4775 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4776 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4777 tcp_lro_flush(lro, queued);
4781 IXGBE_RX_UNLOCK(rxr);
4784 ** We still have cleaning to do?
4785 ** Schedule another interrupt if so.
4787 if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4788 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4796 /*********************************************************************
4798 * Verify that the hardware indicated that the checksum is valid.
4799 * Inform the stack about the status of checksum so that stack
4800 * doesn't spend time verifying the checksum.
4802 *********************************************************************/
4804 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4806 u16 status = (u16) staterr;
4807 u8 errors = (u8) (staterr >> 24);
4810 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4811 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4814 if (status & IXGBE_RXD_STAT_IPCS) {
4815 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4816 /* IP Checksum Good */
4817 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4818 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4821 mp->m_pkthdr.csum_flags = 0;
4823 if (status & IXGBE_RXD_STAT_L4CS) {
4824 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4827 type = CSUM_SCTP_VALID;
4829 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4830 mp->m_pkthdr.csum_flags |= type;
4832 mp->m_pkthdr.csum_data = htons(0xffff);
4840 ** This routine is run via an vlan config EVENT,
4841 ** it enables us to use the HW Filter table since
4842 ** we can get the vlan id. This just creates the
4843 ** entry in the soft version of the VFTA, init will
4844 ** repopulate the real table.
4847 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4849 struct adapter *adapter = ifp->if_softc;
4852 if (ifp->if_softc != arg) /* Not our event */
4855 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4858 IXGBE_CORE_LOCK(adapter);
4859 index = (vtag >> 5) & 0x7F;
4861 adapter->shadow_vfta[index] |= (1 << bit);
4862 ++adapter->num_vlans;
4863 ixgbe_init_locked(adapter);
4864 IXGBE_CORE_UNLOCK(adapter);
4868 ** This routine is run via an vlan
4869 ** unconfig EVENT, remove our entry
4870 ** in the soft vfta.
4873 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4875 struct adapter *adapter = ifp->if_softc;
4878 if (ifp->if_softc != arg)
4881 if ((vtag == 0) || (vtag > 4095)) /* Invalid */
4884 IXGBE_CORE_LOCK(adapter);
4885 index = (vtag >> 5) & 0x7F;
4887 adapter->shadow_vfta[index] &= ~(1 << bit);
4888 --adapter->num_vlans;
4889 /* Re-init to load the changes */
4890 ixgbe_init_locked(adapter);
4891 IXGBE_CORE_UNLOCK(adapter);
4895 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4898 struct ifnet *ifp = adapter->ifp;
4899 struct ixgbe_hw *hw = &adapter->hw;
4900 struct rx_ring *rxr;
4904 ** We get here thru init_locked, meaning
4905 ** a soft reset, this has already cleared
4906 ** the VFTA and other state, so if there
4907 ** have been no vlan's registered do nothing.
4909 if (adapter->num_vlans == 0)
4913 ** A soft reset zero's out the VFTA, so
4914 ** we need to repopulate it now.
4916 for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4917 if (adapter->shadow_vfta[i] != 0)
4918 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4919 adapter->shadow_vfta[i]);
4921 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4922 /* Enable the Filter Table if enabled */
4923 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4924 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4925 ctrl |= IXGBE_VLNCTRL_VFE;
4927 if (hw->mac.type == ixgbe_mac_82598EB)
4928 ctrl |= IXGBE_VLNCTRL_VME;
4929 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4931 /* Setup the queues for vlans */
4932 for (int i = 0; i < adapter->num_queues; i++) {
4933 rxr = &adapter->rx_rings[i];
4934 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4935 if (hw->mac.type != ixgbe_mac_82598EB) {
4936 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4937 ctrl |= IXGBE_RXDCTL_VME;
4938 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4940 rxr->vtag_strip = TRUE;
4946 ixgbe_enable_intr(struct adapter *adapter)
4948 struct ixgbe_hw *hw = &adapter->hw;
4949 struct ix_queue *que = adapter->queues;
4950 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4953 /* Enable Fan Failure detection */
4954 if (hw->device_id == IXGBE_DEV_ID_82598AT)
4955 mask |= IXGBE_EIMS_GPI_SDP1;
4957 mask |= IXGBE_EIMS_ECC;
4958 mask |= IXGBE_EIMS_GPI_SDP0;
4959 mask |= IXGBE_EIMS_GPI_SDP1;
4960 mask |= IXGBE_EIMS_GPI_SDP2;
4962 mask |= IXGBE_EIMS_FLOW_DIR;
4966 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4968 /* With RSS we use auto clear */
4969 if (adapter->msix_mem) {
4970 mask = IXGBE_EIMS_ENABLE_MASK;
4971 /* Don't autoclear Link */
4972 mask &= ~IXGBE_EIMS_OTHER;
4973 mask &= ~IXGBE_EIMS_LSC;
4974 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4978 ** Now enable all queues, this is done separately to
4979 ** allow for handling the extended (beyond 32) MSIX
4980 ** vectors that can be used by 82599
4982 for (int i = 0; i < adapter->num_queues; i++, que++)
4983 ixgbe_enable_queue(adapter, que->msix);
4985 IXGBE_WRITE_FLUSH(hw);
4991 ixgbe_disable_intr(struct adapter *adapter)
4993 if (adapter->msix_mem)
4994 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4995 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4996 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4998 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4999 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
5000 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
5002 IXGBE_WRITE_FLUSH(&adapter->hw);
5007 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
5011 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
5018 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
5020 pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
5027 ** Setup the correct IVAR register for a particular MSIX interrupt
5028 ** (yes this is all very magic and confusing :)
5029 ** - entry is the register array entry
5030 ** - vector is the MSIX vector for this queue
5031 ** - type is RX/TX/MISC
5034 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5036 struct ixgbe_hw *hw = &adapter->hw;
5039 vector |= IXGBE_IVAR_ALLOC_VAL;
5041 switch (hw->mac.type) {
5043 case ixgbe_mac_82598EB:
5045 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5047 entry += (type * 64);
5048 index = (entry >> 2) & 0x1F;
5049 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5050 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5051 ivar |= (vector << (8 * (entry & 0x3)));
5052 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5055 case ixgbe_mac_82599EB:
5056 case ixgbe_mac_X540:
5057 if (type == -1) { /* MISC IVAR */
5058 index = (entry & 1) * 8;
5059 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5060 ivar &= ~(0xFF << index);
5061 ivar |= (vector << index);
5062 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5063 } else { /* RX/TX IVARS */
5064 index = (16 * (entry & 1)) + (8 * type);
5065 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5066 ivar &= ~(0xFF << index);
5067 ivar |= (vector << index);
5068 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5077 ixgbe_configure_ivars(struct adapter *adapter)
5079 struct ix_queue *que = adapter->queues;
5082 if (ixgbe_max_interrupt_rate > 0)
5083 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5087 for (int i = 0; i < adapter->num_queues; i++, que++) {
5088 /* First the RX queue entry */
5089 ixgbe_set_ivar(adapter, i, que->msix, 0);
5090 /* ... and the TX */
5091 ixgbe_set_ivar(adapter, i, que->msix, 1);
5092 /* Set an Initial EITR value */
5093 IXGBE_WRITE_REG(&adapter->hw,
5094 IXGBE_EITR(que->msix), newitr);
5097 /* For the Link interrupt */
5098 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5102 ** ixgbe_sfp_probe - called in the local timer to
5103 ** determine if a port had optics inserted.
5105 static bool ixgbe_sfp_probe(struct adapter *adapter)
5107 struct ixgbe_hw *hw = &adapter->hw;
5108 device_t dev = adapter->dev;
5109 bool result = FALSE;
5111 if ((hw->phy.type == ixgbe_phy_nl) &&
5112 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5113 s32 ret = hw->phy.ops.identify_sfp(hw);
5116 ret = hw->phy.ops.reset(hw);
5117 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5118 device_printf(dev,"Unsupported SFP+ module detected!");
5119 kprintf(" Reload driver with supported module.\n");
5120 adapter->sfp_probe = FALSE;
5123 device_printf(dev,"SFP+ module detected!\n");
5124 /* We now have supported optics */
5125 adapter->sfp_probe = FALSE;
5126 /* Set the optics type so system reports correctly */
5127 ixgbe_setup_optics(adapter);
5135 ** Tasklet handler for MSIX Link interrupts
5136 ** - do outside interrupt since it might sleep
5139 ixgbe_handle_link(void *context, int pending)
5141 struct adapter *adapter = context;
5143 ixgbe_check_link(&adapter->hw,
5144 &adapter->link_speed, &adapter->link_up, 0);
5145 ixgbe_update_link_status(adapter);
5149 ** Tasklet for handling SFP module interrupts
5152 ixgbe_handle_mod(void *context, int pending)
5154 struct adapter *adapter = context;
5155 struct ixgbe_hw *hw = &adapter->hw;
5156 device_t dev = adapter->dev;
5159 err = hw->phy.ops.identify_sfp(hw);
5160 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5162 "Unsupported SFP+ module type was detected.\n");
5165 err = hw->mac.ops.setup_sfp(hw);
5166 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5168 "Setup failure - unsupported SFP+ module type.\n");
5171 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5177 ** Tasklet for handling MSF (multispeed fiber) interrupts
5180 ixgbe_handle_msf(void *context, int pending)
5182 struct adapter *adapter = context;
5183 struct ixgbe_hw *hw = &adapter->hw;
5187 autoneg = hw->phy.autoneg_advertised;
5188 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5189 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5190 if (hw->mac.ops.setup_link)
5191 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5197 ** Tasklet for reinitializing the Flow Director filter table
5200 ixgbe_reinit_fdir(void *context, int pending)
5202 struct adapter *adapter = context;
5203 struct ifnet *ifp = adapter->ifp;
5205 if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5207 ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5208 adapter->fdir_reinit = 0;
5209 /* re-enable flow director interrupts */
5210 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5211 /* Restart the interface */
5212 ifp->if_drv_flags |= IFF_DRV_RUNNING;
5217 /**********************************************************************
5219 * Update the board statistics counters.
5221 **********************************************************************/
5223 ixgbe_update_stats_counters(struct adapter *adapter)
5225 struct ifnet *ifp = adapter->ifp;
5226 struct ixgbe_hw *hw = &adapter->hw;
5227 u32 missed_rx = 0, bprc, lxon, lxoff, total;
5228 u64 total_missed_rx = 0;
5230 adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5231 adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5232 adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5233 adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5235 for (int i = 0; i < 8; i++) {
5237 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5238 /* missed_rx tallies misses for the gprc workaround */
5240 /* global total per queue */
5241 adapter->stats.mpc[i] += mp;
5242 /* Running comprehensive total for stats display */
5243 total_missed_rx += adapter->stats.mpc[i];
5244 if (hw->mac.type == ixgbe_mac_82598EB)
5245 adapter->stats.rnbc[i] +=
5246 IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5247 adapter->stats.pxontxc[i] +=
5248 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5249 adapter->stats.pxonrxc[i] +=
5250 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5251 adapter->stats.pxofftxc[i] +=
5252 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5253 adapter->stats.pxoffrxc[i] +=
5254 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5255 adapter->stats.pxon2offc[i] +=
5256 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5258 for (int i = 0; i < 16; i++) {
5259 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5260 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5261 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5262 adapter->stats.qbrc[i] +=
5263 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5264 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5265 adapter->stats.qbtc[i] +=
5266 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5267 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5269 adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5270 adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5271 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5273 /* Hardware workaround, gprc counts missed packets */
5274 adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5275 adapter->stats.gprc -= missed_rx;
5277 if (hw->mac.type != ixgbe_mac_82598EB) {
5278 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5279 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5280 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5281 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5282 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5283 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5284 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5285 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5287 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5288 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5289 /* 82598 only has a counter in the high register */
5290 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5291 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5292 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5296 * Workaround: mprc hardware is incorrectly counting
5297 * broadcasts, so for now we subtract those.
5299 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5300 adapter->stats.bprc += bprc;
5301 adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5302 if (hw->mac.type == ixgbe_mac_82598EB)
5303 adapter->stats.mprc -= bprc;
5305 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5306 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5307 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5308 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5309 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5310 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5312 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5313 adapter->stats.lxontxc += lxon;
5314 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5315 adapter->stats.lxofftxc += lxoff;
5316 total = lxon + lxoff;
5318 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5319 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5320 adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5321 adapter->stats.gptc -= total;
5322 adapter->stats.mptc -= total;
5323 adapter->stats.ptc64 -= total;
5324 adapter->stats.gotc -= total * ETHER_MIN_LEN;
5326 adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5327 adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5328 adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5329 adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5330 adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5331 adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5332 adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5333 adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5334 adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5335 adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5336 adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5337 adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5338 adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5339 adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5340 adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5341 adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5342 adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5343 adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5344 /* Only read FCOE on 82599 */
5345 if (hw->mac.type != ixgbe_mac_82598EB) {
5346 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5347 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5348 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5349 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5350 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5353 /* Fill out the OS statistics structure */
5354 ifp->if_ipackets = adapter->stats.gprc;
5355 ifp->if_opackets = adapter->stats.gptc;
5356 ifp->if_ibytes = adapter->stats.gorc;
5357 ifp->if_obytes = adapter->stats.gotc;
5358 ifp->if_imcasts = adapter->stats.mprc;
5359 ifp->if_collisions = 0;
5362 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5363 adapter->stats.rlec;
5366 /** ixgbe_sysctl_tdh_handler - Handler function
5367 * Retrieves the TDH value from the hardware
5370 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5374 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5377 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5378 error = sysctl_handle_int(oidp, &val, 0, req);
5379 if (error || !req->newptr)
5384 /** ixgbe_sysctl_tdt_handler - Handler function
5385 * Retrieves the TDT value from the hardware
5388 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5392 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5395 unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5396 error = sysctl_handle_int(oidp, &val, 0, req);
5397 if (error || !req->newptr)
5402 /** ixgbe_sysctl_rdh_handler - Handler function
5403 * Retrieves the RDH value from the hardware
5406 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5410 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5413 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5414 error = sysctl_handle_int(oidp, &val, 0, req);
5415 if (error || !req->newptr)
5420 /** ixgbe_sysctl_rdt_handler - Handler function
5421 * Retrieves the RDT value from the hardware
5424 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5428 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5431 unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5432 error = sysctl_handle_int(oidp, &val, 0, req);
5433 if (error || !req->newptr)
5439 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5442 struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5443 unsigned int reg, usec, rate;
5445 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5446 usec = ((reg & 0x0FF8) >> 3);
5448 rate = 500000 / usec;
5451 error = sysctl_handle_int(oidp, &rate, 0, req);
5452 if (error || !req->newptr)
5454 reg &= ~0xfff; /* default, no limitation */
5455 ixgbe_max_interrupt_rate = 0;
5456 if (rate > 0 && rate < 500000) {
5459 ixgbe_max_interrupt_rate = rate;
5460 reg |= ((4000000/rate) & 0xff8 );
5462 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5467 * Add sysctl variables, one per statistic, to the system.
5470 ixgbe_add_hw_stats(struct adapter *adapter)
5472 struct tx_ring *txr = adapter->tx_rings;
5473 struct rx_ring *rxr = adapter->rx_rings;
5475 struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5476 struct sysctl_oid *tree = adapter->sysctl_tree;
5477 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5478 struct ixgbe_hw_stats *stats = &adapter->stats;
5480 struct sysctl_oid *stat_node, *queue_node;
5481 struct sysctl_oid_list *stat_list, *queue_list;
5483 #define QUEUE_NAME_LEN 32
5484 char namebuf[QUEUE_NAME_LEN];
5486 /* Driver Statistics */
5487 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5488 CTLFLAG_RD, &adapter->dropped_pkts,
5489 "Driver dropped packets");
5490 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5491 CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5492 "m_defrag() failed");
5493 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5494 CTLFLAG_RD, &adapter->no_tx_dma_setup,
5495 "Driver tx dma failure in xmit");
5496 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5497 CTLFLAG_RD, &adapter->watchdog_events,
5498 "Watchdog timeouts");
5500 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5501 CTLFLAG_RD, &adapter->tso_tx,
5504 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5505 CTLFLAG_RD, &adapter->link_irq,
5506 "Link MSIX IRQ Handled");
5508 for (int i = 0; i < adapter->num_queues; i++, txr++) {
5509 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5510 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5511 CTLFLAG_RD, NULL, "Queue Name");
5512 queue_list = SYSCTL_CHILDREN(queue_node);
5514 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5515 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5516 sizeof(&adapter->queues[i]),
5517 ixgbe_sysctl_interrupt_rate_handler, "IU",
5519 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5520 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5521 "irqs on this queue");
5522 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5523 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5524 ixgbe_sysctl_tdh_handler, "IU",
5525 "Transmit Descriptor Head");
5526 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5527 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5528 ixgbe_sysctl_tdt_handler, "IU",
5529 "Transmit Descriptor Tail");
5530 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5531 CTLFLAG_RD, &txr->no_desc_avail, 0,
5532 "Queue No Descriptor Available");
5533 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5534 CTLFLAG_RD, &txr->total_packets, 0,
5535 "Queue Packets Transmitted");
5538 for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5539 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5540 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5541 CTLFLAG_RD, NULL, "Queue Name");
5542 queue_list = SYSCTL_CHILDREN(queue_node);
5545 struct lro_ctrl *lro = &rxr->lro;
5548 ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5549 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5550 CTLFLAG_RD, NULL, "Queue Name");
5551 queue_list = SYSCTL_CHILDREN(queue_node);
5553 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5554 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5555 ixgbe_sysctl_rdh_handler, "IU",
5556 "Receive Descriptor Head");
5557 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5558 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5559 ixgbe_sysctl_rdt_handler, "IU",
5560 "Receive Descriptor Tail");
5561 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5562 CTLFLAG_RD, &rxr->rx_packets, 0,
5563 "Queue Packets Received");
5564 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5565 CTLFLAG_RD, &rxr->rx_bytes, 0,
5566 "Queue Bytes Received");
5568 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5569 CTLFLAG_RD, &lro->lro_queued, 0,
5571 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5572 CTLFLAG_RD, &lro->lro_flushed, 0,
5577 /* MAC stats get the own sub node */
5579 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5580 CTLFLAG_RD, NULL, "MAC Statistics");
5581 stat_list = SYSCTL_CHILDREN(stat_node);
5583 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5584 CTLFLAG_RD, &stats->crcerrs, 0,
5586 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5587 CTLFLAG_RD, &stats->illerrc, 0,
5588 "Illegal Byte Errors");
5589 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5590 CTLFLAG_RD, &stats->errbc, 0,
5592 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5593 CTLFLAG_RD, &stats->mspdc, 0,
5594 "MAC Short Packets Discarded");
5595 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5596 CTLFLAG_RD, &stats->mlfc, 0,
5597 "MAC Local Faults");
5598 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5599 CTLFLAG_RD, &stats->mrfc, 0,
5600 "MAC Remote Faults");
5601 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5602 CTLFLAG_RD, &stats->rlec, 0,
5603 "Receive Length Errors");
5604 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5605 CTLFLAG_RD, &stats->lxontxc, 0,
5606 "Link XON Transmitted");
5607 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5608 CTLFLAG_RD, &stats->lxonrxc, 0,
5609 "Link XON Received");
5610 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5611 CTLFLAG_RD, &stats->lxofftxc, 0,
5612 "Link XOFF Transmitted");
5613 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5614 CTLFLAG_RD, &stats->lxoffrxc, 0,
5615 "Link XOFF Received");
5617 /* Packet Reception Stats */
5618 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5619 CTLFLAG_RD, &stats->tor, 0,
5620 "Total Octets Received");
5621 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5622 CTLFLAG_RD, &stats->gorc, 0,
5623 "Good Octets Received");
5624 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5625 CTLFLAG_RD, &stats->tpr, 0,
5626 "Total Packets Received");
5627 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5628 CTLFLAG_RD, &stats->gprc, 0,
5629 "Good Packets Received");
5630 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5631 CTLFLAG_RD, &stats->mprc, 0,
5632 "Multicast Packets Received");
5633 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5634 CTLFLAG_RD, &stats->bprc, 0,
5635 "Broadcast Packets Received");
5636 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5637 CTLFLAG_RD, &stats->prc64, 0,
5638 "64 byte frames received ");
5639 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5640 CTLFLAG_RD, &stats->prc127, 0,
5641 "65-127 byte frames received");
5642 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5643 CTLFLAG_RD, &stats->prc255, 0,
5644 "128-255 byte frames received");
5645 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5646 CTLFLAG_RD, &stats->prc511, 0,
5647 "256-511 byte frames received");
5648 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5649 CTLFLAG_RD, &stats->prc1023, 0,
5650 "512-1023 byte frames received");
5651 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5652 CTLFLAG_RD, &stats->prc1522, 0,
5653 "1023-1522 byte frames received");
5654 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5655 CTLFLAG_RD, &stats->ruc, 0,
5656 "Receive Undersized");
5657 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5658 CTLFLAG_RD, &stats->rfc, 0,
5659 "Fragmented Packets Received ");
5660 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5661 CTLFLAG_RD, &stats->roc, 0,
5662 "Oversized Packets Received");
5663 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5664 CTLFLAG_RD, &stats->rjc, 0,
5666 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5667 CTLFLAG_RD, &stats->mngprc, 0,
5668 "Management Packets Received");
5669 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5670 CTLFLAG_RD, &stats->mngptc, 0,
5671 "Management Packets Dropped");
5672 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5673 CTLFLAG_RD, &stats->xec, 0,
5676 /* Packet Transmission Stats */
5677 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5678 CTLFLAG_RD, &stats->gotc, 0,
5679 "Good Octets Transmitted");
5680 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5681 CTLFLAG_RD, &stats->tpt, 0,
5682 "Total Packets Transmitted");
5683 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5684 CTLFLAG_RD, &stats->gptc, 0,
5685 "Good Packets Transmitted");
5686 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5687 CTLFLAG_RD, &stats->bptc, 0,
5688 "Broadcast Packets Transmitted");
5689 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5690 CTLFLAG_RD, &stats->mptc, 0,
5691 "Multicast Packets Transmitted");
5692 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5693 CTLFLAG_RD, &stats->mngptc, 0,
5694 "Management Packets Transmitted");
5695 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5696 CTLFLAG_RD, &stats->ptc64, 0,
5697 "64 byte frames transmitted ");
5698 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5699 CTLFLAG_RD, &stats->ptc127, 0,
5700 "65-127 byte frames transmitted");
5701 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5702 CTLFLAG_RD, &stats->ptc255, 0,
5703 "128-255 byte frames transmitted");
5704 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5705 CTLFLAG_RD, &stats->ptc511, 0,
5706 "256-511 byte frames transmitted");
5707 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5708 CTLFLAG_RD, &stats->ptc1023, 0,
5709 "512-1023 byte frames transmitted");
5710 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5711 CTLFLAG_RD, &stats->ptc1522, 0,
5712 "1024-1522 byte frames transmitted");
5715 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5716 CTLFLAG_RD, &stats->fccrc, 0,
5718 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5719 CTLFLAG_RD, &stats->fclast, 0,
5721 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5722 CTLFLAG_RD, &stats->fcoerpdc, 0,
5723 "FCoE Packets Dropped");
5724 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5725 CTLFLAG_RD, &stats->fcoeprc, 0,
5726 "FCoE Packets Received");
5727 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5728 CTLFLAG_RD, &stats->fcoeptc, 0,
5729 "FCoE Packets Transmitted");
5730 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5731 CTLFLAG_RD, &stats->fcoedwrc, 0,
5732 "FCoE DWords Received");
5733 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5734 CTLFLAG_RD, &stats->fcoedwtc, 0,
5735 "FCoE DWords Transmitted");
5739 ** Set flow control using sysctl:
5740 ** Flow control values:
5747 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5750 struct adapter *adapter = (struct adapter *) arg1;
5753 error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5754 if ((error) || (req->newptr == NULL))
5757 /* Don't bother if it's not changed */
5758 if (adapter->fc == last)
5761 switch (adapter->fc) {
5762 case ixgbe_fc_rx_pause:
5763 case ixgbe_fc_tx_pause:
5765 adapter->hw.fc.requested_mode = adapter->fc;
5769 adapter->hw.fc.requested_mode = ixgbe_fc_none;
5771 /* Don't autoneg if forcing a value */
5772 adapter->hw.fc.disable_fc_autoneg = TRUE;
5773 ixgbe_fc_enable(&adapter->hw);
5778 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5779 const char *description, int *limit, int value)
5782 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5783 SYSCTL_CHILDREN(adapter->sysctl_tree),
5784 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5788 ** Control link advertise speed:
5789 ** 1 - advertise only 1G
5790 ** 2 - advertise 100Mb
5791 ** 3 - advertise normal
5794 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5797 struct adapter *adapter;
5799 struct ixgbe_hw *hw;
5800 ixgbe_link_speed speed, last;
5802 adapter = (struct adapter *) arg1;
5805 last = adapter->advertise;
5807 error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5808 if ((error) || (adapter->advertise == -1))
5811 if (adapter->advertise == last) /* no change */
5814 if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5815 (hw->phy.multispeed_fiber)))
5818 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5819 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5823 if (adapter->advertise == 1)
5824 speed = IXGBE_LINK_SPEED_1GB_FULL;
5825 else if (adapter->advertise == 2)
5826 speed = IXGBE_LINK_SPEED_100_FULL;
5827 else if (adapter->advertise == 3)
5828 speed = IXGBE_LINK_SPEED_1GB_FULL |
5829 IXGBE_LINK_SPEED_10GB_FULL;
5830 else /* bogus value */
5833 hw->mac.autotry_restart = TRUE;
5834 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5840 ** Thermal Shutdown Trigger
5841 ** - cause a Thermal Overtemp IRQ
5844 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5846 int error, fire = 0;
5847 struct adapter *adapter = (struct adapter *) arg1;
5848 struct ixgbe_hw *hw = &adapter->hw;
5851 if (hw->mac.type != ixgbe_mac_X540)
5854 error = sysctl_handle_int(oidp, &fire, 0, req);
5855 if ((error) || (req->newptr == NULL))
5859 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5860 reg |= IXGBE_EICR_TS;
5861 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);