805c9041fcf9dbdbde592ffb61b43b1b001f7a65
[dragonfly.git] / sys / dev / netif / ixgbe / ixgbe.c
1 /******************************************************************************
2
3   Copyright (c) 2001-2012, Intel Corporation 
4   All rights reserved.
5   
6   Redistribution and use in source and binary forms, with or without 
7   modification, are permitted provided that the following conditions are met:
8   
9    1. Redistributions of source code must retain the above copyright notice, 
10       this list of conditions and the following disclaimer.
11   
12    2. Redistributions in binary form must reproduce the above copyright 
13       notice, this list of conditions and the following disclaimer in the 
14       documentation and/or other materials provided with the distribution.
15   
16    3. Neither the name of the Intel Corporation nor the names of its 
17       contributors may be used to endorse or promote products derived from 
18       this software without specific prior written permission.
19   
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
34
35 #include "opt_inet.h"
36 #include "opt_inet6.h"
37
38 #include "ixgbe.h"
39
40 /*********************************************************************
41  *  Set this to one to display debug statistics
42  *********************************************************************/
43 int             ixgbe_display_debug_stats = 0;
44
45 /*********************************************************************
46  *  Driver version
47  *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
49
50 /*********************************************************************
51  *  PCI Device ID Table
52  *
53  *  Used by probe to select devices to load on
54  *  Last field stores an index into ixgbe_strings
55  *  Last entry must be all 0s
56  *
57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58  *********************************************************************/
59
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
61 {
62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86         /* required last entry */
87         {0, 0, 0, 0, 0}
88 };
89
90 /*********************************************************************
91  *  Table of branding strings
92  *********************************************************************/
93
94 static char    *ixgbe_strings[] = {
95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
96 };
97
98 /*********************************************************************
99  *  Function prototypes
100  *********************************************************************/
101 static int      ixgbe_probe(device_t);
102 static int      ixgbe_attach(device_t);
103 static int      ixgbe_detach(device_t);
104 static int      ixgbe_shutdown(device_t);
105 static void     ixgbe_start(struct ifnet *);
106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int      ixgbe_mq_start_locked(struct ifnet *,
110                     struct tx_ring *, struct mbuf *);
111 static void     ixgbe_qflush(struct ifnet *);
112 #endif
113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void     ixgbe_init(void *);
115 static void     ixgbe_init_locked(struct adapter *);
116 static void     ixgbe_stop(void *);
117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int      ixgbe_media_change(struct ifnet *);
119 static void     ixgbe_identify_hardware(struct adapter *);
120 static int      ixgbe_allocate_pci_resources(struct adapter *);
121 static int      ixgbe_allocate_msix(struct adapter *);
122 static int      ixgbe_allocate_legacy(struct adapter *);
123 static int      ixgbe_allocate_queues(struct adapter *);
124 static int      ixgbe_setup_msix(struct adapter *);
125 static void     ixgbe_free_pci_resources(struct adapter *);
126 static void     ixgbe_local_timer(void *);
127 static int      ixgbe_setup_interface(device_t, struct adapter *);
128 static void     ixgbe_config_link(struct adapter *);
129
130 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
131 static int      ixgbe_setup_transmit_structures(struct adapter *);
132 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
133 static void     ixgbe_initialize_transmit_units(struct adapter *);
134 static void     ixgbe_free_transmit_structures(struct adapter *);
135 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
136
137 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
138 static int      ixgbe_setup_receive_structures(struct adapter *);
139 static int      ixgbe_setup_receive_ring(struct rx_ring *);
140 static void     ixgbe_initialize_receive_units(struct adapter *);
141 static void     ixgbe_free_receive_structures(struct adapter *);
142 static void     ixgbe_free_receive_buffers(struct rx_ring *);
143 #if 0   /* NET_LRO */
144 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
145 #endif
146
147 static void     ixgbe_enable_intr(struct adapter *);
148 static void     ixgbe_disable_intr(struct adapter *);
149 static void     ixgbe_update_stats_counters(struct adapter *);
150 static void     ixgbe_txeof(struct tx_ring *);
151 static void     ixgbe_rxeof(struct ix_queue *, int);
152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void     ixgbe_set_promisc(struct adapter *);
154 static void     ixgbe_set_multi(struct adapter *);
155 static void     ixgbe_update_link_status(struct adapter *);
156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
162                     struct ixgbe_dma_alloc *, int);
163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
165                     const char *, int *, int);
166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
167 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
168 static int      ixgbe_tso_pullup(struct tx_ring *, struct mbuf **);
169 static void     ixgbe_add_sysctl(struct adapter *);
170 static void     ixgbe_set_eitr(struct adapter *, int, int);
171 static int      ixgbe_sysctl_intr_rate(SYSCTL_HANDLER_ARGS);
172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void     ixgbe_configure_ivars(struct adapter *);
174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
175
176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
179
180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
181
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
184                     struct mbuf *, u32);
185
186 /* Support for pluggable optic modules */
187 static bool     ixgbe_sfp_probe(struct adapter *);
188 static void     ixgbe_setup_optics(struct adapter *);
189
190 /* Legacy (single vector interrupt handler */
191 static void     ixgbe_legacy_irq(void *);
192
193 /* The MSI/X Interrupt handlers */
194 static void     ixgbe_msix_que(void *);
195 static void     ixgbe_msix_link(void *);
196
197 /* Deferred interrupt tasklets */
198 static void     ixgbe_handle_msf(void *, int);
199 static void     ixgbe_handle_mod(void *, int);
200
201 #ifdef IXGBE_FDIR
202 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
203 static void     ixgbe_reinit_fdir(void *, int);
204 #endif
205
206 /*********************************************************************
207  *  FreeBSD Device Interface Entry Points
208  *********************************************************************/
209
210 static device_method_t ixgbe_methods[] = {
211         /* Device interface */
212         DEVMETHOD(device_probe, ixgbe_probe),
213         DEVMETHOD(device_attach, ixgbe_attach),
214         DEVMETHOD(device_detach, ixgbe_detach),
215         DEVMETHOD(device_shutdown, ixgbe_shutdown),
216         {0, 0}
217 };
218
219 static driver_t ixgbe_driver = {
220         "ix", ixgbe_methods, sizeof(struct adapter),
221 };
222
223 devclass_t ixgbe_devclass;
224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
225
226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
228
229 /*
230 ** TUNEABLE PARAMETERS:
231 */
232
233 /* How many packets rxeof tries to clean at a time */
234 static int ixgbe_rx_process_limit = 128;
235 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
236
237 /*
238 ** Smart speed setting, default to on
239 ** this only works as a compile option
240 ** right now as its during attach, set
241 ** this to 'ixgbe_smart_speed_off' to
242 ** disable.
243 */
244 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
245
246 static int ixgbe_msi_enable = 1;
247 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
248
249 /*
250  * MSIX should be the default for best performance,
251  * but this allows it to be forced off for testing.
252  */
253 static int ixgbe_enable_msix = 1;
254 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
255
256 /*
257  * Header split: this causes the hardware to DMA
258  * the header into a separate mbuf from the payload,
259  * it can be a performance win in some workloads, but
260  * in others it actually hurts, its off by default. 
261  */
262 static int ixgbe_header_split = FALSE;
263 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
264
265 /*
266  * Number of Queues, can be set to 0,
267  * it then autoconfigures based on the
268  * number of cpus with a max of 8. This
269  * can be overriden manually here.
270  */
271 static int ixgbe_num_queues = 0;
272 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
273
274 /*
275 ** Number of TX descriptors per ring,
276 ** setting higher than RX as this seems
277 ** the better performing choice.
278 */
279 static int ixgbe_txd = PERFORM_TXD;
280 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
281
282 /* Number of RX descriptors per ring */
283 static int ixgbe_rxd = PERFORM_RXD;
284 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
285
286 /* Keep running tab on them for sanity check */
287 static int ixgbe_total_ports;
288
289 #ifdef IXGBE_FDIR
290 /*
291 ** For Flow Director: this is the
292 ** number of TX packets we sample
293 ** for the filter pool, this means
294 ** every 20th packet will be probed.
295 **
296 ** This feature can be disabled by 
297 ** setting this to 0.
298 */
299 static int atr_sample_rate = 20;
300 /* 
301 ** Flow Director actually 'steals'
302 ** part of the packet buffer as its
303 ** filter pool, this variable controls
304 ** how much it uses:
305 **  0 = 64K, 1 = 128K, 2 = 256K
306 */
307 static int fdir_pballoc = 1;
308 #endif
309
310 #ifdef DEV_NETMAP
311 /*
312  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
313  * be a reference on how to implement netmap support in a driver.
314  * Additional comments are in ixgbe_netmap.h .
315  *
316  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
317  * that extend the standard driver.
318  */
319 #include <dev/netmap/ixgbe_netmap.h>
320 #endif /* DEV_NETMAP */
321
322 /*********************************************************************
323  *  Device identification routine
324  *
325  *  ixgbe_probe determines if the driver should be loaded on
326  *  adapter based on PCI vendor/device id of the adapter.
327  *
328  *  return BUS_PROBE_DEFAULT on success, positive on failure
329  *********************************************************************/
330
331 static int
332 ixgbe_probe(device_t dev)
333 {
334         ixgbe_vendor_info_t *ent;
335
336         u16     pci_vendor_id = 0;
337         u16     pci_device_id = 0;
338         u16     pci_subvendor_id = 0;
339         u16     pci_subdevice_id = 0;
340         char    adapter_name[256];
341
342         INIT_DEBUGOUT("ixgbe_probe: begin");
343
344         pci_vendor_id = pci_get_vendor(dev);
345         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
346                 return (ENXIO);
347
348         pci_device_id = pci_get_device(dev);
349         pci_subvendor_id = pci_get_subvendor(dev);
350         pci_subdevice_id = pci_get_subdevice(dev);
351
352         ent = ixgbe_vendor_info_array;
353         while (ent->vendor_id != 0) {
354                 if ((pci_vendor_id == ent->vendor_id) &&
355                     (pci_device_id == ent->device_id) &&
356
357                     ((pci_subvendor_id == ent->subvendor_id) ||
358                      (ent->subvendor_id == 0)) &&
359
360                     ((pci_subdevice_id == ent->subdevice_id) ||
361                      (ent->subdevice_id == 0))) {
362                         ksprintf(adapter_name, "%s, Version - %s",
363                                 ixgbe_strings[ent->index],
364                                 ixgbe_driver_version);
365                         device_set_desc_copy(dev, adapter_name);
366                         ++ixgbe_total_ports;
367                         return (BUS_PROBE_DEFAULT);
368                 }
369                 ent++;
370         }
371         return (ENXIO);
372 }
373
374 /*********************************************************************
375  *  Device initialization routine
376  *
377  *  The attach entry point is called when the driver is being loaded.
378  *  This routine identifies the type of hardware, allocates all resources
379  *  and initializes the hardware.
380  *
381  *  return 0 on success, positive on failure
382  *********************************************************************/
383
384 static int
385 ixgbe_attach(device_t dev)
386 {
387         struct adapter *adapter;
388         struct ixgbe_hw *hw;
389         int             error = 0;
390         u16             csum;
391         u32             ctrl_ext;
392
393         INIT_DEBUGOUT("ixgbe_attach: begin");
394
395         if (resource_disabled("ixgbe", device_get_unit(dev))) {
396                 device_printf(dev, "Disabled by device hint\n");
397                 return (ENXIO);
398         }
399
400         /* Allocate, clear, and link in our adapter structure */
401         adapter = device_get_softc(dev);
402         adapter->dev = adapter->osdep.dev = dev;
403         hw = &adapter->hw;
404
405         /* Core Lock Init*/
406         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
407
408         /* Set up the timer callout */
409         callout_init_mp(&adapter->timer);
410
411         /* Determine hardware revision */
412         ixgbe_identify_hardware(adapter);
413
414         /* Enable bus mastering */
415         pci_enable_busmaster(dev);
416
417         /* Do base PCI setup - map BAR0 */
418         if (ixgbe_allocate_pci_resources(adapter)) {
419                 device_printf(dev, "Allocation of PCI resources failed\n");
420                 error = ENXIO;
421                 goto err_out;
422         }
423
424         /* Do descriptor calc and sanity checks */
425         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
426             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
427                 device_printf(dev, "TXD config issue, using default!\n");
428                 adapter->num_tx_desc = DEFAULT_TXD;
429         } else
430                 adapter->num_tx_desc = ixgbe_txd;
431
432         /*
433         ** With many RX rings it is easy to exceed the
434         ** system mbuf allocation. Tuning nmbclusters
435         ** can alleviate this.
436         */
437         if (nmbclusters > 0 ) {
438                 int s;
439                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
440                 if (s > nmbclusters) {
441                         device_printf(dev, "RX Descriptors exceed "
442                             "system mbuf max, using default instead!\n");
443                         ixgbe_rxd = DEFAULT_RXD;
444                 }
445         }
446
447         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
448             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
449                 device_printf(dev, "RXD config issue, using default!\n");
450                 adapter->num_rx_desc = DEFAULT_RXD;
451         } else
452                 adapter->num_rx_desc = ixgbe_rxd;
453
454         /* Allocate our TX/RX Queues */
455         if (ixgbe_allocate_queues(adapter)) {
456                 error = ENOMEM;
457                 goto err_out;
458         }
459
460         /* Allocate multicast array memory. */
461         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
462             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
463         if (adapter->mta == NULL) {
464                 device_printf(dev, "Can not allocate multicast setup array\n");
465                 error = ENOMEM;
466                 goto err_late;
467         }
468
469         /* Initialize the shared code */
470         error = ixgbe_init_shared_code(hw);
471         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
472                 /*
473                 ** No optics in this port, set up
474                 ** so the timer routine will probe 
475                 ** for later insertion.
476                 */
477                 adapter->sfp_probe = TRUE;
478                 error = 0;
479         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
480                 device_printf(dev,"Unsupported SFP+ module detected!\n");
481                 error = EIO;
482                 goto err_late;
483         } else if (error) {
484                 device_printf(dev,"Unable to initialize the shared code\n");
485                 error = EIO;
486                 goto err_late;
487         }
488
489         /* Make sure we have a good EEPROM before we read from it */
490         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
491                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
492                 error = EIO;
493                 goto err_late;
494         }
495
496         error = ixgbe_init_hw(hw);
497         switch (error) {
498         case IXGBE_ERR_EEPROM_VERSION:
499                 device_printf(dev, "This device is a pre-production adapter/"
500                     "LOM.  Please be aware there may be issues associated "
501                     "with your hardware.\n If you are experiencing problems "
502                     "please contact your Intel or hardware representative "
503                     "who provided you with this hardware.\n");
504                 break;
505         case IXGBE_ERR_SFP_NOT_SUPPORTED:
506                 device_printf(dev,"Unsupported SFP+ Module\n");
507                 error = EIO;
508                 device_printf(dev,"Hardware Initialization Failure\n");
509                 goto err_late;
510         case IXGBE_ERR_SFP_NOT_PRESENT:
511                 device_printf(dev,"No SFP+ Module found\n");
512                 /* falls thru */
513         default:
514                 break;
515         }
516
517         /* Detect and set physical type */
518         ixgbe_setup_optics(adapter);
519
520         if ((adapter->msix > 1) && (ixgbe_enable_msix)) {
521                 adapter->intr_type = PCI_INTR_TYPE_MSIX;
522                 error = ixgbe_allocate_msix(adapter); 
523         } else {
524                 error = ixgbe_allocate_legacy(adapter); 
525         }
526         if (error) 
527                 goto err_late;
528
529         /* Setup OS specific network interface */
530         if (ixgbe_setup_interface(dev, adapter) != 0)
531                 goto err_late;
532
533         /* Add sysctl tree */
534         ixgbe_add_sysctl(adapter);
535
536         /* Initialize statistics */
537         ixgbe_update_stats_counters(adapter);
538
539         /* Register for VLAN events */
540         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
541             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
542         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
543             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
544
545         /* Print PCIE bus type/speed/width info */
546         ixgbe_get_bus_info(hw);
547         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
548             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
549             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
550             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
551             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
552             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
553             ("Unknown"));
554
555         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
556             (hw->bus.speed == ixgbe_bus_speed_2500)) {
557                 device_printf(dev, "PCI-Express bandwidth available"
558                     " for this card\n     is not sufficient for"
559                     " optimal performance.\n");
560                 device_printf(dev, "For optimal performance a x8 "
561                     "PCIE, or x4 PCIE 2 slot is required.\n");
562         }
563
564         /* let hardware know driver is loaded */
565         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
566         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
567         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
568
569         ixgbe_add_hw_stats(adapter);
570
571 #ifdef DEV_NETMAP
572         ixgbe_netmap_attach(adapter);
573 #endif /* DEV_NETMAP */
574         INIT_DEBUGOUT("ixgbe_attach: end");
575         return (0);
576 err_late:
577         ixgbe_free_transmit_structures(adapter);
578         ixgbe_free_receive_structures(adapter);
579 err_out:
580         if (adapter->ifp != NULL)
581                 if_free(adapter->ifp);
582         ixgbe_free_pci_resources(adapter);
583         kfree(adapter->mta, M_DEVBUF);
584         return (error);
585
586 }
587
588 /*********************************************************************
589  *  Device removal routine
590  *
591  *  The detach entry point is called when the driver is being removed.
592  *  This routine stops the adapter and deallocates all the resources
593  *  that were allocated for driver operation.
594  *
595  *  return 0 on success, positive on failure
596  *********************************************************************/
597
598 static int
599 ixgbe_detach(device_t dev)
600 {
601         struct adapter *adapter = device_get_softc(dev);
602         u32     ctrl_ext;
603
604         INIT_DEBUGOUT("ixgbe_detach: begin");
605
606         /* Make sure VLANS are not using driver */
607         if (adapter->ifp->if_vlantrunks != NULL) {
608                 device_printf(dev,"Vlan in use, detach first\n");
609                 return (EBUSY);
610         }
611
612         IXGBE_CORE_LOCK(adapter);
613         ixgbe_stop(adapter);
614         IXGBE_CORE_UNLOCK(adapter);
615
616         /* Drain the Link queue */
617         if (adapter->tq) {
618                 taskqueue_drain(adapter->tq, &adapter->mod_task);
619                 taskqueue_drain(adapter->tq, &adapter->msf_task);
620 #ifdef IXGBE_FDIR
621                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
622 #endif
623                 taskqueue_free(adapter->tq);
624         }
625
626         /* let hardware know driver is unloading */
627         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
628         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
629         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
630
631         /* Unregister VLAN events */
632         if (adapter->vlan_attach != NULL)
633                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
634         if (adapter->vlan_detach != NULL)
635                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
636
637         ether_ifdetach(adapter->ifp);
638         callout_stop(&adapter->timer);
639 #ifdef DEV_NETMAP
640         netmap_detach(adapter->ifp);
641 #endif /* DEV_NETMAP */
642         ixgbe_free_pci_resources(adapter);
643         bus_generic_detach(dev);
644         if_free(adapter->ifp);
645
646         ixgbe_free_transmit_structures(adapter);
647         ixgbe_free_receive_structures(adapter);
648         kfree(adapter->mta, M_DEVBUF);
649         sysctl_ctx_free(&adapter->sysctl_ctx);
650         
651         IXGBE_CORE_LOCK_DESTROY(adapter);
652         return (0);
653 }
654
655 /*********************************************************************
656  *
657  *  Shutdown entry point
658  *
659  **********************************************************************/
660
661 static int
662 ixgbe_shutdown(device_t dev)
663 {
664         struct adapter *adapter = device_get_softc(dev);
665         IXGBE_CORE_LOCK(adapter);
666         ixgbe_stop(adapter);
667         IXGBE_CORE_UNLOCK(adapter);
668         return (0);
669 }
670
671
672 /*********************************************************************
673  *  Transmit entry point
674  *
675  *  ixgbe_start is called by the stack to initiate a transmit.
676  *  The driver will remain in this routine as long as there are
677  *  packets to transmit and transmit resources are available.
678  *  In case resources are not available stack is notified and
679  *  the packet is requeued.
680  **********************************************************************/
681
682 static void
683 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
684 {
685         struct mbuf    *m_head;
686         struct adapter *adapter = txr->adapter;
687
688         IXGBE_TX_LOCK_ASSERT(txr);
689
690         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
691                 return;
692
693         if (!adapter->link_active) {
694                 ifq_purge(&ifp->if_snd);
695                 return;
696         }
697
698         while (!ifq_is_empty(&ifp->if_snd)) {
699                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
700                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
701                         break;
702                 }
703
704                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
705                 if (m_head == NULL)
706                         break;
707
708                 if (ixgbe_xmit(txr, &m_head)) {
709 #if 0 /* XXX: prepend to an ALTQ queue ? */
710                         if (m_head != NULL)
711                                 IF_PREPEND(&ifp->if_snd, m_head);
712 #endif
713                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
714                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
715                         break;
716                 }
717                 /* Send a copy of the frame to the BPF listener */
718                 ETHER_BPF_MTAP(ifp, m_head);
719
720                 /* Set watchdog on */
721                 txr->watchdog_time = ticks;
722                 txr->queue_status = IXGBE_QUEUE_WORKING;
723
724         }
725         return;
726 }
727
728 /*
729  * Legacy TX start - called by the stack, this
730  * always uses the first tx ring, and should
731  * not be used with multiqueue tx enabled.
732  */
733 static void
734 ixgbe_start(struct ifnet *ifp)
735 {
736         struct adapter *adapter = ifp->if_softc;
737         struct tx_ring  *txr = adapter->tx_rings;
738
739         if (ifp->if_flags & IFF_RUNNING) {
740                 IXGBE_TX_LOCK(txr);
741                 ixgbe_start_locked(txr, ifp);
742                 IXGBE_TX_UNLOCK(txr);
743         }
744         return;
745 }
746
747 #if 0 /* __FreeBSD_version >= 800000 */
748 /*
749 ** Multiqueue Transmit driver
750 **
751 */
752 static int
753 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
754 {
755         struct adapter  *adapter = ifp->if_softc;
756         struct ix_queue *que;
757         struct tx_ring  *txr;
758         int             i = 0, err = 0;
759
760         /* Which queue to use */
761         if ((m->m_flags & M_FLOWID) != 0)
762                 i = m->m_pkthdr.flowid % adapter->num_queues;
763         else
764                 i = curcpu % adapter->num_queues;
765
766         txr = &adapter->tx_rings[i];
767         que = &adapter->queues[i];
768
769         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
770             IXGBE_TX_TRYLOCK(txr)) {
771                 err = ixgbe_mq_start_locked(ifp, txr, m);
772                 IXGBE_TX_UNLOCK(txr);
773         } else {
774                 err = drbr_enqueue(ifp, txr->br, m);
775                 taskqueue_enqueue(que->tq, &que->que_task);
776         }
777
778         return (err);
779 }
780
781 static int
782 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
783 {
784         struct adapter  *adapter = txr->adapter;
785         struct mbuf     *next;
786         int             enqueued, err = 0;
787
788         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
789             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
790             adapter->link_active == 0) {
791                 if (m != NULL)
792                         err = drbr_enqueue(ifp, txr->br, m);
793                 return (err);
794         }
795
796         enqueued = 0;
797         if (m == NULL) {
798                 next = drbr_dequeue(ifp, txr->br);
799         } else if (drbr_needs_enqueue(ifp, txr->br)) {
800                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
801                         return (err);
802                 next = drbr_dequeue(ifp, txr->br);
803         } else
804                 next = m;
805
806         /* Process the queue */
807         while (next != NULL) {
808                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
809                         if (next != NULL)
810                                 err = drbr_enqueue(ifp, txr->br, next);
811                         break;
812                 }
813                 enqueued++;
814                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
815                 /* Send a copy of the frame to the BPF listener */
816                 ETHER_BPF_MTAP(ifp, next);
817                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
818                         break;
819                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
820                         ixgbe_txeof(txr);
821                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
822                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
823                         break;
824                 }
825                 next = drbr_dequeue(ifp, txr->br);
826         }
827
828         if (enqueued > 0) {
829                 /* Set watchdog on */
830                 txr->queue_status |= IXGBE_QUEUE_WORKING;
831                 txr->watchdog_time = ticks;
832         }
833
834         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
835                 ixgbe_txeof(txr);
836
837         return (err);
838 }
839
840 /*
841 ** Flush all ring buffers
842 */
843 static void
844 ixgbe_qflush(struct ifnet *ifp)
845 {
846         struct adapter  *adapter = ifp->if_softc;
847         struct tx_ring  *txr = adapter->tx_rings;
848         struct mbuf     *m;
849
850         for (int i = 0; i < adapter->num_queues; i++, txr++) {
851                 IXGBE_TX_LOCK(txr);
852                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
853                         m_freem(m);
854                 IXGBE_TX_UNLOCK(txr);
855         }
856         if_qflush(ifp);
857 }
858 #endif /* __FreeBSD_version >= 800000 */
859
860 /*********************************************************************
861  *  Ioctl entry point
862  *
863  *  ixgbe_ioctl is called when the user wants to configure the
864  *  interface.
865  *
866  *  return 0 on success, positive on failure
867  **********************************************************************/
868
869 static int
870 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
871 {
872         struct adapter  *adapter = ifp->if_softc;
873         struct ifreq    *ifr = (struct ifreq *) data;
874 #if defined(INET) || defined(INET6)
875         struct ifaddr *ifa = (struct ifaddr *)data;
876         bool            avoid_reset = FALSE;
877 #endif
878         int             error = 0;
879
880         switch (command) {
881
882         case SIOCSIFADDR:
883 #ifdef INET
884                 if (ifa->ifa_addr->sa_family == AF_INET)
885                         avoid_reset = TRUE;
886 #endif
887 #ifdef INET6
888                 if (ifa->ifa_addr->sa_family == AF_INET6)
889                         avoid_reset = TRUE;
890 #endif
891 #if defined(INET) || defined(INET6)
892                 /*
893                 ** Calling init results in link renegotiation,
894                 ** so we avoid doing it when possible.
895                 */
896                 if (avoid_reset) {
897                         ifp->if_flags |= IFF_UP;
898                         if (!(ifp->if_flags & IFF_RUNNING))
899                                 ixgbe_init(adapter);
900                         if (!(ifp->if_flags & IFF_NOARP))
901                                 arp_ifinit(ifp, ifa);
902                 } else
903                         error = ether_ioctl(ifp, command, data);
904 #endif
905                 break;
906         case SIOCSIFMTU:
907                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
908                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
909                         error = EINVAL;
910                 } else {
911                         IXGBE_CORE_LOCK(adapter);
912                         ifp->if_mtu = ifr->ifr_mtu;
913                         adapter->max_frame_size =
914                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
915                         ixgbe_init_locked(adapter);
916                         IXGBE_CORE_UNLOCK(adapter);
917                 }
918                 break;
919         case SIOCSIFFLAGS:
920                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
921                 IXGBE_CORE_LOCK(adapter);
922                 if (ifp->if_flags & IFF_UP) {
923                         if ((ifp->if_flags & IFF_RUNNING)) {
924                                 if ((ifp->if_flags ^ adapter->if_flags) &
925                                     (IFF_PROMISC | IFF_ALLMULTI)) {
926                                         ixgbe_set_promisc(adapter);
927                                 }
928                         } else
929                                 ixgbe_init_locked(adapter);
930                 } else
931                         if (ifp->if_flags & IFF_RUNNING)
932                                 ixgbe_stop(adapter);
933                 adapter->if_flags = ifp->if_flags;
934                 IXGBE_CORE_UNLOCK(adapter);
935                 break;
936         case SIOCADDMULTI:
937         case SIOCDELMULTI:
938                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
939                 if (ifp->if_flags & IFF_RUNNING) {
940                         IXGBE_CORE_LOCK(adapter);
941                         ixgbe_disable_intr(adapter);
942                         ixgbe_set_multi(adapter);
943                         ixgbe_enable_intr(adapter);
944                         IXGBE_CORE_UNLOCK(adapter);
945                 }
946                 break;
947         case SIOCSIFMEDIA:
948         case SIOCGIFMEDIA:
949                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
950                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
951                 break;
952         case SIOCSIFCAP:
953         {
954                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
955                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
956                 if (mask & IFCAP_HWCSUM)
957                         ifp->if_capenable ^= IFCAP_HWCSUM;
958                 if (mask & IFCAP_TSO4)
959                         ifp->if_capenable ^= IFCAP_TSO4;
960                 if (mask & IFCAP_TSO6)
961                         ifp->if_capenable ^= IFCAP_TSO6;
962 #if 0 /* NET_LRO */
963                 if (mask & IFCAP_LRO)
964                         ifp->if_capenable ^= IFCAP_LRO;
965 #endif
966                 if (mask & IFCAP_VLAN_HWTAGGING)
967                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
968                 if (mask & IFCAP_VLAN_HWFILTER)
969                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
970 #if 0 /* NET_TSO */
971                 if (mask & IFCAP_VLAN_HWTSO)
972                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
973 #endif
974                 if (ifp->if_flags & IFF_RUNNING) {
975                         IXGBE_CORE_LOCK(adapter);
976                         ixgbe_init_locked(adapter);
977                         IXGBE_CORE_UNLOCK(adapter);
978                 }
979 #if 0
980                 VLAN_CAPABILITIES(ifp);
981 #endif
982                 break;
983         }
984
985         default:
986                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
987                 error = ether_ioctl(ifp, command, data);
988                 break;
989         }
990
991         return (error);
992 }
993
994 /*********************************************************************
995  *  Init entry point
996  *
997  *  This routine is used in two ways. It is used by the stack as
998  *  init entry point in network interface structure. It is also used
999  *  by the driver as a hw/sw initialization routine to get to a
1000  *  consistent state.
1001  *
1002  *  return 0 on success, positive on failure
1003  **********************************************************************/
1004 #define IXGBE_MHADD_MFS_SHIFT 16
1005
1006 static void
1007 ixgbe_init_locked(struct adapter *adapter)
1008 {
1009         struct ifnet   *ifp = adapter->ifp;
1010         device_t        dev = adapter->dev;
1011         struct ixgbe_hw *hw = &adapter->hw;
1012         u32             k, txdctl, mhadd, gpie;
1013         u32             rxdctl, rxctrl;
1014
1015         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1016         INIT_DEBUGOUT("ixgbe_init: begin");
1017         hw->adapter_stopped = FALSE;
1018         ixgbe_stop_adapter(hw);
1019         callout_stop(&adapter->timer);
1020
1021         /* reprogram the RAR[0] in case user changed it. */
1022         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1023
1024         /* Get the latest mac address, User can use a LAA */
1025         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1026               IXGBE_ETH_LENGTH_OF_ADDRESS);
1027         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1028         hw->addr_ctrl.rar_used_count = 1;
1029
1030         /* Set the various hardware offload abilities */
1031         ifp->if_hwassist = 0;
1032         if (ifp->if_capenable & IFCAP_TSO)
1033                 ifp->if_hwassist |= CSUM_TSO;
1034         if (ifp->if_capenable & IFCAP_TXCSUM) {
1035                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1036 #if 0
1037                 if (hw->mac.type != ixgbe_mac_82598EB)
1038                         ifp->if_hwassist |= CSUM_SCTP;
1039 #endif
1040         }
1041
1042         /* Prepare transmit descriptors and buffers */
1043         if (ixgbe_setup_transmit_structures(adapter)) {
1044                 device_printf(dev,"Could not setup transmit structures\n");
1045                 ixgbe_stop(adapter);
1046                 return;
1047         }
1048
1049         ixgbe_init_hw(hw);
1050         ixgbe_initialize_transmit_units(adapter);
1051
1052         /* Setup Multicast table */
1053         ixgbe_set_multi(adapter);
1054
1055         /*
1056         ** Determine the correct mbuf pool
1057         ** for doing jumbo/headersplit
1058         */
1059         if (adapter->max_frame_size <= 2048)
1060                 adapter->rx_mbuf_sz = MCLBYTES;
1061         else if (adapter->max_frame_size <= 4096)
1062                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1063         else if (adapter->max_frame_size <= 9216)
1064                 adapter->rx_mbuf_sz = MJUM9BYTES;
1065         else
1066                 adapter->rx_mbuf_sz = MJUM16BYTES;
1067
1068         /* Prepare receive descriptors and buffers */
1069         if (ixgbe_setup_receive_structures(adapter)) {
1070                 device_printf(dev,"Could not setup receive structures\n");
1071                 ixgbe_stop(adapter);
1072                 return;
1073         }
1074
1075         /* Configure RX settings */
1076         ixgbe_initialize_receive_units(adapter);
1077
1078         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1079
1080         /* Enable Fan Failure Interrupt */
1081         gpie |= IXGBE_SDP1_GPIEN;
1082
1083         /* Add for Module detection */
1084         if (hw->mac.type == ixgbe_mac_82599EB)
1085                 gpie |= IXGBE_SDP2_GPIEN;
1086
1087         /* Thermal Failure Detection */
1088         if (hw->mac.type == ixgbe_mac_X540)
1089                 gpie |= IXGBE_SDP0_GPIEN;
1090
1091         if (adapter->msix > 1) {
1092                 /* Enable Enhanced MSIX mode */
1093                 gpie |= IXGBE_GPIE_MSIX_MODE;
1094                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1095                     IXGBE_GPIE_OCD;
1096         }
1097         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1098
1099         /* Set MTU size */
1100         if (ifp->if_mtu > ETHERMTU) {
1101                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1102                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1103                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1104                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1105         }
1106         
1107         /* Now enable all the queues */
1108
1109         for (int i = 0; i < adapter->num_queues; i++) {
1110                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1111                 txdctl |= IXGBE_TXDCTL_ENABLE;
1112                 /* Set WTHRESH to 8, burst writeback */
1113                 txdctl |= (8 << 16);
1114                 /*
1115                  * When the internal queue falls below PTHRESH (32),
1116                  * start prefetching as long as there are at least
1117                  * HTHRESH (1) buffers ready. The values are taken
1118                  * from the Intel linux driver 3.8.21.
1119                  * Prefetching enables tx line rate even with 1 queue.
1120                  */
1121                 txdctl |= (32 << 0) | (1 << 8);
1122                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1123         }
1124
1125         for (int i = 0; i < adapter->num_queues; i++) {
1126                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1127                 if (hw->mac.type == ixgbe_mac_82598EB) {
1128                         /*
1129                         ** PTHRESH = 21
1130                         ** HTHRESH = 4
1131                         ** WTHRESH = 8
1132                         */
1133                         rxdctl &= ~0x3FFFFF;
1134                         rxdctl |= 0x080420;
1135                 }
1136                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1137                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1138                 for (k = 0; k < 10; k++) {
1139                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1140                             IXGBE_RXDCTL_ENABLE)
1141                                 break;
1142                         else
1143                                 msec_delay(1);
1144                 }
1145                 wmb();
1146 #ifdef DEV_NETMAP
1147                 /*
1148                  * In netmap mode, we must preserve the buffers made
1149                  * available to userspace before the if_init()
1150                  * (this is true by default on the TX side, because
1151                  * init makes all buffers available to userspace).
1152                  *
1153                  * netmap_reset() and the device specific routines
1154                  * (e.g. ixgbe_setup_receive_rings()) map these
1155                  * buffers at the end of the NIC ring, so here we
1156                  * must set the RDT (tail) register to make sure
1157                  * they are not overwritten.
1158                  *
1159                  * In this driver the NIC ring starts at RDH = 0,
1160                  * RDT points to the last slot available for reception (?),
1161                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1162                  */
1163                 if (ifp->if_capenable & IFCAP_NETMAP) {
1164                         struct netmap_adapter *na = NA(adapter->ifp);
1165                         struct netmap_kring *kring = &na->rx_rings[i];
1166                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1167
1168                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1169                 } else
1170 #endif /* DEV_NETMAP */
1171                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1172         }
1173
1174         /* Set up VLAN support and filter */
1175         ixgbe_setup_vlan_hw_support(adapter);
1176
1177         /* Enable Receive engine */
1178         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1179         if (hw->mac.type == ixgbe_mac_82598EB)
1180                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1181         rxctrl |= IXGBE_RXCTRL_RXEN;
1182         ixgbe_enable_rx_dma(hw, rxctrl);
1183
1184         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1185
1186         /* Set up MSI/X routing */
1187         if (ixgbe_enable_msix)  {
1188                 ixgbe_configure_ivars(adapter);
1189                 /* Set up auto-mask */
1190                 if (hw->mac.type == ixgbe_mac_82598EB)
1191                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1192                 else {
1193                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1194                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1195                 }
1196         } else {  /* Simple settings for Legacy/MSI */
1197                 ixgbe_set_ivar(adapter, 0, 0, 0);
1198                 ixgbe_set_ivar(adapter, 0, 0, 1);
1199                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1200         }
1201
1202 #ifdef IXGBE_FDIR
1203         /* Init Flow director */
1204         if (hw->mac.type != ixgbe_mac_82598EB) {
1205                 u32 hdrm = 32 << fdir_pballoc;
1206
1207                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1208                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1209         }
1210 #endif
1211
1212         /*
1213         ** Check on any SFP devices that
1214         ** need to be kick-started
1215         */
1216         if (hw->phy.type == ixgbe_phy_none) {
1217                 int err = hw->phy.ops.identify(hw);
1218                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1219                         device_printf(dev,
1220                             "Unsupported SFP+ module type was detected.\n");
1221                         return;
1222                 }
1223         }
1224
1225         /* Set moderation on the Link interrupt */
1226         ixgbe_set_eitr(adapter, adapter->linkvec, IXGBE_LINK_ITR);
1227
1228         /* Config/Enable Link */
1229         ixgbe_config_link(adapter);
1230
1231         /* Hardware Packet Buffer & Flow Control setup */
1232         {
1233                 u32 rxpb, frame, size, tmp;
1234
1235                 frame = adapter->max_frame_size;
1236
1237                 /* Calculate High Water */
1238                 if (hw->mac.type == ixgbe_mac_X540)
1239                         tmp = IXGBE_DV_X540(frame, frame);
1240                 else
1241                         tmp = IXGBE_DV(frame, frame);
1242                 size = IXGBE_BT2KB(tmp);
1243                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1244                 hw->fc.high_water[0] = rxpb - size;
1245
1246                 /* Now calculate Low Water */
1247                 if (hw->mac.type == ixgbe_mac_X540)
1248                         tmp = IXGBE_LOW_DV_X540(frame);
1249                 else
1250                         tmp = IXGBE_LOW_DV(frame);
1251                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1252                 
1253                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1254                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1255                 hw->fc.send_xon = TRUE;
1256         }
1257         /* Initialize the FC settings */
1258         ixgbe_start_hw(hw);
1259
1260         /* And now turn on interrupts */
1261         ixgbe_enable_intr(adapter);
1262
1263         /* Now inform the stack we're ready */
1264         ifp->if_flags |= IFF_RUNNING;
1265         ifp->if_flags &= ~IFF_OACTIVE;
1266
1267         return;
1268 }
1269
1270 static void
1271 ixgbe_init(void *arg)
1272 {
1273         struct adapter *adapter = arg;
1274
1275         IXGBE_CORE_LOCK(adapter);
1276         ixgbe_init_locked(adapter);
1277         IXGBE_CORE_UNLOCK(adapter);
1278         return;
1279 }
1280
1281
1282 /*
1283 **
1284 ** MSIX Interrupt Handlers and Tasklets
1285 **
1286 */
1287
1288 static inline void
1289 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1290 {
1291         struct ixgbe_hw *hw = &adapter->hw;
1292         u64     queue = (u64)(1 << vector);
1293         u32     mask;
1294
1295         if (hw->mac.type == ixgbe_mac_82598EB) {
1296                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1297                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1298         } else {
1299                 mask = (queue & 0xFFFFFFFF);
1300                 if (mask)
1301                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1302                 mask = (queue >> 32);
1303                 if (mask)
1304                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1305         }
1306 }
1307
1308 static inline void
1309 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1310 {
1311         struct ixgbe_hw *hw = &adapter->hw;
1312         u64     queue = (u64)(1 << vector);
1313         u32     mask;
1314
1315         if (hw->mac.type == ixgbe_mac_82598EB) {
1316                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1317                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1318         } else {
1319                 mask = (queue & 0xFFFFFFFF);
1320                 if (mask)
1321                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1322                 mask = (queue >> 32);
1323                 if (mask)
1324                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1325         }
1326 }
1327
1328 static inline void
1329 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1330 {
1331         u32 mask;
1332
1333         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1334                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1335                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1336         } else {
1337                 mask = (queues & 0xFFFFFFFF);
1338                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1339                 mask = (queues >> 32);
1340                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1341         }
1342 }
1343
1344 /*********************************************************************
1345  *
1346  *  Legacy Interrupt Service routine
1347  *
1348  **********************************************************************/
1349
1350 static void
1351 ixgbe_legacy_irq(void *arg)
1352 {
1353         struct ix_queue *que = arg;
1354         struct adapter  *adapter = que->adapter;
1355         struct ixgbe_hw *hw = &adapter->hw;
1356         struct          tx_ring *txr = adapter->tx_rings;
1357         u32             reg_eicr;
1358
1359
1360         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1361
1362         ++que->irqs;
1363         if (reg_eicr == 0) {
1364                 ixgbe_enable_intr(adapter);
1365                 return;
1366         }
1367
1368         ixgbe_rxeof(que, adapter->rx_process_limit);
1369
1370         IXGBE_TX_LOCK(txr);
1371         ixgbe_txeof(txr);
1372         if (!ifq_is_empty(&adapter->ifp->if_snd))
1373                 ixgbe_start_locked(txr, adapter->ifp);
1374         IXGBE_TX_UNLOCK(txr);
1375
1376         /* Check for fan failure */
1377         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1378             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1379                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1380                     "REPLACE IMMEDIATELY!!\n");
1381                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1382         }
1383
1384         /* Link status change */
1385         if (reg_eicr & IXGBE_EICR_LSC) {
1386                 ixgbe_check_link(&adapter->hw,
1387                     &adapter->link_speed, &adapter->link_up, 0);
1388                 ixgbe_update_link_status(adapter);
1389         }
1390
1391         ixgbe_enable_intr(adapter);
1392 }
1393
1394
1395 /*********************************************************************
1396  *
1397  *  MSIX Queue Interrupt Service routine
1398  *
1399  **********************************************************************/
1400 void
1401 ixgbe_msix_que(void *arg)
1402 {
1403         struct ix_queue *que = arg;
1404         struct adapter  *adapter = que->adapter;
1405         struct tx_ring  *txr = que->txr;
1406
1407         ixgbe_disable_queue(adapter, que->msix);
1408         ++que->irqs;
1409
1410         ixgbe_rxeof(que, adapter->rx_process_limit);
1411
1412         IXGBE_TX_LOCK(txr);
1413         ixgbe_txeof(txr);
1414         if (!ifq_is_empty(&adapter->ifp->if_snd))
1415                 ixgbe_start_locked(txr, adapter->ifp);
1416         IXGBE_TX_UNLOCK(txr);
1417
1418         /* Reenable this interrupt */
1419         ixgbe_enable_queue(adapter, que->msix);
1420 }
1421
1422
1423 static void
1424 ixgbe_msix_link(void *arg)
1425 {
1426         struct adapter  *adapter = arg;
1427         struct ixgbe_hw *hw = &adapter->hw;
1428         u32             reg_eicr;
1429
1430         ++adapter->link_irq;
1431
1432         /* First get the cause */
1433         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1434         /* Clear interrupt with write */
1435         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1436
1437         /* Link status change */
1438         if (reg_eicr & IXGBE_EICR_LSC) {
1439                 ixgbe_check_link(&adapter->hw,
1440                     &adapter->link_speed, &adapter->link_up, 0);
1441                 ixgbe_update_link_status(adapter);
1442         }
1443
1444         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1445 #ifdef IXGBE_FDIR
1446                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1447                         /* This is probably overkill :) */
1448                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1449                                 return;
1450                         /* Disable the interrupt */
1451                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1452                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1453                 } else
1454 #endif
1455                 if (reg_eicr & IXGBE_EICR_ECC) {
1456                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1457                             "Please Reboot!!\n");
1458                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1459                 } else
1460
1461                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1462                         /* Clear the interrupt */
1463                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1464                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1465                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1466                         /* Clear the interrupt */
1467                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1468                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1469                 }
1470         } 
1471
1472         /* Check for fan failure */
1473         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1474             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1475                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1476                     "REPLACE IMMEDIATELY!!\n");
1477                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1478         }
1479
1480         /* Check for over temp condition */
1481         if ((hw->mac.type == ixgbe_mac_X540) &&
1482             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1483                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1484                     "PHY IS SHUT DOWN!!\n");
1485                 device_printf(adapter->dev, "System shutdown required\n");
1486                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1487         }
1488
1489         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1490         return;
1491 }
1492
1493 /*********************************************************************
1494  *
1495  *  Media Ioctl callback
1496  *
1497  *  This routine is called whenever the user queries the status of
1498  *  the interface using ifconfig.
1499  *
1500  **********************************************************************/
1501 static void
1502 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1503 {
1504         struct adapter *adapter = ifp->if_softc;
1505
1506         ASSERT_IFNET_SERIALIZED_ALL(ifp);
1507
1508         INIT_DEBUGOUT("ixgbe_media_status: begin");
1509         ixgbe_update_link_status(adapter);
1510
1511         ifmr->ifm_status = IFM_AVALID;
1512         ifmr->ifm_active = IFM_ETHER;
1513
1514         if (!adapter->link_active)
1515                 return;
1516
1517         ifmr->ifm_status |= IFM_ACTIVE;
1518
1519         switch (adapter->link_speed) {
1520                 case IXGBE_LINK_SPEED_100_FULL:
1521                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1522                         break;
1523                 case IXGBE_LINK_SPEED_1GB_FULL:
1524                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1525                         break;
1526                 case IXGBE_LINK_SPEED_10GB_FULL:
1527                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1528                         break;
1529         }
1530
1531         return;
1532 }
1533
1534 /*********************************************************************
1535  *
1536  *  Media Ioctl callback
1537  *
1538  *  This routine is called when the user changes speed/duplex using
1539  *  media/mediopt option with ifconfig.
1540  *
1541  **********************************************************************/
1542 static int
1543 ixgbe_media_change(struct ifnet * ifp)
1544 {
1545         struct adapter *adapter = ifp->if_softc;
1546         struct ifmedia *ifm = &adapter->media;
1547
1548         INIT_DEBUGOUT("ixgbe_media_change: begin");
1549
1550         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1551                 return (EINVAL);
1552
1553         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1554         case IFM_AUTO:
1555                 adapter->hw.phy.autoneg_advertised =
1556                     IXGBE_LINK_SPEED_100_FULL |
1557                     IXGBE_LINK_SPEED_1GB_FULL |
1558                     IXGBE_LINK_SPEED_10GB_FULL;
1559                 break;
1560         default:
1561                 device_printf(adapter->dev, "Only auto media type\n");
1562                 return (EINVAL);
1563         }
1564
1565         return (0);
1566 }
1567
1568 /*********************************************************************
1569  *
1570  *  This routine maps the mbufs to tx descriptors, allowing the
1571  *  TX engine to transmit the packets. 
1572  *      - return 0 on success, positive on failure
1573  *
1574  **********************************************************************/
1575
1576 static int
1577 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1578 {
1579         struct adapter  *adapter = txr->adapter;
1580         u32             olinfo_status = 0, cmd_type_len;
1581         u32             paylen = 0;
1582         int             i, j, error, nsegs, maxsegs;
1583         int             first, last = 0;
1584         struct mbuf     *m_head;
1585         bus_dma_segment_t segs[adapter->num_segs];
1586         bus_dmamap_t    map;
1587         struct ixgbe_tx_buf *txbuf;
1588         union ixgbe_adv_tx_desc *txd = NULL;
1589
1590         m_head = *m_headp;
1591
1592         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1593                 error = ixgbe_tso_pullup(txr, m_headp);
1594                 if (error)
1595                         return error;
1596                 m_head = *m_headp;
1597         }
1598
1599         /* Basic descriptor defines */
1600         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1601             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1602
1603         if (m_head->m_flags & M_VLANTAG)
1604                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1605
1606         /*
1607          * Important to capture the first descriptor
1608          * used because it will contain the index of
1609          * the one we tell the hardware to report back
1610          */
1611         first = txr->next_avail_desc;
1612         txbuf = &txr->tx_buffers[first];
1613         map = txbuf->map;
1614
1615         /*
1616          * Map the packet for DMA.
1617          */
1618         maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1619         if (maxsegs > adapter->num_segs)
1620                 maxsegs = adapter->num_segs;
1621
1622         error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1623             segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1624         if (error) {
1625                 if (error == ENOBUFS)
1626                         adapter->mbuf_defrag_failed++;
1627                 else
1628                         adapter->no_tx_dma_setup++;
1629
1630                 m_freem(*m_headp);
1631                 *m_headp = NULL;
1632                 return (error);
1633         }
1634
1635         /* Make certain there are enough descriptors */
1636         if (nsegs > txr->tx_avail - 2) {
1637                 txr->no_desc_avail++;
1638                 error = ENOBUFS;
1639                 goto xmit_fail;
1640         }
1641         m_head = *m_headp;
1642
1643         /*
1644         ** Set up the appropriate offload context
1645         ** this becomes the first descriptor of 
1646         ** a packet.
1647         */
1648         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1649                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1650                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1651                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1652                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1653                         ++adapter->tso_tx;
1654                 } else
1655                         return (ENXIO);
1656         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1657                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1658
1659 #ifdef IXGBE_IEEE1588
1660         /* This is changing soon to an mtag detection */
1661         if (we detect this mbuf has a TSTAMP mtag)
1662                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1663 #endif
1664
1665 #ifdef IXGBE_FDIR
1666         /* Do the flow director magic */
1667         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1668                 ++txr->atr_count;
1669                 if (txr->atr_count >= atr_sample_rate) {
1670                         ixgbe_atr(txr, m_head);
1671                         txr->atr_count = 0;
1672                 }
1673         }
1674 #endif
1675         /* Record payload length */
1676         if (paylen == 0)
1677                 olinfo_status |= m_head->m_pkthdr.len <<
1678                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1679
1680         i = txr->next_avail_desc;
1681         for (j = 0; j < nsegs; j++) {
1682                 bus_size_t seglen;
1683                 bus_addr_t segaddr;
1684
1685                 txbuf = &txr->tx_buffers[i];
1686                 txd = &txr->tx_base[i];
1687                 seglen = segs[j].ds_len;
1688                 segaddr = htole64(segs[j].ds_addr);
1689
1690                 txd->read.buffer_addr = segaddr;
1691                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1692                     cmd_type_len |seglen);
1693                 txd->read.olinfo_status = htole32(olinfo_status);
1694                 last = i; /* descriptor that will get completion IRQ */
1695
1696                 if (++i == adapter->num_tx_desc)
1697                         i = 0;
1698
1699                 txbuf->m_head = NULL;
1700                 txbuf->eop_index = -1;
1701         }
1702
1703         txd->read.cmd_type_len |=
1704             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1705         txr->tx_avail -= nsegs;
1706         txr->next_avail_desc = i;
1707
1708         txbuf->m_head = m_head;
1709         /* Swap the dma map between the first and last descriptor */
1710         txr->tx_buffers[first].map = txbuf->map;
1711         txbuf->map = map;
1712         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1713
1714         /* Set the index of the descriptor that will be marked done */
1715         txbuf = &txr->tx_buffers[first];
1716         txbuf->eop_index = last;
1717
1718         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1719             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1720         /*
1721          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1722          * hardware that this frame is available to transmit.
1723          */
1724         ++txr->total_packets;
1725         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1726
1727         return (0);
1728
1729 xmit_fail:
1730         bus_dmamap_unload(txr->txtag, txbuf->map);
1731         return (error);
1732
1733 }
1734
1735 static void
1736 ixgbe_set_promisc(struct adapter *adapter)
1737 {
1738         u_int32_t       reg_rctl;
1739         struct ifnet   *ifp = adapter->ifp;
1740
1741         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1742         reg_rctl &= (~IXGBE_FCTRL_UPE);
1743         reg_rctl &= (~IXGBE_FCTRL_MPE);
1744         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1745
1746         if (ifp->if_flags & IFF_PROMISC) {
1747                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1748                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1749         } else if (ifp->if_flags & IFF_ALLMULTI) {
1750                 reg_rctl |= IXGBE_FCTRL_MPE;
1751                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1752                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1753         }
1754         return;
1755 }
1756
1757
1758 /*********************************************************************
1759  *  Multicast Update
1760  *
1761  *  This routine is called whenever multicast address list is updated.
1762  *
1763  **********************************************************************/
1764 #define IXGBE_RAR_ENTRIES 16
1765
1766 static void
1767 ixgbe_set_multi(struct adapter *adapter)
1768 {
1769         u32     fctrl;
1770         u8      *mta;
1771         u8      *update_ptr;
1772         struct  ifmultiaddr *ifma;
1773         int     mcnt = 0;
1774         struct ifnet   *ifp = adapter->ifp;
1775
1776         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1777
1778         mta = adapter->mta;
1779         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1780             MAX_NUM_MULTICAST_ADDRESSES);
1781
1782         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1783         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1784         if (ifp->if_flags & IFF_PROMISC)
1785                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1786         else if (ifp->if_flags & IFF_ALLMULTI) {
1787                 fctrl |= IXGBE_FCTRL_MPE;
1788                 fctrl &= ~IXGBE_FCTRL_UPE;
1789         } else
1790                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1791         
1792         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1793
1794         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1795                 if (ifma->ifma_addr->sa_family != AF_LINK)
1796                         continue;
1797                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1798                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1799                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1800                 mcnt++;
1801         }
1802
1803         update_ptr = mta;
1804         ixgbe_update_mc_addr_list(&adapter->hw,
1805             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1806
1807         return;
1808 }
1809
1810 /*
1811  * This is an iterator function now needed by the multicast
1812  * shared code. It simply feeds the shared code routine the
1813  * addresses in the array of ixgbe_set_multi() one by one.
1814  */
1815 static u8 *
1816 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1817 {
1818         u8 *addr = *update_ptr;
1819         u8 *newptr;
1820         *vmdq = 0;
1821
1822         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1823         *update_ptr = newptr;
1824         return addr;
1825 }
1826
1827
1828 /*********************************************************************
1829  *  Timer routine
1830  *
1831  *  This routine checks for link status,updates statistics,
1832  *  and runs the watchdog check.
1833  *
1834  **********************************************************************/
1835
1836 static void
1837 ixgbe_local_timer(void *arg)
1838 {
1839         struct adapter  *adapter = arg;
1840         device_t        dev = adapter->dev;
1841         struct ifnet    *ifp = adapter->ifp;
1842         struct ix_queue *que = adapter->queues;
1843         struct tx_ring  *txr = adapter->tx_rings;
1844         int             hung, busy, paused;
1845
1846         IXGBE_CORE_LOCK(adapter);
1847         hung = busy = paused = 0;
1848
1849         /* Check for pluggable optics */
1850         if (adapter->sfp_probe)
1851                 if (!ixgbe_sfp_probe(adapter))
1852                         goto out; /* Nothing to do */
1853
1854         ixgbe_update_link_status(adapter);
1855         ixgbe_update_stats_counters(adapter);
1856
1857         /*
1858          * If the interface has been paused
1859          * then don't do the watchdog check
1860          */
1861         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
1862                 paused = 1;
1863
1864         /*
1865         ** Check the TX queues status
1866         **      - central locked handling of OACTIVE
1867         **      - watchdog only if all queues show hung
1868         */          
1869         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
1870                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
1871                     (paused == 0))
1872                         ++hung;
1873                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
1874                         ++busy;
1875         }
1876         /* Only truely watchdog if all queues show hung */
1877         if (hung == adapter->num_queues)
1878                 goto watchdog;
1879         /* Only turn off the stack flow when ALL are depleted */
1880         if (busy == adapter->num_queues)
1881                 ifp->if_flags |= IFF_OACTIVE;
1882         else if ((ifp->if_flags & IFF_OACTIVE) &&
1883             (busy < adapter->num_queues))
1884                 ifp->if_flags &= ~IFF_OACTIVE;
1885
1886 out:
1887         ixgbe_rearm_queues(adapter, adapter->que_mask);
1888         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1889         IXGBE_CORE_UNLOCK(adapter);
1890         return;
1891
1892 watchdog:
1893         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1894         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1895             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
1896             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
1897         device_printf(dev,"TX(%d) desc avail = %d,"
1898             "Next TX to Clean = %d\n",
1899             txr->me, txr->tx_avail, txr->next_to_clean);
1900         adapter->ifp->if_flags &= ~IFF_RUNNING;
1901         adapter->watchdog_events++;
1902         ixgbe_init_locked(adapter);
1903
1904         IXGBE_CORE_UNLOCK(adapter);
1905 }
1906
1907 /*
1908 ** Note: this routine updates the OS on the link state
1909 **      the real check of the hardware only happens with
1910 **      a link interrupt.
1911 */
1912 static void
1913 ixgbe_update_link_status(struct adapter *adapter)
1914 {
1915         struct ifnet    *ifp = adapter->ifp;
1916         struct tx_ring *txr = adapter->tx_rings;
1917         device_t dev = adapter->dev;
1918
1919
1920         if (adapter->link_up){ 
1921                 if (adapter->link_active == FALSE) {
1922                         if (bootverbose)
1923                                 device_printf(dev,"Link is up %d Gbps %s \n",
1924                                     ((adapter->link_speed == 128)? 10:1),
1925                                     "Full Duplex");
1926                         adapter->link_active = TRUE;
1927                         /* Update any Flow Control changes */
1928                         ixgbe_fc_enable(&adapter->hw);
1929                         ifp->if_link_state = LINK_STATE_UP;
1930                         if_link_state_change(ifp);
1931                 }
1932         } else { /* Link down */
1933                 if (adapter->link_active == TRUE) {
1934                         if (bootverbose)
1935                                 device_printf(dev,"Link is Down\n");
1936                         ifp->if_link_state = LINK_STATE_DOWN;
1937                         if_link_state_change(ifp);
1938                         adapter->link_active = FALSE;
1939                         for (int i = 0; i < adapter->num_queues;
1940                             i++, txr++)
1941                                 txr->queue_status = IXGBE_QUEUE_IDLE;
1942                 }
1943         }
1944
1945         return;
1946 }
1947
1948
1949 /*********************************************************************
1950  *
1951  *  This routine disables all traffic on the adapter by issuing a
1952  *  global reset on the MAC and deallocates TX/RX buffers.
1953  *
1954  **********************************************************************/
1955
1956 static void
1957 ixgbe_stop(void *arg)
1958 {
1959         struct ifnet   *ifp;
1960         struct adapter *adapter = arg;
1961         struct ixgbe_hw *hw = &adapter->hw;
1962         ifp = adapter->ifp;
1963
1964         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1965
1966         INIT_DEBUGOUT("ixgbe_stop: begin\n");
1967         ixgbe_disable_intr(adapter);
1968         callout_stop(&adapter->timer);
1969
1970         /* Let the stack know...*/
1971         ifp->if_flags &= ~IFF_RUNNING;
1972         ifp->if_flags |= IFF_OACTIVE;
1973
1974         ixgbe_reset_hw(hw);
1975         hw->adapter_stopped = FALSE;
1976         ixgbe_stop_adapter(hw);
1977         /* Turn off the laser */
1978         if (hw->phy.multispeed_fiber)
1979                 ixgbe_disable_tx_laser(hw);
1980
1981         /* reprogram the RAR[0] in case user changed it. */
1982         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1983
1984         return;
1985 }
1986
1987
1988 /*********************************************************************
1989  *
1990  *  Determine hardware revision.
1991  *
1992  **********************************************************************/
1993 static void
1994 ixgbe_identify_hardware(struct adapter *adapter)
1995 {
1996         device_t        dev = adapter->dev;
1997         struct ixgbe_hw *hw = &adapter->hw;
1998
1999         /* Save off the information about this board */
2000         hw->vendor_id = pci_get_vendor(dev);
2001         hw->device_id = pci_get_device(dev);
2002         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2003         hw->subsystem_vendor_id =
2004             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2005         hw->subsystem_device_id =
2006             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2007
2008         /* We need this here to set the num_segs below */
2009         ixgbe_set_mac_type(hw);
2010
2011         /* Pick up the 82599 and VF settings */
2012         if (hw->mac.type != ixgbe_mac_82598EB) {
2013                 hw->phy.smart_speed = ixgbe_smart_speed;
2014                 adapter->num_segs = IXGBE_82599_SCATTER;
2015         } else
2016                 adapter->num_segs = IXGBE_82598_SCATTER;
2017
2018         return;
2019 }
2020
2021 /*********************************************************************
2022  *
2023  *  Determine optic type
2024  *
2025  **********************************************************************/
2026 static void
2027 ixgbe_setup_optics(struct adapter *adapter)
2028 {
2029         struct ixgbe_hw *hw = &adapter->hw;
2030         int             layer;
2031         
2032         layer = ixgbe_get_supported_physical_layer(hw);
2033
2034         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2035                 adapter->optics = IFM_10G_T;
2036                 return;
2037         }
2038
2039         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2040                 adapter->optics = IFM_1000_T;
2041                 return;
2042         }
2043
2044         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2045             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2046                 adapter->optics = IFM_10G_LR;
2047                 return;
2048         }
2049
2050         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2051                 adapter->optics = IFM_10G_SR;
2052                 return;
2053         }
2054
2055         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2056                 adapter->optics = IFM_10G_TWINAX;
2057                 return;
2058         }
2059
2060         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2061             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2062                 adapter->optics = IFM_10G_CX4;
2063                 return;
2064         }
2065
2066         /* If we get here just set the default */
2067         adapter->optics = IFM_ETHER | IFM_AUTO;
2068         return;
2069 }
2070
2071 /*********************************************************************
2072  *
2073  *  Setup the Legacy or MSI Interrupt handler
2074  *
2075  **********************************************************************/
2076 static int
2077 ixgbe_allocate_legacy(struct adapter *adapter)
2078 {
2079         device_t dev = adapter->dev;
2080         struct          ix_queue *que = adapter->queues;
2081         int error, rid = 0;
2082         unsigned int intr_flags;
2083
2084         /* MSI RID at 1 */
2085         if (adapter->msix == 1)
2086                 rid = 1;
2087
2088         /* Try allocating a MSI interrupt first */
2089         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2090                 &rid, &intr_flags);
2091
2092         /* We allocate a single interrupt resource */
2093         adapter->res = bus_alloc_resource_any(dev,
2094             SYS_RES_IRQ, &rid, intr_flags);
2095         if (adapter->res == NULL) {
2096                 device_printf(dev, "Unable to allocate bus resource: "
2097                     "interrupt\n");
2098                 return (ENXIO);
2099         }
2100
2101         /* Tasklets for Link, SFP and Multispeed Fiber */
2102         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2103         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2104 #ifdef IXGBE_FDIR
2105         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2106 #endif
2107
2108         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2109             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2110                 device_printf(dev, "Failed to register fast interrupt "
2111                     "handler: %d\n", error);
2112                 taskqueue_free(adapter->tq);
2113                 adapter->tq = NULL;
2114                 return (error);
2115         }
2116         /* For simplicity in the handlers */
2117         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2118
2119         return (0);
2120 }
2121
2122
2123 /*********************************************************************
2124  *
2125  *  Setup MSIX Interrupt resources and handlers 
2126  *
2127  **********************************************************************/
2128 static int
2129 ixgbe_allocate_msix(struct adapter *adapter)
2130 {
2131         device_t        dev = adapter->dev;
2132         struct          ix_queue *que = adapter->queues;
2133         int             error, rid, vector = 0;
2134         char            desc[16];
2135
2136         error = pci_setup_msix(dev);
2137         if (error) {
2138                 device_printf(dev, "MSI-X setup failed\n");
2139                 return (error);
2140         }
2141
2142         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2143                 rid = vector + 1;
2144
2145                 /*
2146                 ** Bind the msix vector, and thus the
2147                 ** ring to the corresponding cpu.
2148                 */
2149                 error = pci_alloc_msix_vector(dev, vector, &rid, i);
2150                 if (error) {
2151                         device_printf(dev, "pci_alloc_msix_vector failed\n");
2152                         return (error);
2153                 }
2154
2155                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2156                     RF_SHAREABLE | RF_ACTIVE);
2157                 if (que->res == NULL) {
2158                         device_printf(dev,"Unable to allocate"
2159                             " bus resource: que interrupt [%d]\n", vector);
2160                         return (ENXIO);
2161                 }
2162                 /* Set the handler function */
2163                 ksnprintf(desc, sizeof(desc), "%s que %d",
2164                     device_get_nameunit(dev), i);
2165                 error = bus_setup_intr_descr(dev, que->res, INTR_MPSAFE,
2166                     ixgbe_msix_que, que, &que->tag, &que->serializer, desc);
2167                 if (error) {
2168                         que->res = NULL;
2169                         device_printf(dev, "Failed to register QUE handler");
2170                         return (error);
2171                 }
2172                 que->msix = vector;
2173                 adapter->que_mask |= (u64)(1 << que->msix);
2174         }
2175
2176         /* and Link, bind vector to cpu #0 */
2177         rid = vector + 1;
2178         error = pci_alloc_msix_vector(dev, vector, &rid, 0);
2179         if (error) {
2180                 device_printf(dev, "pci_alloc_msix_vector failed\n");
2181                 return (error);
2182         }
2183         adapter->res = bus_alloc_resource_any(dev,
2184             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2185         if (!adapter->res) {
2186                 device_printf(dev,"Unable to allocate"
2187             " bus resource: Link interrupt [%d]\n", rid);
2188                 return (ENXIO);
2189         }
2190         /* Set the link handler function */
2191         error = bus_setup_intr_descr(dev, adapter->res, INTR_MPSAFE,
2192             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer,
2193             "link");
2194         if (error) {
2195                 adapter->res = NULL;
2196                 device_printf(dev, "Failed to register LINK handler");
2197                 return (error);
2198         }
2199         pci_enable_msix(dev);
2200
2201         adapter->linkvec = vector;
2202         /* Tasklets for Link, SFP and Multispeed Fiber */
2203         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2204         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2205 #ifdef IXGBE_FDIR
2206         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2207 #endif
2208
2209         return (0);
2210 }
2211
2212 /*
2213  * Setup Either MSI/X or MSI
2214  */
2215 static int
2216 ixgbe_setup_msix(struct adapter *adapter)
2217 {
2218         device_t dev = adapter->dev;
2219         int rid, want, queues, msgs;
2220
2221         /* Override by tuneable */
2222         if (ixgbe_enable_msix == 0)
2223                 goto msi;
2224
2225         /* First try MSI/X */
2226         rid = PCIR_BAR(MSIX_82598_BAR);
2227         adapter->msix_mem = bus_alloc_resource_any(dev,
2228             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2229         if (!adapter->msix_mem) {
2230                 rid += 4;       /* 82599 maps in higher BAR */
2231                 adapter->msix_mem = bus_alloc_resource_any(dev,
2232                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2233         }
2234         if (!adapter->msix_mem) {
2235                 /* May not be enabled */
2236                 device_printf(adapter->dev,
2237                     "Unable to map MSIX table \n");
2238                 goto msi;
2239         }
2240
2241         msgs = pci_msix_count(dev); 
2242         if (msgs == 0) { /* system has msix disabled */
2243                 bus_release_resource(dev, SYS_RES_MEMORY,
2244                     rid, adapter->msix_mem);
2245                 adapter->msix_mem = NULL;
2246                 goto msi;
2247         }
2248
2249         /* Figure out a reasonable auto config value */
2250         queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2251
2252         if (ixgbe_num_queues != 0)
2253                 queues = ixgbe_num_queues;
2254         /* Set max queues to 8 when autoconfiguring */
2255         else if ((ixgbe_num_queues == 0) && (queues > 8))
2256                 queues = 8;
2257
2258         /*
2259         ** Want one vector (RX/TX pair) per queue
2260         ** plus an additional for Link.
2261         */
2262         want = queues + 1;
2263         if (msgs >= want)
2264                 msgs = want;
2265         else {
2266                 device_printf(adapter->dev,
2267                     "MSIX Configuration Problem, "
2268                     "%d vectors but %d queues wanted!\n",
2269                     msgs, want);
2270                 return (0); /* Will go to Legacy setup */
2271         }
2272         if (msgs) {
2273                 device_printf(adapter->dev,
2274                     "Using MSIX interrupts with %d vectors\n", msgs);
2275                 adapter->num_queues = queues;
2276                 return (msgs);
2277         }
2278 msi:
2279         msgs = pci_msi_count(dev);
2280         return (msgs);
2281 }
2282
2283
2284 static int
2285 ixgbe_allocate_pci_resources(struct adapter *adapter)
2286 {
2287         int             rid;
2288         device_t        dev = adapter->dev;
2289
2290         rid = PCIR_BAR(0);
2291         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2292             &rid, RF_ACTIVE);
2293
2294         if (!(adapter->pci_mem)) {
2295                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2296                 return (ENXIO);
2297         }
2298
2299         adapter->osdep.mem_bus_space_tag =
2300                 rman_get_bustag(adapter->pci_mem);
2301         adapter->osdep.mem_bus_space_handle =
2302                 rman_get_bushandle(adapter->pci_mem);
2303         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2304
2305         /* Legacy defaults */
2306         adapter->num_queues = 1;
2307         adapter->hw.back = &adapter->osdep;
2308
2309         /*
2310         ** Now setup MSI or MSI/X, should
2311         ** return us the number of supported
2312         ** vectors. (Will be 1 for MSI)
2313         */
2314         adapter->msix = ixgbe_setup_msix(adapter);
2315         return (0);
2316 }
2317
2318 static void
2319 ixgbe_free_pci_resources(struct adapter * adapter)
2320 {
2321         struct          ix_queue *que = adapter->queues;
2322         device_t        dev = adapter->dev;
2323         int             rid, memrid;
2324
2325         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2326                 memrid = PCIR_BAR(MSIX_82598_BAR);
2327         else
2328                 memrid = PCIR_BAR(MSIX_82599_BAR);
2329
2330         /*
2331         ** There is a slight possibility of a failure mode
2332         ** in attach that will result in entering this function
2333         ** before interrupt resources have been initialized, and
2334         ** in that case we do not want to execute the loops below
2335         ** We can detect this reliably by the state of the adapter
2336         ** res pointer.
2337         */
2338         if (adapter->res == NULL)
2339                 goto mem;
2340
2341         /*
2342         **  Release all msix queue resources:
2343         */
2344         for (int i = 0; i < adapter->num_queues; i++, que++) {
2345                 rid = que->msix + 1;
2346                 if (que->tag != NULL) {
2347                         bus_teardown_intr(dev, que->res, que->tag);
2348                         que->tag = NULL;
2349                 }
2350                 if (que->res != NULL)
2351                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2352         }
2353
2354
2355         /* Clean the Legacy or Link interrupt last */
2356         if (adapter->linkvec) /* we are doing MSIX */
2357                 rid = adapter->linkvec + 1;
2358         else
2359                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2360
2361         if (adapter->tag != NULL) {
2362                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2363                 adapter->tag = NULL;
2364         }
2365         if (adapter->res != NULL)
2366                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2367         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2368                 pci_release_msi(adapter->dev);
2369
2370 mem:
2371         if (adapter->msix)
2372                 pci_release_msi(dev);
2373
2374         if (adapter->msix_mem != NULL)
2375                 bus_release_resource(dev, SYS_RES_MEMORY,
2376                     memrid, adapter->msix_mem);
2377
2378         if (adapter->pci_mem != NULL)
2379                 bus_release_resource(dev, SYS_RES_MEMORY,
2380                     PCIR_BAR(0), adapter->pci_mem);
2381
2382         return;
2383 }
2384
2385 /*********************************************************************
2386  *
2387  *  Setup networking device structure and register an interface.
2388  *
2389  **********************************************************************/
2390 static int
2391 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2392 {
2393         struct ixgbe_hw *hw = &adapter->hw;
2394         struct ifnet   *ifp;
2395
2396         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2397
2398         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2399         if (ifp == NULL) {
2400                 device_printf(dev, "can not allocate ifnet structure\n");
2401                 return (-1);
2402         }
2403         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2404         ifp->if_baudrate = 1000000000;
2405         ifp->if_init = ixgbe_init;
2406         ifp->if_softc = adapter;
2407         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2408         ifp->if_ioctl = ixgbe_ioctl;
2409         ifp->if_start = ixgbe_start;
2410 #if 0 /* __FreeBSD_version >= 800000 */
2411         ifp->if_transmit = ixgbe_mq_start;
2412         ifp->if_qflush = ixgbe_qflush;
2413 #endif
2414         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2415
2416         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2417
2418         adapter->max_frame_size =
2419             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2420
2421         /*
2422          * Tell the upper layer(s) we support long frames.
2423          */
2424         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2425
2426         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2427         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2428         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2429 #if 0 /* NET_TSO */
2430                              |  IFCAP_VLAN_HWTSO
2431 #endif
2432                              |  IFCAP_VLAN_MTU;
2433         ifp->if_capenable = ifp->if_capabilities;
2434
2435         /* Don't enable LRO by default */
2436 #if 0 /* NET_LRO */
2437         ifp->if_capabilities |= IFCAP_LRO;
2438 #endif
2439
2440         /*
2441         ** Don't turn this on by default, if vlans are
2442         ** created on another pseudo device (eg. lagg)
2443         ** then vlan events are not passed thru, breaking
2444         ** operation, but with HW FILTER off it works. If
2445         ** using vlans directly on the ixgbe driver you can
2446         ** enable this and get full hardware tag filtering.
2447         */
2448         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2449
2450         /*
2451          * Specify the media types supported by this adapter and register
2452          * callbacks to update media and link information
2453          */
2454         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2455                      ixgbe_media_status);
2456         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2457         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2458         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2459                 ifmedia_add(&adapter->media,
2460                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2461                 ifmedia_add(&adapter->media,
2462                     IFM_ETHER | IFM_1000_T, 0, NULL);
2463         }
2464         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2465         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2466
2467         return (0);
2468 }
2469
2470 static void
2471 ixgbe_config_link(struct adapter *adapter)
2472 {
2473         struct ixgbe_hw *hw = &adapter->hw;
2474         u32     autoneg, err = 0;
2475         bool    sfp, negotiate;
2476
2477         sfp = ixgbe_is_sfp(hw);
2478
2479         if (sfp) { 
2480                 if (hw->phy.multispeed_fiber) {
2481                         hw->mac.ops.setup_sfp(hw);
2482                         ixgbe_enable_tx_laser(hw);
2483                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2484                 } else
2485                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2486         } else {
2487                 if (hw->mac.ops.check_link)
2488                         err = ixgbe_check_link(hw, &autoneg,
2489                             &adapter->link_up, FALSE);
2490                 if (err)
2491                         goto out;
2492                 autoneg = hw->phy.autoneg_advertised;
2493                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2494                         err  = hw->mac.ops.get_link_capabilities(hw,
2495                             &autoneg, &negotiate);
2496                 if (err)
2497                         goto out;
2498                 if (hw->mac.ops.setup_link)
2499                         err = hw->mac.ops.setup_link(hw, autoneg,
2500                             negotiate, adapter->link_up);
2501         }
2502 out:
2503         return;
2504 }
2505
2506 /********************************************************************
2507  * Manage DMA'able memory.
2508  *******************************************************************/
2509 static void
2510 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2511 {
2512         if (error)
2513                 return;
2514         *(bus_addr_t *) arg = segs->ds_addr;
2515         return;
2516 }
2517
2518 static int
2519 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2520                 struct ixgbe_dma_alloc *dma, int mapflags)
2521 {
2522         device_t dev = adapter->dev;
2523         int             r;
2524
2525         r = bus_dma_tag_create(NULL,    /* parent */
2526                                DBA_ALIGN, 0,    /* alignment, bounds */
2527                                BUS_SPACE_MAXADDR,       /* lowaddr */
2528                                BUS_SPACE_MAXADDR,       /* highaddr */
2529                                NULL, NULL,      /* filter, filterarg */
2530                                size,    /* maxsize */
2531                                1,       /* nsegments */
2532                                size,    /* maxsegsize */
2533                                BUS_DMA_ALLOCNOW,        /* flags */
2534                                &dma->dma_tag);
2535         if (r != 0) {
2536                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2537                        "error %u\n", r);
2538                 goto fail_0;
2539         }
2540         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2541                              BUS_DMA_NOWAIT, &dma->dma_map);
2542         if (r != 0) {
2543                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2544                        "error %u\n", r);
2545                 goto fail_1;
2546         }
2547         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2548                             size,
2549                             ixgbe_dmamap_cb,
2550                             &dma->dma_paddr,
2551                             mapflags | BUS_DMA_NOWAIT);
2552         if (r != 0) {
2553                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2554                        "error %u\n", r);
2555                 goto fail_2;
2556         }
2557         dma->dma_size = size;
2558         return (0);
2559 fail_2:
2560         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2561 fail_1:
2562         bus_dma_tag_destroy(dma->dma_tag);
2563 fail_0:
2564         dma->dma_map = NULL;
2565         dma->dma_tag = NULL;
2566         return (r);
2567 }
2568
2569 static void
2570 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2571 {
2572         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2573             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2574         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2575         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2576         bus_dma_tag_destroy(dma->dma_tag);
2577 }
2578
2579
2580 /*********************************************************************
2581  *
2582  *  Allocate memory for the transmit and receive rings, and then
2583  *  the descriptors associated with each, called only once at attach.
2584  *
2585  **********************************************************************/
2586 static int
2587 ixgbe_allocate_queues(struct adapter *adapter)
2588 {
2589         device_t        dev = adapter->dev;
2590         struct ix_queue *que;
2591         struct tx_ring  *txr;
2592         struct rx_ring  *rxr;
2593         int rsize, tsize, error = IXGBE_SUCCESS;
2594         int txconf = 0, rxconf = 0;
2595
2596         /* First allocate the top level queue structs */
2597         if (!(adapter->queues =
2598             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2599             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2600                 device_printf(dev, "Unable to allocate queue memory\n");
2601                 error = ENOMEM;
2602                 goto fail;
2603         }
2604
2605         /* First allocate the TX ring struct memory */
2606         if (!(adapter->tx_rings =
2607             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2608             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2609                 device_printf(dev, "Unable to allocate TX ring memory\n");
2610                 error = ENOMEM;
2611                 goto tx_fail;
2612         }
2613
2614         /* Next allocate the RX */
2615         if (!(adapter->rx_rings =
2616             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2617             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2618                 device_printf(dev, "Unable to allocate RX ring memory\n");
2619                 error = ENOMEM;
2620                 goto rx_fail;
2621         }
2622
2623         /* For the ring itself */
2624         tsize = roundup2(adapter->num_tx_desc *
2625             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2626
2627         /*
2628          * Now set up the TX queues, txconf is needed to handle the
2629          * possibility that things fail midcourse and we need to
2630          * undo memory gracefully
2631          */ 
2632         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2633                 /* Set up some basics */
2634                 txr = &adapter->tx_rings[i];
2635                 txr->adapter = adapter;
2636                 txr->me = i;
2637
2638                 /* Initialize the TX side lock */
2639                 IXGBE_TX_LOCK_INIT(txr);
2640
2641                 if (ixgbe_dma_malloc(adapter, tsize,
2642                         &txr->txdma, BUS_DMA_NOWAIT)) {
2643                         device_printf(dev,
2644                             "Unable to allocate TX Descriptor memory\n");
2645                         error = ENOMEM;
2646                         goto err_tx_desc;
2647                 }
2648                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2649                 bzero((void *)txr->tx_base, tsize);
2650
2651                 /* Now allocate transmit buffers for the ring */
2652                 if (ixgbe_allocate_transmit_buffers(txr)) {
2653                         device_printf(dev,
2654                             "Critical Failure setting up transmit buffers\n");
2655                         error = ENOMEM;
2656                         goto err_tx_desc;
2657                 }
2658 #if 0 /* __FreeBSD_version >= 800000 */
2659                 /* Allocate a buf ring */
2660                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2661                     M_WAITOK, &txr->tx_mtx);
2662                 if (txr->br == NULL) {
2663                         device_printf(dev,
2664                             "Critical Failure setting up buf ring\n");
2665                         error = ENOMEM;
2666                         goto err_tx_desc;
2667                 }
2668 #endif
2669         }
2670
2671         /*
2672          * Next the RX queues...
2673          */ 
2674         rsize = roundup2(adapter->num_rx_desc *
2675             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2676         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2677                 rxr = &adapter->rx_rings[i];
2678                 /* Set up some basics */
2679                 rxr->adapter = adapter;
2680                 rxr->me = i;
2681
2682                 /* Initialize the RX side lock */
2683                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2684                     device_get_nameunit(dev), rxr->me);
2685                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2686
2687                 if (ixgbe_dma_malloc(adapter, rsize,
2688                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2689                         device_printf(dev,
2690                             "Unable to allocate RxDescriptor memory\n");
2691                         error = ENOMEM;
2692                         goto err_rx_desc;
2693                 }
2694                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2695                 bzero((void *)rxr->rx_base, rsize);
2696
2697                 /* Allocate receive buffers for the ring*/
2698                 if (ixgbe_allocate_receive_buffers(rxr)) {
2699                         device_printf(dev,
2700                             "Critical Failure setting up receive buffers\n");
2701                         error = ENOMEM;
2702                         goto err_rx_desc;
2703                 }
2704         }
2705
2706         /*
2707         ** Finally set up the queue holding structs
2708         */
2709         for (int i = 0; i < adapter->num_queues; i++) {
2710                 que = &adapter->queues[i];
2711                 que->adapter = adapter;
2712                 que->txr = &adapter->tx_rings[i];
2713                 que->rxr = &adapter->rx_rings[i];
2714         }
2715
2716         return (0);
2717
2718 err_rx_desc:
2719         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2720                 ixgbe_dma_free(adapter, &rxr->rxdma);
2721 err_tx_desc:
2722         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2723                 ixgbe_dma_free(adapter, &txr->txdma);
2724         kfree(adapter->rx_rings, M_DEVBUF);
2725 rx_fail:
2726         kfree(adapter->tx_rings, M_DEVBUF);
2727 tx_fail:
2728         kfree(adapter->queues, M_DEVBUF);
2729 fail:
2730         return (error);
2731 }
2732
2733 /*********************************************************************
2734  *
2735  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2736  *  the information needed to transmit a packet on the wire. This is
2737  *  called only once at attach, setup is done every reset.
2738  *
2739  **********************************************************************/
2740 static int
2741 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2742 {
2743         struct adapter *adapter = txr->adapter;
2744         device_t dev = adapter->dev;
2745         struct ixgbe_tx_buf *txbuf;
2746         int error, i;
2747
2748         /*
2749          * Setup DMA descriptor areas.
2750          */
2751         if ((error = bus_dma_tag_create(
2752                                NULL,    /* parent */
2753                                1, 0,            /* alignment, bounds */
2754                                BUS_SPACE_MAXADDR,       /* lowaddr */
2755                                BUS_SPACE_MAXADDR,       /* highaddr */
2756                                NULL, NULL,              /* filter, filterarg */
2757                                IXGBE_TSO_SIZE,          /* maxsize */
2758                                adapter->num_segs,       /* nsegments */
2759                                PAGE_SIZE,               /* maxsegsize */
2760                                0,                       /* flags */
2761                                &txr->txtag))) {
2762                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2763                 goto fail;
2764         }
2765
2766         if (!(txr->tx_buffers =
2767             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2768             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2769                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2770                 error = ENOMEM;
2771                 goto fail;
2772         }
2773
2774         /* Create the descriptor buffer dma maps */
2775         txbuf = txr->tx_buffers;
2776         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2777                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2778                 if (error != 0) {
2779                         device_printf(dev, "Unable to create TX DMA map\n");
2780                         goto fail;
2781                 }
2782         }
2783
2784         return 0;
2785 fail:
2786         /* We free all, it handles case where we are in the middle */
2787         ixgbe_free_transmit_structures(adapter);
2788         return (error);
2789 }
2790
2791 /*********************************************************************
2792  *
2793  *  Initialize a transmit ring.
2794  *
2795  **********************************************************************/
2796 static void
2797 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2798 {
2799         struct adapter *adapter = txr->adapter;
2800         struct ixgbe_tx_buf *txbuf;
2801         int i;
2802 #ifdef DEV_NETMAP
2803         struct netmap_adapter *na = NA(adapter->ifp);
2804         struct netmap_slot *slot;
2805 #endif /* DEV_NETMAP */
2806
2807         /* Clear the old ring contents */
2808         IXGBE_TX_LOCK(txr);
2809 #ifdef DEV_NETMAP
2810         /*
2811          * (under lock): if in netmap mode, do some consistency
2812          * checks and set slot to entry 0 of the netmap ring.
2813          */
2814         slot = netmap_reset(na, NR_TX, txr->me, 0);
2815 #endif /* DEV_NETMAP */
2816         bzero((void *)txr->tx_base,
2817               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
2818         /* Reset indices */
2819         txr->next_avail_desc = 0;
2820         txr->next_to_clean = 0;
2821
2822         /* Free any existing tx buffers. */
2823         txbuf = txr->tx_buffers;
2824         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2825                 if (txbuf->m_head != NULL) {
2826                         bus_dmamap_sync(txr->txtag, txbuf->map,
2827                             BUS_DMASYNC_POSTWRITE);
2828                         bus_dmamap_unload(txr->txtag, txbuf->map);
2829                         m_freem(txbuf->m_head);
2830                         txbuf->m_head = NULL;
2831                 }
2832 #ifdef DEV_NETMAP
2833                 /*
2834                  * In netmap mode, set the map for the packet buffer.
2835                  * NOTE: Some drivers (not this one) also need to set
2836                  * the physical buffer address in the NIC ring.
2837                  * Slots in the netmap ring (indexed by "si") are
2838                  * kring->nkr_hwofs positions "ahead" wrt the
2839                  * corresponding slot in the NIC ring. In some drivers
2840                  * (not here) nkr_hwofs can be negative. Function
2841                  * netmap_idx_n2k() handles wraparounds properly.
2842                  */
2843                 if (slot) {
2844                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
2845                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
2846                 }
2847 #endif /* DEV_NETMAP */
2848                 /* Clear the EOP index */
2849                 txbuf->eop_index = -1;
2850         }
2851
2852 #ifdef IXGBE_FDIR
2853         /* Set the rate at which we sample packets */
2854         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
2855                 txr->atr_sample = atr_sample_rate;
2856 #endif
2857
2858         /* Set number of descriptors available */
2859         txr->tx_avail = adapter->num_tx_desc;
2860
2861         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2862             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2863         IXGBE_TX_UNLOCK(txr);
2864 }
2865
2866 /*********************************************************************
2867  *
2868  *  Initialize all transmit rings.
2869  *
2870  **********************************************************************/
2871 static int
2872 ixgbe_setup_transmit_structures(struct adapter *adapter)
2873 {
2874         struct tx_ring *txr = adapter->tx_rings;
2875
2876         for (int i = 0; i < adapter->num_queues; i++, txr++)
2877                 ixgbe_setup_transmit_ring(txr);
2878
2879         return (0);
2880 }
2881
2882 /*********************************************************************
2883  *
2884  *  Enable transmit unit.
2885  *
2886  **********************************************************************/
2887 static void
2888 ixgbe_initialize_transmit_units(struct adapter *adapter)
2889 {
2890         struct tx_ring  *txr = adapter->tx_rings;
2891         struct ixgbe_hw *hw = &adapter->hw;
2892
2893         /* Setup the Base and Length of the Tx Descriptor Ring */
2894
2895         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2896                 u64     tdba = txr->txdma.dma_paddr;
2897                 u32     txctrl;
2898
2899                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2900                        (tdba & 0x00000000ffffffffULL));
2901                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2902                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2903                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2904
2905                 /* Setup the HW Tx Head and Tail descriptor pointers */
2906                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2907                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2908
2909                 /* Setup Transmit Descriptor Cmd Settings */
2910                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2911                 txr->queue_status = IXGBE_QUEUE_IDLE;
2912
2913                 /* Disable Head Writeback */
2914                 switch (hw->mac.type) {
2915                 case ixgbe_mac_82598EB:
2916                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2917                         break;
2918                 case ixgbe_mac_82599EB:
2919                 case ixgbe_mac_X540:
2920                 default:
2921                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2922                         break;
2923                 }
2924                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2925                 switch (hw->mac.type) {
2926                 case ixgbe_mac_82598EB:
2927                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2928                         break;
2929                 case ixgbe_mac_82599EB:
2930                 case ixgbe_mac_X540:
2931                 default:
2932                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2933                         break;
2934                 }
2935
2936         }
2937
2938         if (hw->mac.type != ixgbe_mac_82598EB) {
2939                 u32 dmatxctl, rttdcs;
2940                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2941                 dmatxctl |= IXGBE_DMATXCTL_TE;
2942                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2943                 /* Disable arbiter to set MTQC */
2944                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2945                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2946                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2947                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2948                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2949                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2950         }
2951
2952         return;
2953 }
2954
2955 /*********************************************************************
2956  *
2957  *  Free all transmit rings.
2958  *
2959  **********************************************************************/
2960 static void
2961 ixgbe_free_transmit_structures(struct adapter *adapter)
2962 {
2963         struct tx_ring *txr = adapter->tx_rings;
2964
2965         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2966                 IXGBE_TX_LOCK(txr);
2967                 ixgbe_free_transmit_buffers(txr);
2968                 ixgbe_dma_free(adapter, &txr->txdma);
2969                 IXGBE_TX_UNLOCK(txr);
2970                 IXGBE_TX_LOCK_DESTROY(txr);
2971         }
2972         kfree(adapter->tx_rings, M_DEVBUF);
2973 }
2974
2975 /*********************************************************************
2976  *
2977  *  Free transmit ring related data structures.
2978  *
2979  **********************************************************************/
2980 static void
2981 ixgbe_free_transmit_buffers(struct tx_ring *txr)
2982 {
2983         struct adapter *adapter = txr->adapter;
2984         struct ixgbe_tx_buf *tx_buffer;
2985         int             i;
2986
2987         INIT_DEBUGOUT("free_transmit_ring: begin");
2988
2989         if (txr->tx_buffers == NULL)
2990                 return;
2991
2992         tx_buffer = txr->tx_buffers;
2993         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2994                 if (tx_buffer->m_head != NULL) {
2995                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
2996                             BUS_DMASYNC_POSTWRITE);
2997                         bus_dmamap_unload(txr->txtag,
2998                             tx_buffer->map);
2999                         m_freem(tx_buffer->m_head);
3000                         tx_buffer->m_head = NULL;
3001                         if (tx_buffer->map != NULL) {
3002                                 bus_dmamap_destroy(txr->txtag,
3003                                     tx_buffer->map);
3004                                 tx_buffer->map = NULL;
3005                         }
3006                 } else if (tx_buffer->map != NULL) {
3007                         bus_dmamap_unload(txr->txtag,
3008                             tx_buffer->map);
3009                         bus_dmamap_destroy(txr->txtag,
3010                             tx_buffer->map);
3011                         tx_buffer->map = NULL;
3012                 }
3013         }
3014 #if 0 /* __FreeBSD_version >= 800000 */
3015         if (txr->br != NULL)
3016                 buf_ring_free(txr->br, M_DEVBUF);
3017 #endif
3018         if (txr->tx_buffers != NULL) {
3019                 kfree(txr->tx_buffers, M_DEVBUF);
3020                 txr->tx_buffers = NULL;
3021         }
3022         if (txr->txtag != NULL) {
3023                 bus_dma_tag_destroy(txr->txtag);
3024                 txr->txtag = NULL;
3025         }
3026         return;
3027 }
3028
3029 /*********************************************************************
3030  *
3031  *  Advanced Context Descriptor setup for VLAN or CSUM
3032  *
3033  **********************************************************************/
3034
3035 static bool
3036 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3037 {
3038         struct adapter *adapter = txr->adapter;
3039         struct ixgbe_adv_tx_context_desc *TXD;
3040         struct ixgbe_tx_buf        *tx_buffer;
3041         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3042         struct ether_vlan_header *eh;
3043         struct ip *ip;
3044         struct ip6_hdr *ip6;
3045         int  ehdrlen, ip_hlen = 0;
3046         u16     etype;
3047         u8      ipproto = 0;
3048         bool    offload = TRUE;
3049         int ctxd = txr->next_avail_desc;
3050         u16 vtag = 0;
3051
3052
3053         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3054                 offload = FALSE;
3055
3056         tx_buffer = &txr->tx_buffers[ctxd];
3057         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3058
3059         /*
3060         ** In advanced descriptors the vlan tag must 
3061         ** be placed into the descriptor itself.
3062         */
3063         if (mp->m_flags & M_VLANTAG) {
3064                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3065                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3066         } else if (offload == FALSE)
3067                 return FALSE;
3068
3069         /*
3070          * Determine where frame payload starts.
3071          * Jump over vlan headers if already present,
3072          * helpful for QinQ too.
3073          */
3074         eh = mtod(mp, struct ether_vlan_header *);
3075         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3076                 etype = ntohs(eh->evl_proto);
3077                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3078         } else {
3079                 etype = ntohs(eh->evl_encap_proto);
3080                 ehdrlen = ETHER_HDR_LEN;
3081         }
3082
3083         /* Set the ether header length */
3084         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3085
3086         switch (etype) {
3087                 case ETHERTYPE_IP:
3088                         ip = (struct ip *)(mp->m_data + ehdrlen);
3089                         ip_hlen = ip->ip_hl << 2;
3090                         ipproto = ip->ip_p;
3091                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3092                         break;
3093                 case ETHERTYPE_IPV6:
3094                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3095                         ip_hlen = sizeof(struct ip6_hdr);
3096                         /* XXX-BZ this will go badly in case of ext hdrs. */
3097                         ipproto = ip6->ip6_nxt;
3098                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3099                         break;
3100                 default:
3101                         offload = FALSE;
3102                         break;
3103         }
3104
3105         vlan_macip_lens |= ip_hlen;
3106         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3107
3108         switch (ipproto) {
3109                 case IPPROTO_TCP:
3110                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3111                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3112                         break;
3113
3114                 case IPPROTO_UDP:
3115                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3116                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3117                         break;
3118
3119 #if 0
3120                 case IPPROTO_SCTP:
3121                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3122                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3123                         break;
3124 #endif
3125                 default:
3126                         offload = FALSE;
3127                         break;
3128         }
3129
3130         /* Now copy bits into descriptor */
3131         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3132         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3133         TXD->seqnum_seed = htole32(0);
3134         TXD->mss_l4len_idx = htole32(0);
3135
3136         tx_buffer->m_head = NULL;
3137         tx_buffer->eop_index = -1;
3138
3139         /* We've consumed the first desc, adjust counters */
3140         if (++ctxd == adapter->num_tx_desc)
3141                 ctxd = 0;
3142         txr->next_avail_desc = ctxd;
3143         --txr->tx_avail;
3144
3145         return (offload);
3146 }
3147
3148 /**********************************************************************
3149  *
3150  *  Setup work for hardware segmentation offload (TSO) on
3151  *  adapters using advanced tx descriptors
3152  *
3153  **********************************************************************/
3154 static bool
3155 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3156     u32 *olinfo_status)
3157 {
3158         struct adapter *adapter = txr->adapter;
3159         struct ixgbe_adv_tx_context_desc *TXD;
3160         struct ixgbe_tx_buf        *tx_buffer;
3161         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3162         u16 vtag = 0, eh_type;
3163         u32 mss_l4len_idx = 0, len;
3164         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3165         struct ether_vlan_header *eh;
3166 #if 0 /* IPv6 TSO */
3167 #ifdef INET6
3168         struct ip6_hdr *ip6;
3169 #endif
3170 #endif
3171 #ifdef INET
3172         struct ip *ip;
3173 #endif
3174         struct tcphdr *th;
3175
3176
3177         /*
3178          * Determine where frame payload starts.
3179          * Jump over vlan headers if already present
3180          */
3181         eh = mtod(mp, struct ether_vlan_header *);
3182         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3183                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3184                 eh_type = eh->evl_proto;
3185         } else {
3186                 ehdrlen = ETHER_HDR_LEN;
3187                 eh_type = eh->evl_encap_proto;
3188         }
3189
3190         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3191         len = ehdrlen + sizeof(struct tcphdr);
3192         switch (ntohs(eh_type)) {
3193 #if 0 /* IPv6 TSO */
3194 #ifdef INET6
3195         case ETHERTYPE_IPV6:
3196                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3197                         return FALSE;
3198                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3199                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3200                 if (ip6->ip6_nxt != IPPROTO_TCP)
3201                         return FALSE;
3202                 ip_hlen = sizeof(struct ip6_hdr);
3203                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3204                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3205                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3206                 break;
3207 #endif
3208 #endif
3209 #ifdef INET
3210         case ETHERTYPE_IP:
3211                 if (mp->m_len < len + sizeof(struct ip))
3212                         return FALSE;
3213                 ip = (struct ip *)(mp->m_data + ehdrlen);
3214                 if (ip->ip_p != IPPROTO_TCP)
3215                         return FALSE;
3216                 ip->ip_sum = 0;
3217                 ip_hlen = ip->ip_hl << 2;
3218                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3219                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3220                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3221                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3222                 /* Tell transmit desc to also do IPv4 checksum. */
3223                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3224                 break;
3225 #endif
3226         default:
3227                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3228                     __func__, ntohs(eh_type));
3229                 break;
3230         }
3231
3232         ctxd = txr->next_avail_desc;
3233         tx_buffer = &txr->tx_buffers[ctxd];
3234         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3235
3236         tcp_hlen = th->th_off << 2;
3237
3238         /* This is used in the transmit desc in encap */
3239         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3240
3241         /* VLAN MACLEN IPLEN */
3242         if (mp->m_flags & M_VLANTAG) {
3243                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3244                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3245         }
3246
3247         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3248         vlan_macip_lens |= ip_hlen;
3249         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3250
3251         /* ADV DTYPE TUCMD */
3252         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3253         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3254         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3255
3256         /* MSS L4LEN IDX */
3257         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3258         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3259         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3260
3261         TXD->seqnum_seed = htole32(0);
3262         tx_buffer->m_head = NULL;
3263         tx_buffer->eop_index = -1;
3264
3265         if (++ctxd == adapter->num_tx_desc)
3266                 ctxd = 0;
3267
3268         txr->tx_avail--;
3269         txr->next_avail_desc = ctxd;
3270         return TRUE;
3271 }
3272
3273 #ifdef IXGBE_FDIR
3274 /*
3275 ** This routine parses packet headers so that Flow
3276 ** Director can make a hashed filter table entry 
3277 ** allowing traffic flows to be identified and kept
3278 ** on the same cpu.  This would be a performance
3279 ** hit, but we only do it at IXGBE_FDIR_RATE of
3280 ** packets.
3281 */
3282 static void
3283 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3284 {
3285         struct adapter                  *adapter = txr->adapter;
3286         struct ix_queue                 *que;
3287         struct ip                       *ip;
3288         struct tcphdr                   *th;
3289         struct udphdr                   *uh;
3290         struct ether_vlan_header        *eh;
3291         union ixgbe_atr_hash_dword      input = {.dword = 0}; 
3292         union ixgbe_atr_hash_dword      common = {.dword = 0}; 
3293         int                             ehdrlen, ip_hlen;
3294         u16                             etype;
3295
3296         eh = mtod(mp, struct ether_vlan_header *);
3297         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3298                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3299                 etype = eh->evl_proto;
3300         } else {
3301                 ehdrlen = ETHER_HDR_LEN;
3302                 etype = eh->evl_encap_proto;
3303         }
3304
3305         /* Only handling IPv4 */
3306         if (etype != htons(ETHERTYPE_IP))
3307                 return;
3308
3309         ip = (struct ip *)(mp->m_data + ehdrlen);
3310         ip_hlen = ip->ip_hl << 2;
3311
3312         /* check if we're UDP or TCP */
3313         switch (ip->ip_p) {
3314         case IPPROTO_TCP:
3315                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3316                 /* src and dst are inverted */
3317                 common.port.dst ^= th->th_sport;
3318                 common.port.src ^= th->th_dport;
3319                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3320                 break;
3321         case IPPROTO_UDP:
3322                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3323                 /* src and dst are inverted */
3324                 common.port.dst ^= uh->uh_sport;
3325                 common.port.src ^= uh->uh_dport;
3326                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3327                 break;
3328         default:
3329                 return;
3330         }
3331
3332         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3333         if (mp->m_pkthdr.ether_vtag)
3334                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3335         else
3336                 common.flex_bytes ^= etype;
3337         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3338
3339         que = &adapter->queues[txr->me];
3340         /*
3341         ** This assumes the Rx queue and Tx
3342         ** queue are bound to the same CPU
3343         */
3344         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3345             input, common, que->msix);
3346 }
3347 #endif /* IXGBE_FDIR */
3348
3349 /**********************************************************************
3350  *
3351  *  Examine each tx_buffer in the used queue. If the hardware is done
3352  *  processing the packet then free associated resources. The
3353  *  tx_buffer is put back on the free queue.
3354  *
3355  **********************************************************************/
3356 static void
3357 ixgbe_txeof(struct tx_ring *txr)
3358 {
3359         struct adapter  *adapter = txr->adapter;
3360         struct ifnet    *ifp = adapter->ifp;
3361         u32     first, last, done, processed;
3362         struct ixgbe_tx_buf *tx_buffer;
3363         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3364
3365         IXGBE_TX_LOCK_ASSERT(txr);
3366
3367 #ifdef DEV_NETMAP
3368         if (ifp->if_capenable & IFCAP_NETMAP) {
3369                 struct netmap_adapter *na = NA(ifp);
3370                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3371
3372                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3373
3374                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3375                     BUS_DMASYNC_POSTREAD);
3376                 /*
3377                  * In netmap mode, all the work is done in the context
3378                  * of the client thread. Interrupt handlers only wake up
3379                  * clients, which may be sleeping on individual rings
3380                  * or on a global resource for all rings.
3381                  * To implement tx interrupt mitigation, we wake up the client
3382                  * thread roughly every half ring, even if the NIC interrupts
3383                  * more frequently. This is implemented as follows:
3384                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3385                  *   the slot that should wake up the thread (nkr_num_slots
3386                  *   means the user thread should not be woken up);
3387                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3388                  *   or the slot has the DD bit set.
3389                  *
3390                  * When the driver has separate locks, we need to
3391                  * release and re-acquire txlock to avoid deadlocks.
3392                  * XXX see if we can find a better way.
3393                  */
3394                 if (!netmap_mitigate ||
3395                     (kring->nr_kflags < kring->nkr_num_slots &&
3396                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3397                         kring->nr_kflags = kring->nkr_num_slots;
3398                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3399                         IXGBE_TX_UNLOCK(txr);
3400                         IXGBE_CORE_LOCK(adapter);
3401                         selwakeuppri(&na->tx_si, PI_NET);
3402                         IXGBE_CORE_UNLOCK(adapter);
3403                         IXGBE_TX_LOCK(txr);
3404                 }
3405                 return FALSE;
3406         }
3407 #endif /* DEV_NETMAP */
3408
3409         if (txr->tx_avail == adapter->num_tx_desc) {
3410                 txr->queue_status = IXGBE_QUEUE_IDLE;
3411                 return;
3412         }
3413
3414         processed = 0;
3415         first = txr->next_to_clean;
3416         tx_buffer = &txr->tx_buffers[first];
3417         /* For cleanup we just use legacy struct */
3418         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3419         last = tx_buffer->eop_index;
3420         if (last == -1)
3421                 return;
3422         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3423
3424         /*
3425         ** Get the index of the first descriptor
3426         ** BEYOND the EOP and call that 'done'.
3427         ** I do this so the comparison in the
3428         ** inner while loop below can be simple
3429         */
3430         if (++last == adapter->num_tx_desc) last = 0;
3431         done = last;
3432
3433         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3434             BUS_DMASYNC_POSTREAD);
3435         /*
3436         ** Only the EOP descriptor of a packet now has the DD
3437         ** bit set, this is what we look for...
3438         */
3439         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3440                 /* We clean the range of the packet */
3441                 while (first != done) {
3442                         tx_desc->upper.data = 0;
3443                         tx_desc->lower.data = 0;
3444                         tx_desc->buffer_addr = 0;
3445                         ++txr->tx_avail;
3446                         ++processed;
3447
3448                         if (tx_buffer->m_head) {
3449                                 txr->bytes +=
3450                                     tx_buffer->m_head->m_pkthdr.len;
3451                                 bus_dmamap_sync(txr->txtag,
3452                                     tx_buffer->map,
3453                                     BUS_DMASYNC_POSTWRITE);
3454                                 bus_dmamap_unload(txr->txtag,
3455                                     tx_buffer->map);
3456                                 m_freem(tx_buffer->m_head);
3457                                 tx_buffer->m_head = NULL;
3458                                 tx_buffer->map = NULL;
3459                         }
3460                         tx_buffer->eop_index = -1;
3461                         txr->watchdog_time = ticks;
3462
3463                         if (++first == adapter->num_tx_desc)
3464                                 first = 0;
3465
3466                         tx_buffer = &txr->tx_buffers[first];
3467                         tx_desc =
3468                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3469                 }
3470                 ++txr->packets;
3471                 ++ifp->if_opackets;
3472                 /* See if there is more work now */
3473                 last = tx_buffer->eop_index;
3474                 if (last != -1) {
3475                         eop_desc =
3476                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3477                         /* Get next done point */
3478                         if (++last == adapter->num_tx_desc) last = 0;
3479                         done = last;
3480                 } else
3481                         break;
3482         }
3483         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3484             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3485
3486         txr->next_to_clean = first;
3487
3488         /*
3489         ** Watchdog calculation, we know there's
3490         ** work outstanding or the first return
3491         ** would have been taken, so none processed
3492         ** for too long indicates a hang.
3493         */
3494         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3495                 txr->queue_status = IXGBE_QUEUE_HUNG;
3496
3497         /* With a minimum free clear the depleted state bit.  */
3498         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3499                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3500
3501         if (txr->tx_avail == adapter->num_tx_desc) {
3502                 txr->queue_status = IXGBE_QUEUE_IDLE;
3503         }
3504 }
3505
3506 /*********************************************************************
3507  *
3508  *  Refresh mbuf buffers for RX descriptor rings
3509  *   - now keeps its own state so discards due to resource
3510  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3511  *     it just returns, keeping its placeholder, thus it can simply
3512  *     be recalled to try again.
3513  *
3514  **********************************************************************/
3515 static void
3516 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3517 {
3518         struct adapter          *adapter = rxr->adapter;
3519         bus_dma_segment_t       hseg[1];
3520         bus_dma_segment_t       pseg[1];
3521         struct ixgbe_rx_buf     *rxbuf;
3522         struct mbuf             *mh, *mp;
3523         int                     i, j, nsegs, error;
3524         bool                    refreshed = FALSE;
3525
3526         i = j = rxr->next_to_refresh;
3527         /* Control the loop with one beyond */
3528         if (++j == adapter->num_rx_desc)
3529                 j = 0;
3530
3531         while (j != limit) {
3532                 rxbuf = &rxr->rx_buffers[i];
3533                 if (rxr->hdr_split == FALSE)
3534                         goto no_split;
3535
3536                 if (rxbuf->m_head == NULL) {
3537                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3538                         if (mh == NULL)
3539                                 goto update;
3540                 } else
3541                         mh = rxbuf->m_head;
3542
3543                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3544                 mh->m_len = MHLEN;
3545                 mh->m_flags |= M_PKTHDR;
3546                 /* Get the memory mapping */
3547                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3548                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3549                 if (error != 0) {
3550                         kprintf("Refresh mbufs: hdr dmamap load"
3551                             " failure - %d\n", error);
3552                         m_free(mh);
3553                         rxbuf->m_head = NULL;
3554                         goto update;
3555                 }
3556                 rxbuf->m_head = mh;
3557                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3558                     BUS_DMASYNC_PREREAD);
3559                 rxr->rx_base[i].read.hdr_addr =
3560                     htole64(hseg[0].ds_addr);
3561
3562 no_split:
3563                 if (rxbuf->m_pack == NULL) {
3564                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3565                             M_PKTHDR, adapter->rx_mbuf_sz);
3566                         if (mp == NULL)
3567                                 goto update;
3568                 } else
3569                         mp = rxbuf->m_pack;
3570
3571                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3572                 /* Get the memory mapping */
3573                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3574                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3575                 if (error != 0) {
3576                         kprintf("Refresh mbufs: payload dmamap load"
3577                             " failure - %d\n", error);
3578                         m_free(mp);
3579                         rxbuf->m_pack = NULL;
3580                         goto update;
3581                 }
3582                 rxbuf->m_pack = mp;
3583                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3584                     BUS_DMASYNC_PREREAD);
3585                 rxr->rx_base[i].read.pkt_addr =
3586                     htole64(pseg[0].ds_addr);
3587
3588                 refreshed = TRUE;
3589                 /* Next is precalculated */
3590                 i = j;
3591                 rxr->next_to_refresh = i;
3592                 if (++j == adapter->num_rx_desc)
3593                         j = 0;
3594         }
3595 update:
3596         if (refreshed) /* Update hardware tail index */
3597                 IXGBE_WRITE_REG(&adapter->hw,
3598                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3599         return;
3600 }
3601
3602 /*********************************************************************
3603  *
3604  *  Allocate memory for rx_buffer structures. Since we use one
3605  *  rx_buffer per received packet, the maximum number of rx_buffer's
3606  *  that we'll need is equal to the number of receive descriptors
3607  *  that we've allocated.
3608  *
3609  **********************************************************************/
3610 static int
3611 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3612 {
3613         struct  adapter         *adapter = rxr->adapter;
3614         device_t                dev = adapter->dev;
3615         struct ixgbe_rx_buf     *rxbuf;
3616         int                     i, bsize, error;
3617
3618         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3619         if (!(rxr->rx_buffers =
3620             (struct ixgbe_rx_buf *) kmalloc(bsize,
3621             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3622                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3623                 error = ENOMEM;
3624                 goto fail;
3625         }
3626
3627         if ((error = bus_dma_tag_create(NULL,   /* parent */
3628                                    1, 0,        /* alignment, bounds */
3629                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3630                                    BUS_SPACE_MAXADDR,   /* highaddr */
3631                                    NULL, NULL,          /* filter, filterarg */
3632                                    MSIZE,               /* maxsize */
3633                                    1,                   /* nsegments */
3634                                    MSIZE,               /* maxsegsize */
3635                                    0,                   /* flags */
3636                                    &rxr->htag))) {
3637                 device_printf(dev, "Unable to create RX DMA tag\n");
3638                 goto fail;
3639         }
3640
3641         if ((error = bus_dma_tag_create(NULL,   /* parent */
3642                                    1, 0,        /* alignment, bounds */
3643                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3644                                    BUS_SPACE_MAXADDR,   /* highaddr */
3645                                    NULL, NULL,          /* filter, filterarg */
3646                                    MJUM16BYTES,         /* maxsize */
3647                                    1,                   /* nsegments */
3648                                    MJUM16BYTES,         /* maxsegsize */
3649                                    0,                   /* flags */
3650                                    &rxr->ptag))) {
3651                 device_printf(dev, "Unable to create RX DMA tag\n");
3652                 goto fail;
3653         }
3654
3655         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3656                 rxbuf = &rxr->rx_buffers[i];
3657                 error = bus_dmamap_create(rxr->htag,
3658                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3659                 if (error) {
3660                         device_printf(dev, "Unable to create RX head map\n");
3661                         goto fail;
3662                 }
3663                 error = bus_dmamap_create(rxr->ptag,
3664                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3665                 if (error) {
3666                         device_printf(dev, "Unable to create RX pkt map\n");
3667                         goto fail;
3668                 }
3669         }
3670
3671         return (0);
3672
3673 fail:
3674         /* Frees all, but can handle partial completion */
3675         ixgbe_free_receive_structures(adapter);
3676         return (error);
3677 }
3678
3679 /*
3680 ** Used to detect a descriptor that has
3681 ** been merged by Hardware RSC.
3682 */
3683 static inline u32
3684 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3685 {
3686         return (le32toh(rx->wb.lower.lo_dword.data) &
3687             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3688 }
3689
3690 /*********************************************************************
3691  *
3692  *  Initialize Hardware RSC (LRO) feature on 82599
3693  *  for an RX ring, this is toggled by the LRO capability
3694  *  even though it is transparent to the stack.
3695  *
3696  **********************************************************************/
3697 #if 0   /* NET_LRO */
3698 static void
3699 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3700 {
3701         struct  adapter         *adapter = rxr->adapter;
3702         struct  ixgbe_hw        *hw = &adapter->hw;
3703         u32                     rscctrl, rdrxctl;
3704
3705         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3706         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3707 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3708         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3709 #endif /* DEV_NETMAP */
3710         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3711         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3712         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3713
3714         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3715         rscctrl |= IXGBE_RSCCTL_RSCEN;
3716         /*
3717         ** Limit the total number of descriptors that
3718         ** can be combined, so it does not exceed 64K
3719         */
3720         if (adapter->rx_mbuf_sz == MCLBYTES)
3721                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3722         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3723                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3724         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3725                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3726         else  /* Using 16K cluster */
3727                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3728
3729         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3730
3731         /* Enable TCP header recognition */
3732         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3733             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3734             IXGBE_PSRTYPE_TCPHDR));
3735
3736         /* Disable RSC for ACK packets */
3737         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3738             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3739
3740         rxr->hw_rsc = TRUE;
3741 }
3742 #endif
3743
3744 static void     
3745 ixgbe_free_receive_ring(struct rx_ring *rxr)
3746
3747         struct  adapter         *adapter;
3748         struct ixgbe_rx_buf       *rxbuf;
3749         int i;
3750
3751         adapter = rxr->adapter;
3752         for (i = 0; i < adapter->num_rx_desc; i++) {
3753                 rxbuf = &rxr->rx_buffers[i];
3754                 if (rxbuf->m_head != NULL) {
3755                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3756                             BUS_DMASYNC_POSTREAD);
3757                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3758                         rxbuf->m_head->m_flags |= M_PKTHDR;
3759                         m_freem(rxbuf->m_head);
3760                 }
3761                 if (rxbuf->m_pack != NULL) {
3762                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3763                             BUS_DMASYNC_POSTREAD);
3764                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3765                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3766                         m_freem(rxbuf->m_pack);
3767                 }
3768                 rxbuf->m_head = NULL;
3769                 rxbuf->m_pack = NULL;
3770         }
3771 }
3772
3773
3774 /*********************************************************************
3775  *
3776  *  Initialize a receive ring and its buffers.
3777  *
3778  **********************************************************************/
3779 static int
3780 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3781 {
3782         struct  adapter         *adapter;
3783         struct ifnet            *ifp;
3784         device_t                dev;
3785         struct ixgbe_rx_buf     *rxbuf;
3786         bus_dma_segment_t       pseg[1], hseg[1];
3787 #if 0   /* NET_LRO */
3788         struct lro_ctrl         *lro = &rxr->lro;
3789 #endif
3790         int                     rsize, nsegs, error = 0;
3791 #ifdef DEV_NETMAP
3792         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3793         struct netmap_slot *slot;
3794 #endif /* DEV_NETMAP */
3795
3796         adapter = rxr->adapter;
3797         ifp = adapter->ifp;
3798         dev = adapter->dev;
3799
3800         /* Clear the ring contents */
3801         IXGBE_RX_LOCK(rxr);
3802 #ifdef DEV_NETMAP
3803         /* same as in ixgbe_setup_transmit_ring() */
3804         slot = netmap_reset(na, NR_RX, rxr->me, 0);
3805 #endif /* DEV_NETMAP */
3806         rsize = roundup2(adapter->num_rx_desc *
3807             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
3808         bzero((void *)rxr->rx_base, rsize);
3809
3810         /* Free current RX buffer structs and their mbufs */
3811         ixgbe_free_receive_ring(rxr);
3812
3813         /* Configure header split? */
3814         if (ixgbe_header_split)
3815                 rxr->hdr_split = TRUE;
3816
3817         /* Now replenish the mbufs */
3818         for (int j = 0; j != adapter->num_rx_desc; ++j) {
3819                 struct mbuf     *mh, *mp;
3820
3821                 rxbuf = &rxr->rx_buffers[j];
3822 #ifdef DEV_NETMAP
3823                 /*
3824                  * In netmap mode, fill the map and set the buffer
3825                  * address in the NIC ring, considering the offset
3826                  * between the netmap and NIC rings (see comment in
3827                  * ixgbe_setup_transmit_ring() ). No need to allocate
3828                  * an mbuf, so end the block with a continue;
3829                  */
3830                 if (slot) {
3831                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
3832                         uint64_t paddr;
3833                         void *addr;
3834
3835                         addr = PNMB(slot + sj, &paddr);
3836                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
3837                         /* Update descriptor */
3838                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
3839                         continue;
3840                 }
3841 #endif /* DEV_NETMAP */
3842                 /*
3843                 ** Don't allocate mbufs if not
3844                 ** doing header split, its wasteful
3845                 */ 
3846                 if (rxr->hdr_split == FALSE)
3847                         goto skip_head;
3848
3849                 /* First the header */
3850                 rxbuf->m_head = m_gethdr(MB_DONTWAIT, MT_DATA);
3851                 if (rxbuf->m_head == NULL) {
3852                         error = ENOBUFS;
3853                         goto fail;
3854                 }
3855                 m_adj(rxbuf->m_head, ETHER_ALIGN);
3856                 mh = rxbuf->m_head;
3857                 mh->m_len = mh->m_pkthdr.len = MHLEN;
3858                 mh->m_flags |= M_PKTHDR;
3859                 /* Get the memory mapping */
3860                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3861                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
3862                     &nsegs, BUS_DMA_NOWAIT);
3863
3864                 if (error != 0) /* Nothing elegant to do here */
3865                         goto fail;
3866                 bus_dmamap_sync(rxr->htag,
3867                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
3868                 /* Update descriptor */
3869                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
3870
3871 skip_head:
3872                 /* Now the payload cluster */
3873                 rxbuf->m_pack = m_getjcl(MB_DONTWAIT, MT_DATA,
3874                     M_PKTHDR, adapter->rx_mbuf_sz);
3875                 if (rxbuf->m_pack == NULL) {
3876                         error = ENOBUFS;
3877                         goto fail;
3878                 }
3879                 mp = rxbuf->m_pack;
3880                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3881                 /* Get the memory mapping */
3882                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3883                     rxbuf->pmap, mp, hseg, 1,
3884                     &nsegs, BUS_DMA_NOWAIT);
3885                 if (error != 0)
3886                         goto fail;
3887                 bus_dmamap_sync(rxr->ptag,
3888                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
3889                 /* Update descriptor */
3890                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
3891         }
3892
3893
3894         /* Setup our descriptor indices */
3895         rxr->next_to_check = 0;
3896         rxr->next_to_refresh = 0;
3897         rxr->lro_enabled = FALSE;
3898         rxr->rx_split_packets = 0;
3899         rxr->rx_bytes = 0;
3900         rxr->discard = FALSE;
3901         rxr->vtag_strip = FALSE;
3902
3903         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3904             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3905
3906         /*
3907         ** Now set up the LRO interface:
3908         ** 82598 uses software LRO, the
3909         ** 82599 and X540 use a hardware assist.
3910         */
3911 #if 0 /* NET_LRO */
3912         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
3913             (ifp->if_capenable & IFCAP_RXCSUM) &&
3914             (ifp->if_capenable & IFCAP_LRO))
3915                 ixgbe_setup_hw_rsc(rxr);
3916         else if (ifp->if_capenable & IFCAP_LRO) {
3917                 int err = tcp_lro_init(lro);
3918                 if (err) {
3919                         device_printf(dev, "LRO Initialization failed!\n");
3920                         goto fail;
3921                 }
3922                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
3923                 rxr->lro_enabled = TRUE;
3924                 lro->ifp = adapter->ifp;
3925         }
3926 #endif
3927
3928         IXGBE_RX_UNLOCK(rxr);
3929         return (0);
3930
3931 fail:
3932         ixgbe_free_receive_ring(rxr);
3933         IXGBE_RX_UNLOCK(rxr);
3934         return (error);
3935 }
3936
3937 /*********************************************************************
3938  *
3939  *  Initialize all receive rings.
3940  *
3941  **********************************************************************/
3942 static int
3943 ixgbe_setup_receive_structures(struct adapter *adapter)
3944 {
3945         struct rx_ring *rxr = adapter->rx_rings;
3946         int j;
3947
3948         for (j = 0; j < adapter->num_queues; j++, rxr++)
3949                 if (ixgbe_setup_receive_ring(rxr))
3950                         goto fail;
3951
3952         return (0);
3953 fail:
3954         /*
3955          * Free RX buffers allocated so far, we will only handle
3956          * the rings that completed, the failing case will have
3957          * cleaned up for itself. 'j' failed, so its the terminus.
3958          */
3959         for (int i = 0; i < j; ++i) {
3960                 rxr = &adapter->rx_rings[i];
3961                 ixgbe_free_receive_ring(rxr);
3962         }
3963
3964         return (ENOBUFS);
3965 }
3966
3967 /*********************************************************************
3968  *
3969  *  Setup receive registers and features.
3970  *
3971  **********************************************************************/
3972 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
3973
3974 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
3975         
3976 static void
3977 ixgbe_initialize_receive_units(struct adapter *adapter)
3978 {
3979         struct  rx_ring *rxr = adapter->rx_rings;
3980         struct ixgbe_hw *hw = &adapter->hw;
3981         struct ifnet   *ifp = adapter->ifp;
3982         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
3983         u32             reta, mrqc = 0, hlreg, random[10];
3984
3985
3986         /*
3987          * Make sure receives are disabled while
3988          * setting up the descriptor ring
3989          */
3990         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3991         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
3992             rxctrl & ~IXGBE_RXCTRL_RXEN);
3993
3994         /* Enable broadcasts */
3995         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3996         fctrl |= IXGBE_FCTRL_BAM;
3997         fctrl |= IXGBE_FCTRL_DPF;
3998         fctrl |= IXGBE_FCTRL_PMCF;
3999         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4000
4001         /* Set for Jumbo Frames? */
4002         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4003         if (ifp->if_mtu > ETHERMTU)
4004                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4005         else
4006                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4007 #ifdef DEV_NETMAP
4008         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */