Merge branch 'vendor/XZ'
[dragonfly.git] / sys / dev / netif / ixgbe / ixgbe.c
1 /******************************************************************************
2
3   Copyright (c) 2001-2012, Intel Corporation 
4   All rights reserved.
5   
6   Redistribution and use in source and binary forms, with or without 
7   modification, are permitted provided that the following conditions are met:
8   
9    1. Redistributions of source code must retain the above copyright notice, 
10       this list of conditions and the following disclaimer.
11   
12    2. Redistributions in binary form must reproduce the above copyright 
13       notice, this list of conditions and the following disclaimer in the 
14       documentation and/or other materials provided with the distribution.
15   
16    3. Neither the name of the Intel Corporation nor the names of its 
17       contributors may be used to endorse or promote products derived from 
18       this software without specific prior written permission.
19   
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.69 2012/06/07 22:57:26 emax Exp $*/
34
35 #include "opt_inet.h"
36 #include "opt_inet6.h"
37
38 #include "ixgbe.h"
39
40 /*********************************************************************
41  *  Set this to one to display debug statistics
42  *********************************************************************/
43 int             ixgbe_display_debug_stats = 0;
44
45 /*********************************************************************
46  *  Driver version
47  *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.5";
49
50 /*********************************************************************
51  *  PCI Device ID Table
52  *
53  *  Used by probe to select devices to load on
54  *  Last field stores an index into ixgbe_strings
55  *  Last entry must be all 0s
56  *
57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58  *********************************************************************/
59
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
61 {
62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
84         /* required last entry */
85         {0, 0, 0, 0, 0}
86 };
87
88 /*********************************************************************
89  *  Table of branding strings
90  *********************************************************************/
91
92 static char    *ixgbe_strings[] = {
93         "Intel(R) PRO/10GbE PCI-Express Network Driver"
94 };
95
96 /*********************************************************************
97  *  Function prototypes
98  *********************************************************************/
99 static int      ixgbe_probe(device_t);
100 static int      ixgbe_attach(device_t);
101 static int      ixgbe_detach(device_t);
102 static int      ixgbe_shutdown(device_t);
103 static void     ixgbe_start(struct ifnet *);
104 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
105 #if 0 /* __FreeBSD_version >= 800000 */
106 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
107 static int      ixgbe_mq_start_locked(struct ifnet *,
108                     struct tx_ring *, struct mbuf *);
109 static void     ixgbe_qflush(struct ifnet *);
110 #endif
111 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
112 static void     ixgbe_init(void *);
113 static void     ixgbe_init_locked(struct adapter *);
114 static void     ixgbe_stop(void *);
115 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
116 static int      ixgbe_media_change(struct ifnet *);
117 static void     ixgbe_identify_hardware(struct adapter *);
118 static int      ixgbe_allocate_pci_resources(struct adapter *);
119 static int      ixgbe_allocate_msix(struct adapter *);
120 static int      ixgbe_allocate_legacy(struct adapter *);
121 static int      ixgbe_allocate_queues(struct adapter *);
122 #if 0   /* HAVE_MSIX */
123 static int      ixgbe_setup_msix(struct adapter *);
124 #endif
125 static void     ixgbe_free_pci_resources(struct adapter *);
126 static void     ixgbe_local_timer(void *);
127 static int      ixgbe_setup_interface(device_t, struct adapter *);
128 static void     ixgbe_config_link(struct adapter *);
129
130 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
131 static int      ixgbe_setup_transmit_structures(struct adapter *);
132 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
133 static void     ixgbe_initialize_transmit_units(struct adapter *);
134 static void     ixgbe_free_transmit_structures(struct adapter *);
135 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
136
137 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
138 static int      ixgbe_setup_receive_structures(struct adapter *);
139 static int      ixgbe_setup_receive_ring(struct rx_ring *);
140 static void     ixgbe_initialize_receive_units(struct adapter *);
141 static void     ixgbe_free_receive_structures(struct adapter *);
142 static void     ixgbe_free_receive_buffers(struct rx_ring *);
143 #if 0   /* NET_LRO */
144 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
145 #endif
146
147 static void     ixgbe_enable_intr(struct adapter *);
148 static void     ixgbe_disable_intr(struct adapter *);
149 static void     ixgbe_update_stats_counters(struct adapter *);
150 static bool     ixgbe_txeof(struct tx_ring *);
151 static bool     ixgbe_rxeof(struct ix_queue *, int);
152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
153 static void     ixgbe_set_promisc(struct adapter *);
154 static void     ixgbe_set_multi(struct adapter *);
155 static void     ixgbe_update_link_status(struct adapter *);
156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
162                     struct ixgbe_dma_alloc *, int);
163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
165                     const char *, int *, int);
166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
167 #if 0   /* NET_TSO */
168 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
169 #endif
170 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
171 static void     ixgbe_configure_ivars(struct adapter *);
172 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
173
174 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
175 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
176 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
177
178 static void     ixgbe_add_hw_stats(struct adapter *adapter);
179
180 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
181 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
182                     struct mbuf *, u32);
183
184 /* Support for pluggable optic modules */
185 static bool     ixgbe_sfp_probe(struct adapter *);
186 static void     ixgbe_setup_optics(struct adapter *);
187
188 /* Legacy (single vector interrupt handler */
189 static void     ixgbe_legacy_irq(void *);
190
191 /* The MSI/X Interrupt handlers */
192 static void     ixgbe_msix_que(void *);
193 static void     ixgbe_msix_link(void *);
194
195 /* Deferred interrupt tasklets */
196 static void     ixgbe_handle_que(void *, int);
197 static void     ixgbe_handle_link(void *, int);
198 static void     ixgbe_handle_msf(void *, int);
199 static void     ixgbe_handle_mod(void *, int);
200
201 #ifdef IXGBE_FDIR
202 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
203 static void     ixgbe_reinit_fdir(void *, int);
204 #endif
205
206 /*********************************************************************
207  *  FreeBSD Device Interface Entry Points
208  *********************************************************************/
209
210 static device_method_t ixgbe_methods[] = {
211         /* Device interface */
212         DEVMETHOD(device_probe, ixgbe_probe),
213         DEVMETHOD(device_attach, ixgbe_attach),
214         DEVMETHOD(device_detach, ixgbe_detach),
215         DEVMETHOD(device_shutdown, ixgbe_shutdown),
216         {0, 0}
217 };
218
219 static driver_t ixgbe_driver = {
220         "ix", ixgbe_methods, sizeof(struct adapter),
221 };
222
223 devclass_t ixgbe_devclass;
224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
225
226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
228
229 /*
230 ** TUNEABLE PARAMETERS:
231 */
232
233 /*
234 ** AIM: Adaptive Interrupt Moderation
235 ** which means that the interrupt rate
236 ** is varied over time based on the
237 ** traffic for that interrupt vector
238 */
239 static int ixgbe_enable_aim = TRUE;
240 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
241
242 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
243 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
244
245 /* How many packets rxeof tries to clean at a time */
246 static int ixgbe_rx_process_limit = 128;
247 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
248
249 /* Flow control setting, default to full */
250 static int ixgbe_flow_control = ixgbe_fc_full;
251 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
252
253 /*
254 ** Smart speed setting, default to on
255 ** this only works as a compile option
256 ** right now as its during attach, set
257 ** this to 'ixgbe_smart_speed_off' to
258 ** disable.
259 */
260 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
261
262 /*
263  * MSIX should be the default for best performance,
264  * but this allows it to be forced off for testing.
265  */
266 static int ixgbe_enable_msix = 1;
267 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
268
269 /*
270  * Header split: this causes the hardware to DMA
271  * the header into a separate mbuf from the payload,
272  * it can be a performance win in some workloads, but
273  * in others it actually hurts, its off by default. 
274  */
275 static int ixgbe_header_split = FALSE;
276 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
277
278 /*
279  * Number of Queues, can be set to 0,
280  * it then autoconfigures based on the
281  * number of cpus with a max of 8. This
282  * can be overriden manually here.
283  */
284 static int ixgbe_num_queues = 0;
285 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
286
287 /*
288 ** Number of TX descriptors per ring,
289 ** setting higher than RX as this seems
290 ** the better performing choice.
291 */
292 static int ixgbe_txd = PERFORM_TXD;
293 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
294
295 /* Number of RX descriptors per ring */
296 static int ixgbe_rxd = PERFORM_RXD;
297 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
298
299 /* Keep running tab on them for sanity check */
300 static int ixgbe_total_ports;
301
302 #ifdef IXGBE_FDIR
303 /*
304 ** For Flow Director: this is the
305 ** number of TX packets we sample
306 ** for the filter pool, this means
307 ** every 20th packet will be probed.
308 **
309 ** This feature can be disabled by 
310 ** setting this to 0.
311 */
312 static int atr_sample_rate = 20;
313 /* 
314 ** Flow Director actually 'steals'
315 ** part of the packet buffer as its
316 ** filter pool, this variable controls
317 ** how much it uses:
318 **  0 = 64K, 1 = 128K, 2 = 256K
319 */
320 static int fdir_pballoc = 1;
321 #endif
322
323 #ifdef DEV_NETMAP
324 /*
325  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
326  * be a reference on how to implement netmap support in a driver.
327  * Additional comments are in ixgbe_netmap.h .
328  *
329  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
330  * that extend the standard driver.
331  */
332 #include <dev/netmap/ixgbe_netmap.h>
333 #endif /* DEV_NETMAP */
334
335 /*********************************************************************
336  *  Device identification routine
337  *
338  *  ixgbe_probe determines if the driver should be loaded on
339  *  adapter based on PCI vendor/device id of the adapter.
340  *
341  *  return BUS_PROBE_DEFAULT on success, positive on failure
342  *********************************************************************/
343
344 static int
345 ixgbe_probe(device_t dev)
346 {
347         ixgbe_vendor_info_t *ent;
348
349         u16     pci_vendor_id = 0;
350         u16     pci_device_id = 0;
351         u16     pci_subvendor_id = 0;
352         u16     pci_subdevice_id = 0;
353         char    adapter_name[256];
354
355         INIT_DEBUGOUT("ixgbe_probe: begin");
356
357         pci_vendor_id = pci_get_vendor(dev);
358         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
359                 return (ENXIO);
360
361         pci_device_id = pci_get_device(dev);
362         pci_subvendor_id = pci_get_subvendor(dev);
363         pci_subdevice_id = pci_get_subdevice(dev);
364
365         ent = ixgbe_vendor_info_array;
366         while (ent->vendor_id != 0) {
367                 if ((pci_vendor_id == ent->vendor_id) &&
368                     (pci_device_id == ent->device_id) &&
369
370                     ((pci_subvendor_id == ent->subvendor_id) ||
371                      (ent->subvendor_id == 0)) &&
372
373                     ((pci_subdevice_id == ent->subdevice_id) ||
374                      (ent->subdevice_id == 0))) {
375                         ksprintf(adapter_name, "%s, Version - %s",
376                                 ixgbe_strings[ent->index],
377                                 ixgbe_driver_version);
378                         device_set_desc_copy(dev, adapter_name);
379                         ++ixgbe_total_ports;
380                         return (BUS_PROBE_DEFAULT);
381                 }
382                 ent++;
383         }
384         return (ENXIO);
385 }
386
387 /*********************************************************************
388  *  Device initialization routine
389  *
390  *  The attach entry point is called when the driver is being loaded.
391  *  This routine identifies the type of hardware, allocates all resources
392  *  and initializes the hardware.
393  *
394  *  return 0 on success, positive on failure
395  *********************************************************************/
396
397 static int
398 ixgbe_attach(device_t dev)
399 {
400         struct adapter *adapter;
401         struct ixgbe_hw *hw;
402         int             error = 0;
403         u16             csum;
404         u32             ctrl_ext;
405
406         INIT_DEBUGOUT("ixgbe_attach: begin");
407
408         if (resource_disabled("ixgbe", device_get_unit(dev))) {
409                 device_printf(dev, "Disabled by device hint\n");
410                 return (ENXIO);
411         }
412
413         /* Allocate, clear, and link in our adapter structure */
414         adapter = device_get_softc(dev);
415         adapter->dev = adapter->osdep.dev = dev;
416         hw = &adapter->hw;
417
418         /* Core Lock Init*/
419         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
420         spin_init(&adapter->mcast_spin);
421
422         /* SYSCTL APIs */
423
424         sysctl_ctx_init(&adapter->sysctl_ctx);
425         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
426             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
427             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
428         if (adapter->sysctl_tree == NULL) {
429                 device_printf(adapter->dev, "can't add sysctl node\n");
430                 return (EINVAL);
431         }
432         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
433                         SYSCTL_CHILDREN(adapter->sysctl_tree),
434                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
435                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
436
437         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
438                         SYSCTL_CHILDREN(adapter->sysctl_tree),
439                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
440                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
441
442         /*
443         ** Allow a kind of speed control by forcing the autoneg
444         ** advertised speed list to only a certain value, this
445         ** supports 1G on 82599 devices, and 100Mb on x540.
446         */
447         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
448                         SYSCTL_CHILDREN(adapter->sysctl_tree),
449                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
450                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
451
452         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
453                         SYSCTL_CHILDREN(adapter->sysctl_tree),
454                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
455                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
456
457         /* Set up the timer callout */
458         /* XXX: shouldn't this be a spin lock ? */
459         lockinit(&adapter->core_lock, "ixgbe core lock", 0, LK_CANRECURSE);
460         callout_init(&adapter->timer);
461
462         /* Determine hardware revision */
463         ixgbe_identify_hardware(adapter);
464
465         /* Do base PCI setup - map BAR0 */
466         if (ixgbe_allocate_pci_resources(adapter)) {
467                 device_printf(dev, "Allocation of PCI resources failed\n");
468                 error = ENXIO;
469                 goto err_out;
470         }
471
472         /* Do descriptor calc and sanity checks */
473         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
474             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
475                 device_printf(dev, "TXD config issue, using default!\n");
476                 adapter->num_tx_desc = DEFAULT_TXD;
477         } else
478                 adapter->num_tx_desc = ixgbe_txd;
479
480         /*
481         ** With many RX rings it is easy to exceed the
482         ** system mbuf allocation. Tuning nmbclusters
483         ** can alleviate this.
484         */
485         if (nmbclusters > 0 ) {
486                 int s;
487                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
488                 if (s > nmbclusters) {
489                         device_printf(dev, "RX Descriptors exceed "
490                             "system mbuf max, using default instead!\n");
491                         ixgbe_rxd = DEFAULT_RXD;
492                 }
493         }
494
495         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
496             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
497                 device_printf(dev, "RXD config issue, using default!\n");
498                 adapter->num_rx_desc = DEFAULT_RXD;
499         } else
500                 adapter->num_rx_desc = ixgbe_rxd;
501
502         /* Allocate our TX/RX Queues */
503         if (ixgbe_allocate_queues(adapter)) {
504                 error = ENOMEM;
505                 goto err_out;
506         }
507
508         /* Allocate multicast array memory. */
509         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
510             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
511         if (adapter->mta == NULL) {
512                 device_printf(dev, "Can not allocate multicast setup array\n");
513                 error = ENOMEM;
514                 goto err_late;
515         }
516
517         /* Initialize the shared code */
518         error = ixgbe_init_shared_code(hw);
519         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
520                 /*
521                 ** No optics in this port, set up
522                 ** so the timer routine will probe 
523                 ** for later insertion.
524                 */
525                 adapter->sfp_probe = TRUE;
526                 error = 0;
527         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
528                 device_printf(dev,"Unsupported SFP+ module detected!\n");
529                 error = EIO;
530                 goto err_late;
531         } else if (error) {
532                 device_printf(dev,"Unable to initialize the shared code\n");
533                 error = EIO;
534                 goto err_late;
535         }
536
537         /* Make sure we have a good EEPROM before we read from it */
538         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
539                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
540                 error = EIO;
541                 goto err_late;
542         }
543
544         /* Get Hardware Flow Control setting */
545         hw->fc.requested_mode = ixgbe_fc_full;
546         adapter->fc = hw->fc.requested_mode;
547         hw->fc.pause_time = IXGBE_FC_PAUSE;
548         hw->fc.low_water = IXGBE_FC_LO;
549         hw->fc.high_water[0] = IXGBE_FC_HI;
550         hw->fc.send_xon = TRUE;
551
552         error = ixgbe_init_hw(hw);
553         if (error == IXGBE_ERR_EEPROM_VERSION) {
554                 device_printf(dev, "This device is a pre-production adapter/"
555                     "LOM.  Please be aware there may be issues associated "
556                     "with your hardware.\n If you are experiencing problems "
557                     "please contact your Intel or hardware representative "
558                     "who provided you with this hardware.\n");
559         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED)
560                 device_printf(dev,"Unsupported SFP+ Module\n");
561
562         if (error) {
563                 error = EIO;
564                 device_printf(dev,"Hardware Initialization Failure\n");
565                 goto err_late;
566         }
567
568         /* Detect and set physical type */
569         ixgbe_setup_optics(adapter);
570
571         if ((adapter->msix > 1) && (ixgbe_enable_msix))
572                 error = ixgbe_allocate_msix(adapter); 
573         else
574                 error = ixgbe_allocate_legacy(adapter); 
575         if (error) 
576                 goto err_late;
577
578         /* Setup OS specific network interface */
579         if (ixgbe_setup_interface(dev, adapter) != 0)
580                 goto err_late;
581
582         /* Sysctl for limiting the amount of work done in the taskqueue */
583         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
584             "max number of rx packets to process", &adapter->rx_process_limit,
585             ixgbe_rx_process_limit);
586
587         /* Initialize statistics */
588         ixgbe_update_stats_counters(adapter);
589
590         /* Register for VLAN events */
591         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
592             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
593         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
594             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
595
596         /* Print PCIE bus type/speed/width info */
597         ixgbe_get_bus_info(hw);
598         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
599             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
600             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
601             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
602             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
603             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
604             ("Unknown"));
605
606         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
607             (hw->bus.speed == ixgbe_bus_speed_2500)) {
608                 device_printf(dev, "PCI-Express bandwidth available"
609                     " for this card\n     is not sufficient for"
610                     " optimal performance.\n");
611                 device_printf(dev, "For optimal performance a x8 "
612                     "PCIE, or x4 PCIE 2 slot is required.\n");
613         }
614
615         /* let hardware know driver is loaded */
616         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
617         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
618         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
619
620         ixgbe_add_hw_stats(adapter);
621
622 #ifdef DEV_NETMAP
623         ixgbe_netmap_attach(adapter);
624 #endif /* DEV_NETMAP */
625         INIT_DEBUGOUT("ixgbe_attach: end");
626         return (0);
627 err_late:
628         ixgbe_free_transmit_structures(adapter);
629         ixgbe_free_receive_structures(adapter);
630 err_out:
631         if (adapter->ifp != NULL)
632                 if_free(adapter->ifp);
633         ixgbe_free_pci_resources(adapter);
634         kfree(adapter->mta, M_DEVBUF);
635         return (error);
636
637 }
638
639 /*********************************************************************
640  *  Device removal routine
641  *
642  *  The detach entry point is called when the driver is being removed.
643  *  This routine stops the adapter and deallocates all the resources
644  *  that were allocated for driver operation.
645  *
646  *  return 0 on success, positive on failure
647  *********************************************************************/
648
649 static int
650 ixgbe_detach(device_t dev)
651 {
652         struct adapter *adapter = device_get_softc(dev);
653         struct ix_queue *que = adapter->queues;
654         u32     ctrl_ext;
655
656         INIT_DEBUGOUT("ixgbe_detach: begin");
657
658 #ifdef NET_VLAN
659         /* Make sure VLANS are not using driver */
660         if (adapter->ifp->if_vlantrunk != NULL) {
661                 device_printf(dev,"Vlan in use, detach first\n");
662                 return (EBUSY);
663         }
664 #endif
665
666         IXGBE_CORE_LOCK(adapter);
667         ixgbe_stop(adapter);
668         IXGBE_CORE_UNLOCK(adapter);
669
670         for (int i = 0; i < adapter->num_queues; i++, que++) {
671                 if (que->tq) {
672                         taskqueue_drain(que->tq, &que->que_task);
673                         taskqueue_free(que->tq);
674                 }
675         }
676
677         /* Drain the Link queue */
678         if (adapter->tq) {
679                 taskqueue_drain(adapter->tq, &adapter->link_task);
680                 taskqueue_drain(adapter->tq, &adapter->mod_task);
681                 taskqueue_drain(adapter->tq, &adapter->msf_task);
682 #ifdef IXGBE_FDIR
683                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
684 #endif
685                 taskqueue_free(adapter->tq);
686         }
687
688         /* let hardware know driver is unloading */
689         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
690         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
691         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
692
693         /* Unregister VLAN events */
694 #ifdef NET_VLAN
695         if (adapter->vlan_attach != NULL)
696                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
697         if (adapter->vlan_detach != NULL)
698                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
699 #endif
700
701         ether_ifdetach(adapter->ifp);
702         callout_stop(&adapter->timer);
703         lockuninit(&adapter->core_lock);
704 #ifdef DEV_NETMAP
705         netmap_detach(adapter->ifp);
706 #endif /* DEV_NETMAP */
707         ixgbe_free_pci_resources(adapter);
708         bus_generic_detach(dev);
709         if_free(adapter->ifp);
710
711         ixgbe_free_transmit_structures(adapter);
712         ixgbe_free_receive_structures(adapter);
713         kfree(adapter->mta, M_DEVBUF);
714         sysctl_ctx_free(&adapter->sysctl_ctx);
715         
716         spin_uninit(&adapter->mcast_spin);
717         IXGBE_CORE_LOCK_DESTROY(adapter);
718         return (0);
719 }
720
721 /*********************************************************************
722  *
723  *  Shutdown entry point
724  *
725  **********************************************************************/
726
727 static int
728 ixgbe_shutdown(device_t dev)
729 {
730         struct adapter *adapter = device_get_softc(dev);
731         IXGBE_CORE_LOCK(adapter);
732         ixgbe_stop(adapter);
733         IXGBE_CORE_UNLOCK(adapter);
734         return (0);
735 }
736
737
738 /*********************************************************************
739  *  Transmit entry point
740  *
741  *  ixgbe_start is called by the stack to initiate a transmit.
742  *  The driver will remain in this routine as long as there are
743  *  packets to transmit and transmit resources are available.
744  *  In case resources are not available stack is notified and
745  *  the packet is requeued.
746  **********************************************************************/
747
748 static void
749 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
750 {
751         struct mbuf    *m_head;
752         struct adapter *adapter = txr->adapter;
753
754         IXGBE_TX_LOCK_ASSERT(txr);
755
756         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
757                 return;
758         if (!adapter->link_active)
759                 return;
760
761         while (!ifq_is_empty(&ifp->if_snd)) {
762                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
763                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
764                         break;
765                 }
766
767                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
768                 if (m_head == NULL)
769                         break;
770
771                 if (ixgbe_xmit(txr, &m_head)) {
772 #if 0 /* XXX: prepend to an ALTQ queue ? */
773                         if (m_head != NULL)
774                                 IF_PREPEND(&ifp->if_snd, m_head);
775 #endif
776                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
777                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
778                         break;
779                 }
780                 /* Send a copy of the frame to the BPF listener */
781                 ETHER_BPF_MTAP(ifp, m_head);
782
783                 /* Set watchdog on */
784                 txr->watchdog_time = ticks;
785                 txr->queue_status = IXGBE_QUEUE_WORKING;
786
787         }
788         return;
789 }
790
791 /*
792  * Legacy TX start - called by the stack, this
793  * always uses the first tx ring, and should
794  * not be used with multiqueue tx enabled.
795  */
796 static void
797 ixgbe_start(struct ifnet *ifp)
798 {
799         struct adapter *adapter = ifp->if_softc;
800         struct tx_ring  *txr = adapter->tx_rings;
801
802         if (ifp->if_flags & IFF_RUNNING) {
803                 IXGBE_TX_LOCK(txr);
804                 ixgbe_start_locked(txr, ifp);
805                 IXGBE_TX_UNLOCK(txr);
806         }
807         return;
808 }
809
810 #if 0 /* __FreeBSD_version >= 800000 */
811 /*
812 ** Multiqueue Transmit driver
813 **
814 */
815 static int
816 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
817 {
818         struct adapter  *adapter = ifp->if_softc;
819         struct ix_queue *que;
820         struct tx_ring  *txr;
821         int             i = 0, err = 0;
822
823         /* Which queue to use */
824         if ((m->m_flags & M_FLOWID) != 0)
825                 i = m->m_pkthdr.flowid % adapter->num_queues;
826         else
827                 i = curcpu % adapter->num_queues;
828
829         txr = &adapter->tx_rings[i];
830         que = &adapter->queues[i];
831
832         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
833             IXGBE_TX_TRYLOCK(txr)) {
834                 err = ixgbe_mq_start_locked(ifp, txr, m);
835                 IXGBE_TX_UNLOCK(txr);
836         } else {
837                 err = drbr_enqueue(ifp, txr->br, m);
838                 taskqueue_enqueue(que->tq, &que->que_task);
839         }
840
841         return (err);
842 }
843
844 static int
845 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
846 {
847         struct adapter  *adapter = txr->adapter;
848         struct mbuf     *next;
849         int             enqueued, err = 0;
850
851         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
852             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
853             adapter->link_active == 0) {
854                 if (m != NULL)
855                         err = drbr_enqueue(ifp, txr->br, m);
856                 return (err);
857         }
858
859         enqueued = 0;
860         if (m == NULL) {
861                 next = drbr_dequeue(ifp, txr->br);
862         } else if (drbr_needs_enqueue(ifp, txr->br)) {
863                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
864                         return (err);
865                 next = drbr_dequeue(ifp, txr->br);
866         } else
867                 next = m;
868
869         /* Process the queue */
870         while (next != NULL) {
871                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
872                         if (next != NULL)
873                                 err = drbr_enqueue(ifp, txr->br, next);
874                         break;
875                 }
876                 enqueued++;
877                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
878                 /* Send a copy of the frame to the BPF listener */
879                 ETHER_BPF_MTAP(ifp, next);
880                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
881                         break;
882                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
883                         ixgbe_txeof(txr);
884                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
885                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
886                         break;
887                 }
888                 next = drbr_dequeue(ifp, txr->br);
889         }
890
891         if (enqueued > 0) {
892                 /* Set watchdog on */
893                 txr->queue_status |= IXGBE_QUEUE_WORKING;
894                 txr->watchdog_time = ticks;
895         }
896
897         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
898                 ixgbe_txeof(txr);
899
900         return (err);
901 }
902
903 /*
904 ** Flush all ring buffers
905 */
906 static void
907 ixgbe_qflush(struct ifnet *ifp)
908 {
909         struct adapter  *adapter = ifp->if_softc;
910         struct tx_ring  *txr = adapter->tx_rings;
911         struct mbuf     *m;
912
913         for (int i = 0; i < adapter->num_queues; i++, txr++) {
914                 IXGBE_TX_LOCK(txr);
915                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
916                         m_freem(m);
917                 IXGBE_TX_UNLOCK(txr);
918         }
919         if_qflush(ifp);
920 }
921 #endif /* __FreeBSD_version >= 800000 */
922
923 /*********************************************************************
924  *  Ioctl entry point
925  *
926  *  ixgbe_ioctl is called when the user wants to configure the
927  *  interface.
928  *
929  *  return 0 on success, positive on failure
930  **********************************************************************/
931
932 static int
933 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
934 {
935         struct adapter  *adapter = ifp->if_softc;
936         struct ifreq    *ifr = (struct ifreq *) data;
937 #if defined(INET) || defined(INET6)
938         struct ifaddr *ifa = (struct ifaddr *)data;
939         bool            avoid_reset = FALSE;
940 #endif
941         int             error = 0;
942
943         switch (command) {
944
945         case SIOCSIFADDR:
946 #ifdef INET
947                 if (ifa->ifa_addr->sa_family == AF_INET)
948                         avoid_reset = TRUE;
949 #endif
950 #ifdef INET6
951                 if (ifa->ifa_addr->sa_family == AF_INET6)
952                         avoid_reset = TRUE;
953 #endif
954 #if defined(INET) || defined(INET6)
955                 /*
956                 ** Calling init results in link renegotiation,
957                 ** so we avoid doing it when possible.
958                 */
959                 if (avoid_reset) {
960                         ifp->if_flags |= IFF_UP;
961                         if (!(ifp->if_flags & IFF_RUNNING))
962                                 ixgbe_init(adapter);
963                         if (!(ifp->if_flags & IFF_NOARP))
964                                 arp_ifinit(ifp, ifa);
965                 } else
966                         error = ether_ioctl(ifp, command, data);
967 #endif
968                 break;
969         case SIOCSIFMTU:
970                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
971                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
972                         error = EINVAL;
973                 } else {
974                         IXGBE_CORE_LOCK(adapter);
975                         ifp->if_mtu = ifr->ifr_mtu;
976                         adapter->max_frame_size =
977                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
978                         ixgbe_init_locked(adapter);
979                         IXGBE_CORE_UNLOCK(adapter);
980                 }
981                 break;
982         case SIOCSIFFLAGS:
983                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
984                 IXGBE_CORE_LOCK(adapter);
985                 if (ifp->if_flags & IFF_UP) {
986                         if ((ifp->if_flags & IFF_RUNNING)) {
987                                 if ((ifp->if_flags ^ adapter->if_flags) &
988                                     (IFF_PROMISC | IFF_ALLMULTI)) {
989                                         ixgbe_set_promisc(adapter);
990                                 }
991                         } else
992                                 ixgbe_init_locked(adapter);
993                 } else
994                         if (ifp->if_flags & IFF_RUNNING)
995                                 ixgbe_stop(adapter);
996                 adapter->if_flags = ifp->if_flags;
997                 IXGBE_CORE_UNLOCK(adapter);
998                 break;
999         case SIOCADDMULTI:
1000         case SIOCDELMULTI:
1001                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1002                 if (ifp->if_flags & IFF_RUNNING) {
1003                         IXGBE_CORE_LOCK(adapter);
1004                         ixgbe_disable_intr(adapter);
1005                         ixgbe_set_multi(adapter);
1006                         ixgbe_enable_intr(adapter);
1007                         IXGBE_CORE_UNLOCK(adapter);
1008                 }
1009                 break;
1010         case SIOCSIFMEDIA:
1011         case SIOCGIFMEDIA:
1012                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1013                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1014                 break;
1015         case SIOCSIFCAP:
1016         {
1017                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1018                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1019                 if (mask & IFCAP_HWCSUM)
1020                         ifp->if_capenable ^= IFCAP_HWCSUM;
1021 #if 0 /* NET_TSO */
1022                 if (mask & IFCAP_TSO4)
1023                         ifp->if_capenable ^= IFCAP_TSO4;
1024                 if (mask & IFCAP_TSO6)
1025                         ifp->if_capenable ^= IFCAP_TSO6;
1026 #endif
1027 #if 0 /* NET_LRO */
1028                 if (mask & IFCAP_LRO)
1029                         ifp->if_capenable ^= IFCAP_LRO;
1030 #endif
1031                 if (mask & IFCAP_VLAN_HWTAGGING)
1032                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1033                 if (mask & IFCAP_VLAN_HWFILTER)
1034                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1035 #if 0 /* NET_TSO */
1036                 if (mask & IFCAP_VLAN_HWTSO)
1037                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1038 #endif
1039                 if (ifp->if_flags & IFF_RUNNING) {
1040                         IXGBE_CORE_LOCK(adapter);
1041                         ixgbe_init_locked(adapter);
1042                         IXGBE_CORE_UNLOCK(adapter);
1043                 }
1044 #if 0
1045                 VLAN_CAPABILITIES(ifp);
1046 #endif
1047                 break;
1048         }
1049
1050         default:
1051                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1052                 error = ether_ioctl(ifp, command, data);
1053                 break;
1054         }
1055
1056         return (error);
1057 }
1058
1059 /*********************************************************************
1060  *  Init entry point
1061  *
1062  *  This routine is used in two ways. It is used by the stack as
1063  *  init entry point in network interface structure. It is also used
1064  *  by the driver as a hw/sw initialization routine to get to a
1065  *  consistent state.
1066  *
1067  *  return 0 on success, positive on failure
1068  **********************************************************************/
1069 #define IXGBE_MHADD_MFS_SHIFT 16
1070
1071 static void
1072 ixgbe_init_locked(struct adapter *adapter)
1073 {
1074         struct ifnet   *ifp = adapter->ifp;
1075         device_t        dev = adapter->dev;
1076         struct ixgbe_hw *hw = &adapter->hw;
1077         u32             k, txdctl, mhadd, gpie;
1078         u32             rxdctl, rxctrl;
1079
1080         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1081         INIT_DEBUGOUT("ixgbe_init: begin");
1082         hw->adapter_stopped = FALSE;
1083         ixgbe_stop_adapter(hw);
1084         callout_stop(&adapter->timer);
1085
1086         /* reprogram the RAR[0] in case user changed it. */
1087         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1088
1089         /* Get the latest mac address, User can use a LAA */
1090         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1091               IXGBE_ETH_LENGTH_OF_ADDRESS);
1092         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1093         hw->addr_ctrl.rar_used_count = 1;
1094
1095         /* Set the various hardware offload abilities */
1096         ifp->if_hwassist = 0;
1097 #if 0 /* NET_TSO */
1098         if (ifp->if_capenable & IFCAP_TSO)
1099                 ifp->if_hwassist |= CSUM_TSO;
1100 #endif
1101         if (ifp->if_capenable & IFCAP_TXCSUM) {
1102                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1103 #if 0
1104                 if (hw->mac.type != ixgbe_mac_82598EB)
1105                         ifp->if_hwassist |= CSUM_SCTP;
1106 #endif
1107         }
1108
1109         /* Prepare transmit descriptors and buffers */
1110         if (ixgbe_setup_transmit_structures(adapter)) {
1111                 device_printf(dev,"Could not setup transmit structures\n");
1112                 ixgbe_stop(adapter);
1113                 return;
1114         }
1115
1116         ixgbe_init_hw(hw);
1117         ixgbe_initialize_transmit_units(adapter);
1118
1119         /* Setup Multicast table */
1120         ixgbe_set_multi(adapter);
1121
1122         /*
1123         ** Determine the correct mbuf pool
1124         ** for doing jumbo/headersplit
1125         */
1126         if (adapter->max_frame_size <= 2048)
1127                 adapter->rx_mbuf_sz = MCLBYTES;
1128         else if (adapter->max_frame_size <= 4096)
1129                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1130         else if (adapter->max_frame_size <= 9216)
1131                 adapter->rx_mbuf_sz = MJUM9BYTES;
1132         else
1133                 adapter->rx_mbuf_sz = MJUM16BYTES;
1134
1135         /* Prepare receive descriptors and buffers */
1136         if (ixgbe_setup_receive_structures(adapter)) {
1137                 device_printf(dev,"Could not setup receive structures\n");
1138                 ixgbe_stop(adapter);
1139                 return;
1140         }
1141
1142         /* Configure RX settings */
1143         ixgbe_initialize_receive_units(adapter);
1144
1145         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1146
1147         /* Enable Fan Failure Interrupt */
1148         gpie |= IXGBE_SDP1_GPIEN;
1149
1150         /* Add for Module detection */
1151         if (hw->mac.type == ixgbe_mac_82599EB)
1152                 gpie |= IXGBE_SDP2_GPIEN;
1153
1154         /* Thermal Failure Detection */
1155         if (hw->mac.type == ixgbe_mac_X540)
1156                 gpie |= IXGBE_SDP0_GPIEN;
1157
1158         if (adapter->msix > 1) {
1159                 /* Enable Enhanced MSIX mode */
1160                 gpie |= IXGBE_GPIE_MSIX_MODE;
1161                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1162                     IXGBE_GPIE_OCD;
1163         }
1164         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1165
1166         /* Set MTU size */
1167         if (ifp->if_mtu > ETHERMTU) {
1168                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1169                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1170                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1171                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1172         }
1173         
1174         /* Now enable all the queues */
1175
1176         for (int i = 0; i < adapter->num_queues; i++) {
1177                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1178                 txdctl |= IXGBE_TXDCTL_ENABLE;
1179                 /* Set WTHRESH to 8, burst writeback */
1180                 txdctl |= (8 << 16);
1181                 /*
1182                  * When the internal queue falls below PTHRESH (32),
1183                  * start prefetching as long as there are at least
1184                  * HTHRESH (1) buffers ready. The values are taken
1185                  * from the Intel linux driver 3.8.21.
1186                  * Prefetching enables tx line rate even with 1 queue.
1187                  */
1188                 txdctl |= (32 << 0) | (1 << 8);
1189                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1190         }
1191
1192         for (int i = 0; i < adapter->num_queues; i++) {
1193                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1194                 if (hw->mac.type == ixgbe_mac_82598EB) {
1195                         /*
1196                         ** PTHRESH = 21
1197                         ** HTHRESH = 4
1198                         ** WTHRESH = 8
1199                         */
1200                         rxdctl &= ~0x3FFFFF;
1201                         rxdctl |= 0x080420;
1202                 }
1203                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1204                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1205                 for (k = 0; k < 10; k++) {
1206                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1207                             IXGBE_RXDCTL_ENABLE)
1208                                 break;
1209                         else
1210                                 msec_delay(1);
1211                 }
1212                 wmb();
1213 #ifdef DEV_NETMAP
1214                 /*
1215                  * In netmap mode, we must preserve the buffers made
1216                  * available to userspace before the if_init()
1217                  * (this is true by default on the TX side, because
1218                  * init makes all buffers available to userspace).
1219                  *
1220                  * netmap_reset() and the device specific routines
1221                  * (e.g. ixgbe_setup_receive_rings()) map these
1222                  * buffers at the end of the NIC ring, so here we
1223                  * must set the RDT (tail) register to make sure
1224                  * they are not overwritten.
1225                  *
1226                  * In this driver the NIC ring starts at RDH = 0,
1227                  * RDT points to the last slot available for reception (?),
1228                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1229                  */
1230                 if (ifp->if_capenable & IFCAP_NETMAP) {
1231                         struct netmap_adapter *na = NA(adapter->ifp);
1232                         struct netmap_kring *kring = &na->rx_rings[i];
1233                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1234
1235                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1236                 } else
1237 #endif /* DEV_NETMAP */
1238                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1239         }
1240
1241         /* Set up VLAN support and filter */
1242         ixgbe_setup_vlan_hw_support(adapter);
1243
1244         /* Enable Receive engine */
1245         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1246         if (hw->mac.type == ixgbe_mac_82598EB)
1247                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1248         rxctrl |= IXGBE_RXCTRL_RXEN;
1249         ixgbe_enable_rx_dma(hw, rxctrl);
1250
1251         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1252
1253         /* Set up MSI/X routing */
1254         if (ixgbe_enable_msix)  {
1255                 ixgbe_configure_ivars(adapter);
1256                 /* Set up auto-mask */
1257                 if (hw->mac.type == ixgbe_mac_82598EB)
1258                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1259                 else {
1260                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1261                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1262                 }
1263         } else {  /* Simple settings for Legacy/MSI */
1264                 ixgbe_set_ivar(adapter, 0, 0, 0);
1265                 ixgbe_set_ivar(adapter, 0, 0, 1);
1266                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1267         }
1268
1269 #ifdef IXGBE_FDIR
1270         /* Init Flow director */
1271         if (hw->mac.type != ixgbe_mac_82598EB) {
1272                 u32 hdrm = 64 << fdir_pballoc;
1273
1274                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1275                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1276         }
1277 #endif
1278
1279         /*
1280         ** Check on any SFP devices that
1281         ** need to be kick-started
1282         */
1283         if (hw->phy.type == ixgbe_phy_none) {
1284                 int err = hw->phy.ops.identify(hw);
1285                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1286                         device_printf(dev,
1287                             "Unsupported SFP+ module type was detected.\n");
1288                         return;
1289                 }
1290         }
1291
1292         /* Set moderation on the Link interrupt */
1293         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1294
1295         /* Config/Enable Link */
1296         ixgbe_config_link(adapter);
1297
1298         /* And now turn on interrupts */
1299         ixgbe_enable_intr(adapter);
1300
1301         /* Now inform the stack we're ready */
1302         ifp->if_flags |= IFF_RUNNING;
1303         ifp->if_flags &= ~IFF_OACTIVE;
1304
1305         return;
1306 }
1307
1308 static void
1309 ixgbe_init(void *arg)
1310 {
1311         struct adapter *adapter = arg;
1312
1313         IXGBE_CORE_LOCK(adapter);
1314         ixgbe_init_locked(adapter);
1315         IXGBE_CORE_UNLOCK(adapter);
1316         return;
1317 }
1318
1319
1320 /*
1321 **
1322 ** MSIX Interrupt Handlers and Tasklets
1323 **
1324 */
1325
1326 static inline void
1327 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1328 {
1329         struct ixgbe_hw *hw = &adapter->hw;
1330         u64     queue = (u64)(1 << vector);
1331         u32     mask;
1332
1333         if (hw->mac.type == ixgbe_mac_82598EB) {
1334                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1335                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1336         } else {
1337                 mask = (queue & 0xFFFFFFFF);
1338                 if (mask)
1339                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1340                 mask = (queue >> 32);
1341                 if (mask)
1342                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1343         }
1344 }
1345
1346 static inline void
1347 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1348 {
1349         struct ixgbe_hw *hw = &adapter->hw;
1350         u64     queue = (u64)(1 << vector);
1351         u32     mask;
1352
1353         if (hw->mac.type == ixgbe_mac_82598EB) {
1354                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1355                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1356         } else {
1357                 mask = (queue & 0xFFFFFFFF);
1358                 if (mask)
1359                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1360                 mask = (queue >> 32);
1361                 if (mask)
1362                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1363         }
1364 }
1365
1366 static inline void
1367 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1368 {
1369         u32 mask;
1370
1371         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1372                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1373                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1374         } else {
1375                 mask = (queues & 0xFFFFFFFF);
1376                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1377                 mask = (queues >> 32);
1378                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1379         }
1380 }
1381
1382
1383 static void
1384 ixgbe_handle_que(void *context, int pending)
1385 {
1386         struct ix_queue *que = context;
1387         struct adapter  *adapter = que->adapter;
1388         struct tx_ring  *txr = que->txr;
1389         struct ifnet    *ifp = adapter->ifp;
1390         bool            more;
1391
1392         if (ifp->if_flags & IFF_RUNNING) {
1393                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1394                 IXGBE_TX_LOCK(txr);
1395                 ixgbe_txeof(txr);
1396 #if 0 /*__FreeBSD_version >= 800000*/
1397                 if (!drbr_empty(ifp, txr->br))
1398                         ixgbe_mq_start_locked(ifp, txr, NULL);
1399 #else
1400                 if (!ifq_is_empty(&ifp->if_snd))
1401                         ixgbe_start_locked(txr, ifp);
1402 #endif
1403                 IXGBE_TX_UNLOCK(txr);
1404                 if (more) {
1405                         taskqueue_enqueue(que->tq, &que->que_task);
1406                         return;
1407                 }
1408         }
1409
1410         /* Reenable this interrupt */
1411         ixgbe_enable_queue(adapter, que->msix);
1412         return;
1413 }
1414
1415
1416 /*********************************************************************
1417  *
1418  *  Legacy Interrupt Service routine
1419  *
1420  **********************************************************************/
1421
1422 static void
1423 ixgbe_legacy_irq(void *arg)
1424 {
1425         struct ix_queue *que = arg;
1426         struct adapter  *adapter = que->adapter;
1427         struct ixgbe_hw *hw = &adapter->hw;
1428         struct          tx_ring *txr = adapter->tx_rings;
1429         bool            more_tx, more_rx;
1430         u32             reg_eicr, loop = MAX_LOOP;
1431
1432
1433         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1434
1435         ++que->irqs;
1436         if (reg_eicr == 0) {
1437                 ixgbe_enable_intr(adapter);
1438                 return;
1439         }
1440
1441         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1442
1443         IXGBE_TX_LOCK(txr);
1444         do {
1445                 more_tx = ixgbe_txeof(txr);
1446         } while (loop-- && more_tx);
1447         IXGBE_TX_UNLOCK(txr);
1448
1449         if (more_rx || more_tx)
1450                 taskqueue_enqueue(que->tq, &que->que_task);
1451
1452         /* Check for fan failure */
1453         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1454             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1455                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1456                     "REPLACE IMMEDIATELY!!\n");
1457                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1458         }
1459
1460         /* Link status change */
1461         if (reg_eicr & IXGBE_EICR_LSC)
1462                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1463
1464         ixgbe_enable_intr(adapter);
1465         return;
1466 }
1467
1468
1469 /*********************************************************************
1470  *
1471  *  MSIX Queue Interrupt Service routine
1472  *
1473  **********************************************************************/
1474 void
1475 ixgbe_msix_que(void *arg)
1476 {
1477         struct ix_queue *que = arg;
1478         struct adapter  *adapter = que->adapter;
1479         struct tx_ring  *txr = que->txr;
1480         struct rx_ring  *rxr = que->rxr;
1481         bool            more_tx, more_rx;
1482         u32             newitr = 0;
1483
1484         ixgbe_disable_queue(adapter, que->msix);
1485         ++que->irqs;
1486
1487         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1488
1489         IXGBE_TX_LOCK(txr);
1490         more_tx = ixgbe_txeof(txr);
1491         /*
1492         ** Make certain that if the stack 
1493         ** has anything queued the task gets
1494         ** scheduled to handle it.
1495         */
1496 #if 0
1497 #if __FreeBSD_version < 800000
1498         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1499 #else
1500         if (!drbr_empty(adapter->ifp, txr->br))
1501 #endif
1502 #endif
1503         if (!ifq_is_empty(&adapter->ifp->if_snd))
1504                 more_tx = 1;
1505         IXGBE_TX_UNLOCK(txr);
1506
1507         /* Do AIM now? */
1508
1509         if (ixgbe_enable_aim == FALSE)
1510                 goto no_calc;
1511         /*
1512         ** Do Adaptive Interrupt Moderation:
1513         **  - Write out last calculated setting
1514         **  - Calculate based on average size over
1515         **    the last interval.
1516         */
1517         if (que->eitr_setting)
1518                 IXGBE_WRITE_REG(&adapter->hw,
1519                     IXGBE_EITR(que->msix), que->eitr_setting);
1520  
1521         que->eitr_setting = 0;
1522
1523         /* Idle, do nothing */
1524         if ((txr->bytes == 0) && (rxr->bytes == 0))
1525                 goto no_calc;
1526                                 
1527         if ((txr->bytes) && (txr->packets))
1528                 newitr = txr->bytes/txr->packets;
1529         if ((rxr->bytes) && (rxr->packets))
1530                 newitr = max(newitr,
1531                     (rxr->bytes / rxr->packets));
1532         newitr += 24; /* account for hardware frame, crc */
1533
1534         /* set an upper boundary */
1535         newitr = min(newitr, 3000);
1536
1537         /* Be nice to the mid range */
1538         if ((newitr > 300) && (newitr < 1200))
1539                 newitr = (newitr / 3);
1540         else
1541                 newitr = (newitr / 2);
1542
1543         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1544                 newitr |= newitr << 16;
1545         else
1546                 newitr |= IXGBE_EITR_CNT_WDIS;
1547                  
1548         /* save for next interrupt */
1549         que->eitr_setting = newitr;
1550
1551         /* Reset state */
1552         txr->bytes = 0;
1553         txr->packets = 0;
1554         rxr->bytes = 0;
1555         rxr->packets = 0;
1556
1557 no_calc:
1558         if (more_tx || more_rx)
1559                 taskqueue_enqueue(que->tq, &que->que_task);
1560         else /* Reenable this interrupt */
1561                 ixgbe_enable_queue(adapter, que->msix);
1562         return;
1563 }
1564
1565
1566 static void
1567 ixgbe_msix_link(void *arg)
1568 {
1569         struct adapter  *adapter = arg;
1570         struct ixgbe_hw *hw = &adapter->hw;
1571         u32             reg_eicr;
1572
1573         ++adapter->link_irq;
1574
1575         /* First get the cause */
1576         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1577         /* Clear interrupt with write */
1578         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1579
1580         /* Link status change */
1581         if (reg_eicr & IXGBE_EICR_LSC)
1582                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1583
1584         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1585 #ifdef IXGBE_FDIR
1586                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1587                         /* This is probably overkill :) */
1588                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1589                                 return;
1590                         /* Clear the interrupt */
1591                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR);
1592                         /* Turn off the interface */
1593                         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1594                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1595                 } else
1596 #endif
1597                 if (reg_eicr & IXGBE_EICR_ECC) {
1598                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1599                             "Please Reboot!!\n");
1600                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1601                 } else
1602
1603                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1604                         /* Clear the interrupt */
1605                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1606                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1607                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1608                         /* Clear the interrupt */
1609                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1610                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1611                 }
1612         } 
1613
1614         /* Check for fan failure */
1615         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1616             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1617                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1618                     "REPLACE IMMEDIATELY!!\n");
1619                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1620         }
1621
1622         /* Check for over temp condition */
1623         if ((hw->mac.type == ixgbe_mac_X540) &&
1624             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1625                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1626                     "PHY IS SHUT DOWN!!\n");
1627                 device_printf(adapter->dev, "System shutdown required\n");
1628                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1629         }
1630
1631         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1632         return;
1633 }
1634
1635 /*********************************************************************
1636  *
1637  *  Media Ioctl callback
1638  *
1639  *  This routine is called whenever the user queries the status of
1640  *  the interface using ifconfig.
1641  *
1642  **********************************************************************/
1643 static void
1644 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1645 {
1646         struct adapter *adapter = ifp->if_softc;
1647
1648         INIT_DEBUGOUT("ixgbe_media_status: begin");
1649         IXGBE_CORE_LOCK(adapter);
1650         ixgbe_update_link_status(adapter);
1651
1652         ifmr->ifm_status = IFM_AVALID;
1653         ifmr->ifm_active = IFM_ETHER;
1654
1655         if (!adapter->link_active) {
1656                 IXGBE_CORE_UNLOCK(adapter);
1657                 return;
1658         }
1659
1660         ifmr->ifm_status |= IFM_ACTIVE;
1661
1662         switch (adapter->link_speed) {
1663                 case IXGBE_LINK_SPEED_100_FULL:
1664                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1665                         break;
1666                 case IXGBE_LINK_SPEED_1GB_FULL:
1667                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1668                         break;
1669                 case IXGBE_LINK_SPEED_10GB_FULL:
1670                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1671                         break;
1672         }
1673
1674         IXGBE_CORE_UNLOCK(adapter);
1675
1676         return;
1677 }
1678
1679 /*********************************************************************
1680  *
1681  *  Media Ioctl callback
1682  *
1683  *  This routine is called when the user changes speed/duplex using
1684  *  media/mediopt option with ifconfig.
1685  *
1686  **********************************************************************/
1687 static int
1688 ixgbe_media_change(struct ifnet * ifp)
1689 {
1690         struct adapter *adapter = ifp->if_softc;
1691         struct ifmedia *ifm = &adapter->media;
1692
1693         INIT_DEBUGOUT("ixgbe_media_change: begin");
1694
1695         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1696                 return (EINVAL);
1697
1698         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1699         case IFM_AUTO:
1700                 adapter->hw.phy.autoneg_advertised =
1701                     IXGBE_LINK_SPEED_100_FULL |
1702                     IXGBE_LINK_SPEED_1GB_FULL |
1703                     IXGBE_LINK_SPEED_10GB_FULL;
1704                 break;
1705         default:
1706                 device_printf(adapter->dev, "Only auto media type\n");
1707                 return (EINVAL);
1708         }
1709
1710         return (0);
1711 }
1712
1713 /*********************************************************************
1714  *
1715  *  This routine maps the mbufs to tx descriptors, allowing the
1716  *  TX engine to transmit the packets. 
1717  *      - return 0 on success, positive on failure
1718  *
1719  **********************************************************************/
1720
1721 static int
1722 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1723 {
1724         struct adapter  *adapter = txr->adapter;
1725         u32             olinfo_status = 0, cmd_type_len;
1726         u32             paylen = 0;
1727         int             i, j, error, nsegs;
1728         int             first, last = 0;
1729         struct mbuf     *m_head;
1730         bus_dma_segment_t segs[1];
1731         bus_dmamap_t    map;
1732         struct ixgbe_tx_buf *txbuf;
1733         union ixgbe_adv_tx_desc *txd = NULL;
1734
1735         m_head = *m_headp;
1736
1737         /* Basic descriptor defines */
1738         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1739             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1740
1741         if (m_head->m_flags & M_VLANTAG)
1742                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1743
1744         /*
1745          * Important to capture the first descriptor
1746          * used because it will contain the index of
1747          * the one we tell the hardware to report back
1748          */
1749         first = txr->next_avail_desc;
1750         txbuf = &txr->tx_buffers[first];
1751         map = txbuf->map;
1752
1753         /*
1754          * Map the packet for DMA.
1755          */
1756         error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1757             *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1758
1759         if (error == EFBIG) {
1760                 struct mbuf *m;
1761
1762                 m = m_defrag(*m_headp, MB_DONTWAIT);
1763                 if (m == NULL) {
1764                         adapter->mbuf_defrag_failed++;
1765                         m_freem(*m_headp);
1766                         *m_headp = NULL;
1767                         return (ENOBUFS);
1768                 }
1769                 *m_headp = m;
1770
1771                 /* Try it again */
1772                 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1773                     *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1774
1775                 if (error == ENOMEM) {
1776                         adapter->no_tx_dma_setup++;
1777                         return (error);
1778                 } else if (error != 0) {
1779                         adapter->no_tx_dma_setup++;
1780                         m_freem(*m_headp);
1781                         *m_headp = NULL;
1782                         return (error);
1783                 }
1784         } else if (error == ENOMEM) {
1785                 adapter->no_tx_dma_setup++;
1786                 return (error);
1787         } else if (error != 0) {
1788                 adapter->no_tx_dma_setup++;
1789                 m_freem(*m_headp);
1790                 *m_headp = NULL;
1791                 return (error);
1792         }
1793
1794         /* Make certain there are enough descriptors */
1795         if (nsegs > txr->tx_avail - 2) {
1796                 txr->no_desc_avail++;
1797                 error = ENOBUFS;
1798                 goto xmit_fail;
1799         }
1800         m_head = *m_headp;
1801
1802         /*
1803         ** Set up the appropriate offload context
1804         ** this becomes the first descriptor of 
1805         ** a packet.
1806         */
1807 #if 0 /* NET_TSO */
1808         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1809                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1810                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1811                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1812                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1813                         ++adapter->tso_tx;
1814                 } else
1815                         return (ENXIO);
1816         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1817 #endif
1818         if (ixgbe_tx_ctx_setup(txr, m_head))
1819                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1820
1821 #ifdef IXGBE_IEEE1588
1822         /* This is changing soon to an mtag detection */
1823         if (we detect this mbuf has a TSTAMP mtag)
1824                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1825 #endif
1826
1827 #ifdef IXGBE_FDIR
1828         /* Do the flow director magic */
1829         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1830                 ++txr->atr_count;
1831                 if (txr->atr_count >= atr_sample_rate) {
1832                         ixgbe_atr(txr, m_head);
1833                         txr->atr_count = 0;
1834                 }
1835         }
1836 #endif
1837         /* Record payload length */
1838         if (paylen == 0)
1839                 olinfo_status |= m_head->m_pkthdr.len <<
1840                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1841
1842         i = txr->next_avail_desc;
1843         for (j = 0; j < nsegs; j++) {
1844                 bus_size_t seglen;
1845                 bus_addr_t segaddr;
1846
1847                 txbuf = &txr->tx_buffers[i];
1848                 txd = &txr->tx_base[i];
1849                 seglen = segs[j].ds_len;
1850                 segaddr = htole64(segs[j].ds_addr);
1851
1852                 txd->read.buffer_addr = segaddr;
1853                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1854                     cmd_type_len |seglen);
1855                 txd->read.olinfo_status = htole32(olinfo_status);
1856                 last = i; /* descriptor that will get completion IRQ */
1857
1858                 if (++i == adapter->num_tx_desc)
1859                         i = 0;
1860
1861                 txbuf->m_head = NULL;
1862                 txbuf->eop_index = -1;
1863         }
1864
1865         txd->read.cmd_type_len |=
1866             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1867         txr->tx_avail -= nsegs;
1868         txr->next_avail_desc = i;
1869
1870         txbuf->m_head = m_head;
1871         /* Swap the dma map between the first and last descriptor */
1872         txr->tx_buffers[first].map = txbuf->map;
1873         txbuf->map = map;
1874         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1875
1876         /* Set the index of the descriptor that will be marked done */
1877         txbuf = &txr->tx_buffers[first];
1878         txbuf->eop_index = last;
1879
1880         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1881             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1882         /*
1883          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1884          * hardware that this frame is available to transmit.
1885          */
1886         ++txr->total_packets;
1887         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1888
1889         return (0);
1890
1891 xmit_fail:
1892         bus_dmamap_unload(txr->txtag, txbuf->map);
1893         return (error);
1894
1895 }
1896
1897 static void
1898 ixgbe_set_promisc(struct adapter *adapter)
1899 {
1900         u_int32_t       reg_rctl;
1901         struct ifnet   *ifp = adapter->ifp;
1902
1903         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1904         reg_rctl &= (~IXGBE_FCTRL_UPE);
1905         reg_rctl &= (~IXGBE_FCTRL_MPE);
1906         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1907
1908         if (ifp->if_flags & IFF_PROMISC) {
1909                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1910                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1911         } else if (ifp->if_flags & IFF_ALLMULTI) {
1912                 reg_rctl |= IXGBE_FCTRL_MPE;
1913                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1914                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1915         }
1916         return;
1917 }
1918
1919
1920 /*********************************************************************
1921  *  Multicast Update
1922  *
1923  *  This routine is called whenever multicast address list is updated.
1924  *
1925  **********************************************************************/
1926 #define IXGBE_RAR_ENTRIES 16
1927
1928 static void
1929 ixgbe_set_multi(struct adapter *adapter)
1930 {
1931         u32     fctrl;
1932         u8      *mta;
1933         u8      *update_ptr;
1934         struct  ifmultiaddr *ifma;
1935         int     mcnt = 0;
1936         struct ifnet   *ifp = adapter->ifp;
1937
1938         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1939
1940         mta = adapter->mta;
1941         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1942             MAX_NUM_MULTICAST_ADDRESSES);
1943
1944         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1945         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1946         if (ifp->if_flags & IFF_PROMISC)
1947                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1948         else if (ifp->if_flags & IFF_ALLMULTI) {
1949                 fctrl |= IXGBE_FCTRL_MPE;
1950                 fctrl &= ~IXGBE_FCTRL_UPE;
1951         } else
1952                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1953         
1954         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1955
1956         spin_lock(&adapter->mcast_spin);
1957         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1958                 if (ifma->ifma_addr->sa_family != AF_LINK)
1959                         continue;
1960                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1961                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1962                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1963                 mcnt++;
1964         }
1965         spin_unlock(&adapter->mcast_spin);
1966
1967         update_ptr = mta;
1968         ixgbe_update_mc_addr_list(&adapter->hw,
1969             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1970
1971         return;
1972 }
1973
1974 /*
1975  * This is an iterator function now needed by the multicast
1976  * shared code. It simply feeds the shared code routine the
1977  * addresses in the array of ixgbe_set_multi() one by one.
1978  */
1979 static u8 *
1980 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1981 {
1982         u8 *addr = *update_ptr;
1983         u8 *newptr;
1984         *vmdq = 0;
1985
1986         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1987         *update_ptr = newptr;
1988         return addr;
1989 }
1990
1991
1992 /*********************************************************************
1993  *  Timer routine
1994  *
1995  *  This routine checks for link status,updates statistics,
1996  *  and runs the watchdog check.
1997  *
1998  **********************************************************************/
1999
2000 static void
2001 ixgbe_local_timer(void *arg)
2002 {
2003         struct adapter  *adapter = arg;
2004         device_t        dev = adapter->dev;
2005         struct ifnet    *ifp = adapter->ifp;
2006         struct ix_queue *que = adapter->queues;
2007         struct tx_ring  *txr = adapter->tx_rings;
2008         int             hung, busy, paused;
2009
2010         lockmgr(&adapter->core_lock, LK_EXCLUSIVE);
2011         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2012         hung = busy = paused = 0;
2013
2014         /* Check for pluggable optics */
2015         if (adapter->sfp_probe)
2016                 if (!ixgbe_sfp_probe(adapter))
2017                         goto out; /* Nothing to do */
2018
2019         ixgbe_update_link_status(adapter);
2020         ixgbe_update_stats_counters(adapter);
2021
2022         /*
2023          * If the interface has been paused
2024          * then don't do the watchdog check
2025          */
2026         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2027                 paused = 1;
2028
2029         /*
2030         ** Check the TX queues status
2031         **      - central locked handling of OACTIVE
2032         **      - watchdog only if all queues show hung
2033         */          
2034         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2035                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2036                     (paused == 0))
2037                         ++hung;
2038                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2039                         ++busy;
2040                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2041                         taskqueue_enqueue(que->tq, &que->que_task);
2042         }
2043         /* Only truely watchdog if all queues show hung */
2044         if (hung == adapter->num_queues)
2045                 goto watchdog;
2046         /* Only turn off the stack flow when ALL are depleted */
2047         if (busy == adapter->num_queues)
2048                 ifp->if_flags |= IFF_OACTIVE;
2049         else if ((ifp->if_flags & IFF_OACTIVE) &&
2050             (busy < adapter->num_queues))
2051                 ifp->if_flags &= ~IFF_OACTIVE;
2052
2053 out:
2054         ixgbe_rearm_queues(adapter, adapter->que_mask);
2055         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2056         lockmgr(&adapter->core_lock, LK_RELEASE);
2057         return;
2058
2059 watchdog:
2060         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2061         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2062             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2063             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2064         device_printf(dev,"TX(%d) desc avail = %d,"
2065             "Next TX to Clean = %d\n",
2066             txr->me, txr->tx_avail, txr->next_to_clean);
2067         adapter->ifp->if_flags &= ~IFF_RUNNING;
2068         adapter->watchdog_events++;
2069         ixgbe_init_locked(adapter);
2070
2071         lockmgr(&adapter->core_lock, LK_RELEASE);
2072 }
2073
2074 /*
2075 ** Note: this routine updates the OS on the link state
2076 **      the real check of the hardware only happens with
2077 **      a link interrupt.
2078 */
2079 static void
2080 ixgbe_update_link_status(struct adapter *adapter)
2081 {
2082         struct ifnet    *ifp = adapter->ifp;
2083         struct tx_ring *txr = adapter->tx_rings;
2084         device_t dev = adapter->dev;
2085
2086
2087         if (adapter->link_up){ 
2088                 if (adapter->link_active == FALSE) {
2089                         if (bootverbose)
2090                                 device_printf(dev,"Link is up %d Gbps %s \n",
2091                                     ((adapter->link_speed == 128)? 10:1),
2092                                     "Full Duplex");
2093                         adapter->link_active = TRUE;
2094                         ifp->if_link_state = LINK_STATE_UP;
2095                         if_link_state_change(ifp);
2096                 }
2097         } else { /* Link down */
2098                 if (adapter->link_active == TRUE) {
2099                         if (bootverbose)
2100                                 device_printf(dev,"Link is Down\n");
2101                         ifp->if_link_state = LINK_STATE_DOWN;
2102                         if_link_state_change(ifp);
2103                         adapter->link_active = FALSE;
2104                         for (int i = 0; i < adapter->num_queues;
2105                             i++, txr++)
2106                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2107                 }
2108         }
2109
2110         return;
2111 }
2112
2113
2114 /*********************************************************************
2115  *
2116  *  This routine disables all traffic on the adapter by issuing a
2117  *  global reset on the MAC and deallocates TX/RX buffers.
2118  *
2119  **********************************************************************/
2120
2121 static void
2122 ixgbe_stop(void *arg)
2123 {
2124         struct ifnet   *ifp;
2125         struct adapter *adapter = arg;
2126         struct ixgbe_hw *hw = &adapter->hw;
2127         ifp = adapter->ifp;
2128
2129         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2130
2131         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2132         ixgbe_disable_intr(adapter);
2133         callout_stop(&adapter->timer);
2134
2135         /* Let the stack know...*/
2136         ifp->if_flags &= ~IFF_RUNNING;
2137         ifp->if_flags |= IFF_OACTIVE;
2138
2139         ixgbe_reset_hw(hw);
2140         hw->adapter_stopped = FALSE;
2141         ixgbe_stop_adapter(hw);
2142         /* Turn off the laser */
2143         if (hw->phy.multispeed_fiber)
2144                 ixgbe_disable_tx_laser(hw);
2145
2146         /* reprogram the RAR[0] in case user changed it. */
2147         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2148
2149         return;
2150 }
2151
2152
2153 /*********************************************************************
2154  *
2155  *  Determine hardware revision.
2156  *
2157  **********************************************************************/
2158 static void
2159 ixgbe_identify_hardware(struct adapter *adapter)
2160 {
2161         device_t        dev = adapter->dev;
2162         struct ixgbe_hw *hw = &adapter->hw;
2163
2164         /* Save off the information about this board */
2165         hw->vendor_id = pci_get_vendor(dev);
2166         hw->device_id = pci_get_device(dev);
2167         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2168         hw->subsystem_vendor_id =
2169             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2170         hw->subsystem_device_id =
2171             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2172
2173         /* We need this here to set the num_segs below */
2174         ixgbe_set_mac_type(hw);
2175
2176         /* Pick up the 82599 and VF settings */
2177         if (hw->mac.type != ixgbe_mac_82598EB) {
2178                 hw->phy.smart_speed = ixgbe_smart_speed;
2179                 adapter->num_segs = IXGBE_82599_SCATTER;
2180         } else
2181                 adapter->num_segs = IXGBE_82598_SCATTER;
2182
2183         return;
2184 }
2185
2186 /*********************************************************************
2187  *
2188  *  Determine optic type
2189  *
2190  **********************************************************************/
2191 static void
2192 ixgbe_setup_optics(struct adapter *adapter)
2193 {
2194         struct ixgbe_hw *hw = &adapter->hw;
2195         int             layer;
2196         
2197         layer = ixgbe_get_supported_physical_layer(hw);
2198
2199         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2200                 adapter->optics = IFM_10G_T;
2201                 return;
2202         }
2203
2204         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2205                 adapter->optics = IFM_1000_T;
2206                 return;
2207         }
2208
2209         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2210             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2211                 adapter->optics = IFM_10G_LR;
2212                 return;
2213         }
2214
2215         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2216                 adapter->optics = IFM_10G_SR;
2217                 return;
2218         }
2219
2220         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2221                 adapter->optics = IFM_10G_TWINAX;
2222                 return;
2223         }
2224
2225         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2226             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2227                 adapter->optics = IFM_10G_CX4;
2228                 return;
2229         }
2230
2231         /* If we get here just set the default */
2232         adapter->optics = IFM_ETHER | IFM_AUTO;
2233         return;
2234 }
2235
2236 /*********************************************************************
2237  *
2238  *  Setup the Legacy or MSI Interrupt handler
2239  *
2240  **********************************************************************/
2241 static int
2242 ixgbe_allocate_legacy(struct adapter *adapter)
2243 {
2244         device_t dev = adapter->dev;
2245         struct          ix_queue *que = adapter->queues;
2246         int error, rid = 0;
2247
2248         /* MSI RID at 1 */
2249         if (adapter->msix == 1)
2250                 rid = 1;
2251
2252         /* We allocate a single interrupt resource */
2253         adapter->res = bus_alloc_resource_any(dev,
2254             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2255         if (adapter->res == NULL) {
2256                 device_printf(dev, "Unable to allocate bus resource: "
2257                     "interrupt\n");
2258                 return (ENXIO);
2259         }
2260
2261         /*
2262          * Try allocating a fast interrupt and the associated deferred
2263          * processing contexts.
2264          */
2265         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2266         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2267             taskqueue_thread_enqueue, &que->tq);
2268         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2269             device_get_nameunit(adapter->dev));
2270
2271         /* Tasklets for Link, SFP and Multispeed Fiber */
2272         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2273         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2274         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2275 #ifdef IXGBE_FDIR
2276         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2277 #endif
2278         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2279             taskqueue_thread_enqueue, &adapter->tq);
2280         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2281             device_get_nameunit(adapter->dev));
2282
2283         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2284             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2285                 device_printf(dev, "Failed to register fast interrupt "
2286                     "handler: %d\n", error);
2287                 taskqueue_free(que->tq);
2288                 taskqueue_free(adapter->tq);
2289                 que->tq = NULL;
2290                 adapter->tq = NULL;
2291                 return (error);
2292         }
2293         /* For simplicity in the handlers */
2294         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2295
2296         return (0);
2297 }
2298
2299
2300 /*********************************************************************
2301  *
2302  *  Setup MSIX Interrupt resources and handlers 
2303  *
2304  **********************************************************************/
2305 static int
2306 ixgbe_allocate_msix(struct adapter *adapter)
2307 {
2308         device_t        dev = adapter->dev;
2309         struct          ix_queue *que = adapter->queues;
2310         int             error, rid, vector = 0;
2311
2312         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2313                 rid = vector + 1;
2314                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2315                     RF_SHAREABLE | RF_ACTIVE);
2316                 if (que->res == NULL) {
2317                         device_printf(dev,"Unable to allocate"
2318                             " bus resource: que interrupt [%d]\n", vector);
2319                         return (ENXIO);
2320                 }
2321                 /* Set the handler function */
2322                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2323                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2324                 if (error) {
2325                         que->res = NULL;
2326                         device_printf(dev, "Failed to register QUE handler");
2327                         return (error);
2328                 }
2329 #if 0 /* __FreeBSD_version >= 800504 */
2330                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2331 #endif
2332                 que->msix = vector;
2333                 adapter->que_mask |= (u64)(1 << que->msix);
2334                 /*
2335                 ** Bind the msix vector, and thus the
2336                 ** ring to the corresponding cpu.
2337                 */
2338 #if 0 /* XXX */
2339                 if (adapter->num_queues > 1)
2340                         bus_bind_intr(dev, que->res, i);
2341 #endif
2342
2343                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2344                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2345                     taskqueue_thread_enqueue, &que->tq);
2346                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2347                     device_get_nameunit(adapter->dev));
2348         }
2349
2350         /* and Link */
2351         rid = vector + 1;
2352         adapter->res = bus_alloc_resource_any(dev,
2353             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2354         if (!adapter->res) {
2355                 device_printf(dev,"Unable to allocate"
2356             " bus resource: Link interrupt [%d]\n", rid);
2357                 return (ENXIO);
2358         }
2359         /* Set the link handler function */
2360         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2361             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2362         if (error) {
2363                 adapter->res = NULL;
2364                 device_printf(dev, "Failed to register LINK handler");
2365                 return (error);
2366         }
2367 #if 0 /* __FreeBSD_version >= 800504 */
2368         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2369 #endif
2370         adapter->linkvec = vector;
2371         /* Tasklets for Link, SFP and Multispeed Fiber */
2372         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2373         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2374         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2375 #ifdef IXGBE_FDIR
2376         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2377 #endif
2378         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2379             taskqueue_thread_enqueue, &adapter->tq);
2380         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2381             device_get_nameunit(adapter->dev));
2382
2383         return (0);
2384 }
2385
2386 #if 0   /* HAVE_MSIX */
2387 /*
2388  * Setup Either MSI/X or MSI
2389  */
2390 static int
2391 ixgbe_setup_msix(struct adapter *adapter)
2392 {
2393         device_t dev = adapter->dev;
2394         int rid, want, queues, msgs;
2395
2396         /* Override by tuneable */
2397         if (ixgbe_enable_msix == 0)
2398                 goto msi;
2399
2400         /* First try MSI/X */
2401         rid = PCIR_BAR(MSIX_82598_BAR);
2402         adapter->msix_mem = bus_alloc_resource_any(dev,
2403             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2404         if (!adapter->msix_mem) {
2405                 rid += 4;       /* 82599 maps in higher BAR */
2406                 adapter->msix_mem = bus_alloc_resource_any(dev,
2407                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2408         }
2409         if (!adapter->msix_mem) {
2410                 /* May not be enabled */
2411                 device_printf(adapter->dev,
2412                     "Unable to map MSIX table \n");
2413                 goto msi;
2414         }
2415
2416         msgs = pci_msix_count(dev); 
2417         if (msgs == 0) { /* system has msix disabled */
2418                 bus_release_resource(dev, SYS_RES_MEMORY,
2419                     rid, adapter->msix_mem);
2420                 adapter->msix_mem = NULL;
2421                 goto msi;
2422         }
2423
2424         /* Figure out a reasonable auto config value */
2425         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2426
2427         if (ixgbe_num_queues != 0)
2428                 queues = ixgbe_num_queues;
2429         /* Set max queues to 8 when autoconfiguring */
2430         else if ((ixgbe_num_queues == 0) && (queues > 8))
2431                 queues = 8;
2432
2433         /*
2434         ** Want one vector (RX/TX pair) per queue
2435         ** plus an additional for Link.
2436         */
2437         want = queues + 1;
2438         if (msgs >= want)
2439                 msgs = want;
2440         else {
2441                 device_printf(adapter->dev,
2442                     "MSIX Configuration Problem, "
2443                     "%d vectors but %d queues wanted!\n",
2444                     msgs, want);
2445                 return (0); /* Will go to Legacy setup */
2446         }
2447         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2448                 device_printf(adapter->dev,
2449                     "Using MSIX interrupts with %d vectors\n", msgs);
2450                 adapter->num_queues = queues;
2451                 return (msgs);
2452         }
2453 msi:
2454         msgs = pci_msi_count(dev);
2455         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2456                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2457         else
2458                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2459         return (msgs);
2460 }
2461 #endif
2462
2463
2464 static int
2465 ixgbe_allocate_pci_resources(struct adapter *adapter)
2466 {
2467         int             rid;
2468         device_t        dev = adapter->dev;
2469
2470         rid = PCIR_BAR(0);
2471         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2472             &rid, RF_ACTIVE);
2473
2474         if (!(adapter->pci_mem)) {
2475                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2476                 return (ENXIO);
2477         }
2478
2479         adapter->osdep.mem_bus_space_tag =
2480                 rman_get_bustag(adapter->pci_mem);
2481         adapter->osdep.mem_bus_space_handle =
2482                 rman_get_bushandle(adapter->pci_mem);
2483         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2484
2485         /* Legacy defaults */
2486         adapter->num_queues = 1;
2487         adapter->hw.back = &adapter->osdep;
2488
2489         /*
2490         ** Now setup MSI or MSI/X, should
2491         ** return us the number of supported
2492         ** vectors. (Will be 1 for MSI)
2493         */
2494 #if 0   /* HAVE_MSIX */
2495         adapter->msix = ixgbe_setup_msix(adapter);
2496 #endif
2497         return (0);
2498 }
2499
2500 static void
2501 ixgbe_free_pci_resources(struct adapter * adapter)
2502 {
2503         struct          ix_queue *que = adapter->queues;
2504         device_t        dev = adapter->dev;
2505         int             rid, memrid;
2506
2507         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2508                 memrid = PCIR_BAR(MSIX_82598_BAR);
2509         else
2510                 memrid = PCIR_BAR(MSIX_82599_BAR);
2511
2512         /*
2513         ** There is a slight possibility of a failure mode
2514         ** in attach that will result in entering this function
2515         ** before interrupt resources have been initialized, and
2516         ** in that case we do not want to execute the loops below
2517         ** We can detect this reliably by the state of the adapter
2518         ** res pointer.
2519         */
2520         if (adapter->res == NULL)
2521                 goto mem;
2522
2523         /*
2524         **  Release all msix queue resources:
2525         */
2526         for (int i = 0; i < adapter->num_queues; i++, que++) {
2527                 rid = que->msix + 1;
2528                 if (que->tag != NULL) {
2529                         bus_teardown_intr(dev, que->res, que->tag);
2530                         que->tag = NULL;
2531                 }
2532                 if (que->res != NULL)
2533                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2534         }
2535
2536
2537         /* Clean the Legacy or Link interrupt last */
2538         if (adapter->linkvec) /* we are doing MSIX */
2539                 rid = adapter->linkvec + 1;
2540         else
2541                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2542
2543         if (adapter->tag != NULL) {
2544                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2545                 adapter->tag = NULL;
2546         }
2547         if (adapter->res != NULL)
2548                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2549
2550 mem:
2551         if (adapter->msix)
2552                 pci_release_msi(dev);
2553
2554         if (adapter->msix_mem != NULL)
2555                 bus_release_resource(dev, SYS_RES_MEMORY,
2556                     memrid, adapter->msix_mem);
2557
2558         if (adapter->pci_mem != NULL)
2559                 bus_release_resource(dev, SYS_RES_MEMORY,
2560                     PCIR_BAR(0), adapter->pci_mem);
2561
2562         return;
2563 }
2564
2565 /*********************************************************************
2566  *
2567  *  Setup networking device structure and register an interface.
2568  *
2569  **********************************************************************/
2570 static int
2571 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2572 {
2573         struct ixgbe_hw *hw = &adapter->hw;
2574         struct ifnet   *ifp;
2575
2576         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2577
2578         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2579         if (ifp == NULL) {
2580                 device_printf(dev, "can not allocate ifnet structure\n");
2581                 return (-1);
2582         }
2583         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2584         ifp->if_baudrate = 1000000000;
2585         ifp->if_init = ixgbe_init;
2586         ifp->if_softc = adapter;
2587         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2588         ifp->if_ioctl = ixgbe_ioctl;
2589         ifp->if_start = ixgbe_start;
2590 #if 0 /* __FreeBSD_version >= 800000 */
2591         ifp->if_transmit = ixgbe_mq_start;
2592         ifp->if_qflush = ixgbe_qflush;
2593 #endif
2594         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2595
2596         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2597
2598         adapter->max_frame_size =
2599             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2600
2601         /*
2602          * Tell the upper layer(s) we support long frames.
2603          */
2604         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2605
2606 #if 0 /* NET_TSO */
2607         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2608 #endif
2609         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2610         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2611         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2612 #if 0 /* NET_TSO */
2613                              |  IFCAP_VLAN_HWTSO
2614 #endif
2615                              |  IFCAP_VLAN_MTU;
2616         ifp->if_capenable = ifp->if_capabilities;
2617
2618         /* Don't enable LRO by default */
2619 #if 0 /* NET_LRO */
2620         ifp->if_capabilities |= IFCAP_LRO;
2621 #endif
2622
2623         /*
2624         ** Don't turn this on by default, if vlans are
2625         ** created on another pseudo device (eg. lagg)
2626         ** then vlan events are not passed thru, breaking
2627         ** operation, but with HW FILTER off it works. If
2628         ** using vlans directly on the ixgbe driver you can
2629         ** enable this and get full hardware tag filtering.
2630         */
2631         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2632
2633         /*
2634          * Specify the media types supported by this adapter and register
2635          * callbacks to update media and link information
2636          */
2637         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2638                      ixgbe_media_status);
2639         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2640         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2641         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2642                 ifmedia_add(&adapter->media,
2643                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2644                 ifmedia_add(&adapter->media,
2645                     IFM_ETHER | IFM_1000_T, 0, NULL);
2646         }
2647         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2648         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2649
2650         return (0);
2651 }
2652
2653 static void
2654 ixgbe_config_link(struct adapter *adapter)
2655 {
2656         struct ixgbe_hw *hw = &adapter->hw;
2657         u32     autoneg, err = 0;
2658         bool    sfp, negotiate;
2659
2660         sfp = ixgbe_is_sfp(hw);
2661
2662         if (sfp) { 
2663                 if (hw->phy.multispeed_fiber) {
2664                         hw->mac.ops.setup_sfp(hw);
2665                         ixgbe_enable_tx_laser(hw);
2666                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2667                 } else
2668                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2669         } else {
2670                 if (hw->mac.ops.check_link)
2671                         err = ixgbe_check_link(hw, &autoneg,
2672                             &adapter->link_up, FALSE);
2673                 if (err)
2674                         goto out;
2675                 autoneg = hw->phy.autoneg_advertised;
2676                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2677                         err  = hw->mac.ops.get_link_capabilities(hw,
2678                             &autoneg, &negotiate);
2679                 if (err)
2680                         goto out;
2681                 if (hw->mac.ops.setup_link)
2682                         err = hw->mac.ops.setup_link(hw, autoneg,
2683                             negotiate, adapter->link_up);
2684         }
2685 out:
2686         return;
2687 }
2688
2689 /********************************************************************
2690  * Manage DMA'able memory.
2691  *******************************************************************/
2692 static void
2693 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2694 {
2695         if (error)
2696                 return;
2697         *(bus_addr_t *) arg = segs->ds_addr;
2698         return;
2699 }
2700
2701 static int
2702 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2703                 struct ixgbe_dma_alloc *dma, int mapflags)
2704 {
2705         device_t dev = adapter->dev;
2706         int             r;
2707
2708         r = bus_dma_tag_create(NULL,    /* parent */
2709                                DBA_ALIGN, 0,    /* alignment, bounds */
2710                                BUS_SPACE_MAXADDR,       /* lowaddr */
2711                                BUS_SPACE_MAXADDR,       /* highaddr */
2712                                NULL, NULL,      /* filter, filterarg */
2713                                size,    /* maxsize */
2714                                1,       /* nsegments */
2715                                size,    /* maxsegsize */
2716                                BUS_DMA_ALLOCNOW,        /* flags */
2717                                &dma->dma_tag);
2718         if (r != 0) {
2719                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2720                        "error %u\n", r);
2721                 goto fail_0;
2722         }
2723         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2724                              BUS_DMA_NOWAIT, &dma->dma_map);
2725         if (r != 0) {
2726                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2727                        "error %u\n", r);
2728                 goto fail_1;
2729         }
2730         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2731                             size,
2732                             ixgbe_dmamap_cb,
2733                             &dma->dma_paddr,
2734                             mapflags | BUS_DMA_NOWAIT);
2735         if (r != 0) {
2736                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2737                        "error %u\n", r);
2738                 goto fail_2;
2739         }
2740         dma->dma_size = size;
2741         return (0);
2742 fail_2:
2743         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2744 fail_1:
2745         bus_dma_tag_destroy(dma->dma_tag);
2746 fail_0:
2747         dma->dma_map = NULL;
2748         dma->dma_tag = NULL;
2749         return (r);
2750 }
2751
2752 static void
2753 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2754 {
2755         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2756             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2757         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2758         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2759         bus_dma_tag_destroy(dma->dma_tag);
2760 }
2761
2762
2763 /*********************************************************************
2764  *
2765  *  Allocate memory for the transmit and receive rings, and then
2766  *  the descriptors associated with each, called only once at attach.
2767  *
2768  **********************************************************************/
2769 static int
2770 ixgbe_allocate_queues(struct adapter *adapter)
2771 {
2772         device_t        dev = adapter->dev;
2773         struct ix_queue *que;
2774         struct tx_ring  *txr;
2775         struct rx_ring  *rxr;
2776         int rsize, tsize, error = IXGBE_SUCCESS;
2777         int txconf = 0, rxconf = 0;
2778
2779         /* First allocate the top level queue structs */
2780         if (!(adapter->queues =
2781             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2782             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2783                 device_printf(dev, "Unable to allocate queue memory\n");
2784                 error = ENOMEM;
2785                 goto fail;
2786         }
2787
2788         /* First allocate the TX ring struct memory */
2789         if (!(adapter->tx_rings =
2790             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2791             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2792                 device_printf(dev, "Unable to allocate TX ring memory\n");
2793                 error = ENOMEM;
2794                 goto tx_fail;
2795         }
2796
2797         /* Next allocate the RX */
2798         if (!(adapter->rx_rings =
2799             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2800             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2801                 device_printf(dev, "Unable to allocate RX ring memory\n");
2802                 error = ENOMEM;
2803                 goto rx_fail;
2804         }
2805
2806         /* For the ring itself */
2807         tsize = roundup2(adapter->num_tx_desc *
2808             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2809
2810         /*
2811          * Now set up the TX queues, txconf is needed to handle the
2812          * possibility that things fail midcourse and we need to
2813          * undo memory gracefully
2814          */ 
2815         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2816                 /* Set up some basics */
2817                 txr = &adapter->tx_rings[i];
2818                 txr->adapter = adapter;
2819                 txr->me = i;
2820
2821                 /* Initialize the TX side lock */
2822                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2823                     device_get_nameunit(dev), txr->me);
2824                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2825
2826                 if (ixgbe_dma_malloc(adapter, tsize,
2827                         &txr->txdma, BUS_DMA_NOWAIT)) {
2828                         device_printf(dev,
2829                             "Unable to allocate TX Descriptor memory\n");
2830                         error = ENOMEM;
2831                         goto err_tx_desc;
2832                 }
2833                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2834                 bzero((void *)txr->tx_base, tsize);
2835
2836                 /* Now allocate transmit buffers for the ring */
2837                 if (ixgbe_allocate_transmit_buffers(txr)) {
2838                         device_printf(dev,
2839                             "Critical Failure setting up transmit buffers\n");
2840                         error = ENOMEM;
2841                         goto err_tx_desc;
2842                 }
2843 #if 0 /* __FreeBSD_version >= 800000 */
2844                 /* Allocate a buf ring */
2845                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2846                     M_WAITOK, &txr->tx_mtx);
2847                 if (txr->br == NULL) {
2848                         device_printf(dev,
2849                             "Critical Failure setting up buf ring\n");
2850                         error = ENOMEM;
2851                         goto err_tx_desc;
2852                 }
2853 #endif
2854         }
2855
2856         /*
2857          * Next the RX queues...
2858          */ 
2859         rsize = roundup2(adapter->num_rx_desc *
2860             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2861         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2862                 rxr = &adapter->rx_rings[i];
2863                 /* Set up some basics */
2864                 rxr->adapter = adapter;
2865                 rxr->me = i;
2866
2867                 /* Initialize the RX side lock */
2868                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2869                     device_get_nameunit(dev), rxr->me);
2870                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2871
2872                 if (ixgbe_dma_malloc(adapter, rsize,
2873                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2874                         device_printf(dev,
2875                             "Unable to allocate RxDescriptor memory\n");
2876                         error = ENOMEM;
2877                         goto err_rx_desc;
2878                 }
2879                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2880                 bzero((void *)rxr->rx_base, rsize);
2881
2882                 /* Allocate receive buffers for the ring*/
2883                 if (ixgbe_allocate_receive_buffers(rxr)) {
2884                         device_printf(dev,
2885                             "Critical Failure setting up receive buffers\n");
2886                         error = ENOMEM;
2887                         goto err_rx_desc;
2888                 }
2889         }
2890
2891         /*
2892         ** Finally set up the queue holding structs
2893         */
2894         for (int i = 0; i < adapter->num_queues; i++) {
2895                 que = &adapter->queues[i];
2896                 que->adapter = adapter;
2897                 que->txr = &adapter->tx_rings[i];
2898                 que->rxr = &adapter->rx_rings[i];
2899         }
2900
2901         return (0);
2902
2903 err_rx_desc:
2904         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2905                 ixgbe_dma_free(adapter, &rxr->rxdma);
2906 err_tx_desc:
2907         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2908                 ixgbe_dma_free(adapter, &txr->txdma);
2909         kfree(adapter->rx_rings, M_DEVBUF);
2910 rx_fail:
2911         kfree(adapter->tx_rings, M_DEVBUF);
2912 tx_fail:
2913         kfree(adapter->queues, M_DEVBUF);
2914 fail:
2915         return (error);
2916 }
2917
2918 /*********************************************************************
2919  *
2920  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2921  *  the information needed to transmit a packet on the wire. This is
2922  *  called only once at attach, setup is done every reset.
2923  *
2924  **********************************************************************/
2925 static int
2926 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2927 {
2928         struct adapter *adapter = txr->adapter;
2929         device_t dev = adapter->dev;
2930         struct ixgbe_tx_buf *txbuf;
2931         int error, i;
2932
2933         /*
2934          * Setup DMA descriptor areas.
2935          */
2936         if ((error = bus_dma_tag_create(
2937                                NULL,    /* parent */
2938                                1, 0,            /* alignment, bounds */
2939                                BUS_SPACE_MAXADDR,       /* lowaddr */
2940                                BUS_SPACE_MAXADDR,       /* highaddr */
2941                                NULL, NULL,              /* filter, filterarg */
2942                                IXGBE_TSO_SIZE,          /* maxsize */
2943                                adapter->num_segs,       /* nsegments */
2944                                PAGE_SIZE,               /* maxsegsize */
2945                                0,                       /* flags */
2946                                &txr->txtag))) {
2947                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2948                 goto fail;
2949         }
2950
2951         if (!(txr->tx_buffers =
2952             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2953             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2954                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2955                 error = ENOMEM;
2956                 goto fail;
2957         }
2958
2959         /* Create the descriptor buffer dma maps */
2960         txbuf = txr->tx_buffers;
2961         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2962                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2963                 if (error != 0) {
2964                         device_printf(dev, "Unable to create TX DMA map\n");
2965                         goto fail;
2966                 }
2967         }
2968
2969         return 0;
2970 fail:
2971         /* We free all, it handles case where we are in the middle */
2972         ixgbe_free_transmit_structures(adapter);
2973         return (error);
2974 }
2975
2976 /*********************************************************************
2977  *
2978  *  Initialize a transmit ring.
2979  *
2980  **********************************************************************/
2981 static void
2982 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2983 {
2984         struct adapter *adapter = txr->adapter;
2985         struct ixgbe_tx_buf *txbuf;
2986         int i;
2987 #ifdef DEV_NETMAP
2988         struct netmap_adapter *na = NA(adapter->ifp);
2989         struct netmap_slot *slot;
2990 #endif /* DEV_NETMAP */
2991
2992         /* Clear the old ring contents */
2993         IXGBE_TX_LOCK(txr);
2994 #ifdef DEV_NETMAP
2995         /*
2996          * (under lock): if in netmap mode, do some consistency
2997          * checks and set slot to entry 0 of the netmap ring.
2998          */
2999         slot = netmap_reset(na, NR_TX, txr->me, 0);
3000 #endif /* DEV_NETMAP */
3001         bzero((void *)txr->tx_base,
3002               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3003         /* Reset indices */
3004         txr->next_avail_desc = 0;
3005         txr->next_to_clean = 0;
3006
3007         /* Free any existing tx buffers. */
3008         txbuf = txr->tx_buffers;
3009         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3010                 if (txbuf->m_head != NULL) {
3011                         bus_dmamap_sync(txr->txtag, txbuf->map,
3012                             BUS_DMASYNC_POSTWRITE);
3013                         bus_dmamap_unload(txr->txtag, txbuf->map);
3014                         m_freem(txbuf->m_head);
3015                         txbuf->m_head = NULL;
3016                 }
3017 #ifdef DEV_NETMAP
3018                 /*
3019                  * In netmap mode, set the map for the packet buffer.
3020                  * NOTE: Some drivers (not this one) also need to set
3021                  * the physical buffer address in the NIC ring.
3022                  * Slots in the netmap ring (indexed by "si") are
3023                  * kring->nkr_hwofs positions "ahead" wrt the
3024                  * corresponding slot in the NIC ring. In some drivers
3025                  * (not here) nkr_hwofs can be negative. Function
3026                  * netmap_idx_n2k() handles wraparounds properly.
3027                  */
3028                 if (slot) {
3029                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3030                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3031                 }
3032 #endif /* DEV_NETMAP */
3033                 /* Clear the EOP index */
3034                 txbuf->eop_index = -1;
3035         }
3036
3037 #ifdef IXGBE_FDIR
3038         /* Set the rate at which we sample packets */
3039         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3040                 txr->atr_sample = atr_sample_rate;
3041 #endif
3042
3043         /* Set number of descriptors available */
3044         txr->tx_avail = adapter->num_tx_desc;
3045
3046         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3047             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3048         IXGBE_TX_UNLOCK(txr);
3049 }
3050
3051 /*********************************************************************
3052  *
3053  *  Initialize all transmit rings.
3054  *
3055  **********************************************************************/
3056 static int
3057 ixgbe_setup_transmit_structures(struct adapter *adapter)
3058 {
3059         struct tx_ring *txr = adapter->tx_rings;
3060
3061         for (int i = 0; i < adapter->num_queues; i++, txr++)
3062                 ixgbe_setup_transmit_ring(txr);
3063
3064         return (0);
3065 }
3066
3067 /*********************************************************************
3068  *
3069  *  Enable transmit unit.
3070  *
3071  **********************************************************************/
3072 static void
3073 ixgbe_initialize_transmit_units(struct adapter *adapter)
3074 {
3075         struct tx_ring  *txr = adapter->tx_rings;
3076         struct ixgbe_hw *hw = &adapter->hw;
3077
3078         /* Setup the Base and Length of the Tx Descriptor Ring */
3079
3080         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3081                 u64     tdba = txr->txdma.dma_paddr;
3082                 u32     txctrl;
3083
3084                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3085                        (tdba & 0x00000000ffffffffULL));
3086                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3087                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3088                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3089
3090                 /* Setup the HW Tx Head and Tail descriptor pointers */
3091                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3092                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3093
3094                 /* Setup Transmit Descriptor Cmd Settings */
3095                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3096                 txr->queue_status = IXGBE_QUEUE_IDLE;
3097
3098                 /* Disable Head Writeback */
3099                 switch (hw->mac.type) {
3100                 case ixgbe_mac_82598EB:
3101                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3102                         break;
3103                 case ixgbe_mac_82599EB:
3104                 case ixgbe_mac_X540:
3105                 default:
3106                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3107                         break;
3108                 }
3109                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
3110                 switch (hw->mac.type) {
3111                 case ixgbe_mac_82598EB:
3112                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3113                         break;
3114                 case ixgbe_mac_82599EB:
3115                 case ixgbe_mac_X540:
3116                 default:
3117                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3118                         break;
3119                 }
3120
3121         }
3122
3123         if (hw->mac.type != ixgbe_mac_82598EB) {
3124                 u32 dmatxctl, rttdcs;
3125                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3126                 dmatxctl |= IXGBE_DMATXCTL_TE;
3127                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3128                 /* Disable arbiter to set MTQC */
3129                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3130                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3131                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3132                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3133                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3134                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3135         }
3136
3137         return;
3138 }
3139
3140 /*********************************************************************
3141  *
3142  *  Free all transmit rings.
3143  *
3144  **********************************************************************/
3145 static void
3146 ixgbe_free_transmit_structures(struct adapter *adapter)
3147 {
3148         struct tx_ring *txr = adapter->tx_rings;
3149
3150         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3151                 IXGBE_TX_LOCK(txr);
3152                 ixgbe_free_transmit_buffers(txr);
3153                 ixgbe_dma_free(adapter, &txr->txdma);
3154                 IXGBE_TX_UNLOCK(txr);
3155                 IXGBE_TX_LOCK_DESTROY(txr);
3156         }
3157         kfree(adapter->tx_rings, M_DEVBUF);
3158 }
3159
3160 /*********************************************************************
3161  *
3162  *  Free transmit ring related data structures.
3163  *
3164  **********************************************************************/
3165 static void
3166 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3167 {
3168         struct adapter *adapter = txr->adapter;
3169         struct ixgbe_tx_buf *tx_buffer;
3170         int             i;
3171
3172         INIT_DEBUGOUT("free_transmit_ring: begin");
3173
3174         if (txr->tx_buffers == NULL)
3175                 return;
3176
3177         tx_buffer = txr->tx_buffers;
3178         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3179                 if (tx_buffer->m_head != NULL) {
3180                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3181                             BUS_DMASYNC_POSTWRITE);
3182                         bus_dmamap_unload(txr->txtag,
3183                             tx_buffer->map);
3184                         m_freem(tx_buffer->m_head);
3185                         tx_buffer->m_head = NULL;
3186                         if (tx_buffer->map != NULL) {
3187                                 bus_dmamap_destroy(txr->txtag,
3188                                     tx_buffer->map);
3189                                 tx_buffer->map = NULL;
3190                         }
3191                 } else if (tx_buffer->map != NULL) {
3192                         bus_dmamap_unload(txr->txtag,
3193                             tx_buffer->map);
3194                         bus_dmamap_destroy(txr->txtag,
3195                             tx_buffer->map);
3196                         tx_buffer->map = NULL;
3197                 }
3198         }
3199 #if 0 /* __FreeBSD_version >= 800000 */
3200         if (txr->br != NULL)
3201                 buf_ring_free(txr->br, M_DEVBUF);
3202 #endif
3203         if (txr->tx_buffers != NULL) {
3204                 kfree(txr->tx_buffers, M_DEVBUF);
3205                 txr->tx_buffers = NULL;
3206         }
3207         if (txr->txtag != NULL) {
3208                 bus_dma_tag_destroy(txr->txtag);
3209                 txr->txtag = NULL;
3210         }
3211         return;
3212 }
3213
3214 /*********************************************************************
3215  *
3216  *  Advanced Context Descriptor setup for VLAN or CSUM
3217  *
3218  **********************************************************************/
3219
3220 static bool
3221 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3222 {
3223         struct adapter *adapter = txr->adapter;
3224         struct ixgbe_adv_tx_context_desc *TXD;
3225         struct ixgbe_tx_buf        *tx_buffer;
3226         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3227         struct ether_vlan_header *eh;
3228         struct ip *ip;
3229         struct ip6_hdr *ip6;
3230         int  ehdrlen, ip_hlen = 0;
3231         u16     etype;
3232         u8      ipproto = 0;
3233         bool    offload = TRUE;
3234         int ctxd = txr->next_avail_desc;
3235 #ifdef NET_VLAN
3236         u16 vtag = 0;
3237 #endif
3238
3239
3240         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3241                 offload = FALSE;
3242
3243         tx_buffer = &txr->tx_buffers[ctxd];
3244         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3245
3246         /*
3247         ** In advanced descriptors the vlan tag must 
3248         ** be placed into the descriptor itself.
3249         */
3250 #ifdef NET_VLAN
3251         if (mp->m_flags & M_VLANTAG) {
3252                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3253                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3254         } else if (offload == FALSE)
3255                 return FALSE;
3256 #endif
3257
3258         /*
3259          * Determine where frame payload starts.
3260          * Jump over vlan headers if already present,
3261          * helpful for QinQ too.
3262          */
3263         eh = mtod(mp, struct ether_vlan_header *);
3264         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3265                 etype = ntohs(eh->evl_proto);
3266                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3267         } else {
3268                 etype = ntohs(eh->evl_encap_proto);
3269                 ehdrlen = ETHER_HDR_LEN;
3270         }
3271
3272         /* Set the ether header length */
3273         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3274
3275         switch (etype) {
3276                 case ETHERTYPE_IP:
3277                         ip = (struct ip *)(mp->m_data + ehdrlen);
3278                         ip_hlen = ip->ip_hl << 2;
3279                         ipproto = ip->ip_p;
3280                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3281                         break;
3282                 case ETHERTYPE_IPV6:
3283                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3284                         ip_hlen = sizeof(struct ip6_hdr);
3285                         /* XXX-BZ this will go badly in case of ext hdrs. */
3286                         ipproto = ip6->ip6_nxt;
3287                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3288                         break;
3289                 default:
3290                         offload = FALSE;
3291                         break;
3292         }
3293
3294         vlan_macip_lens |= ip_hlen;
3295         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3296
3297         switch (ipproto) {
3298                 case IPPROTO_TCP:
3299                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3300                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3301                         break;
3302
3303                 case IPPROTO_UDP:
3304                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3305                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3306                         break;
3307
3308 #if 0
3309                 case IPPROTO_SCTP:
3310                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3311                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3312                         break;
3313 #endif
3314                 default:
3315                         offload = FALSE;
3316                         break;
3317         }
3318
3319         /* Now copy bits into descriptor */
3320         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3321         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3322         TXD->seqnum_seed = htole32(0);
3323         TXD->mss_l4len_idx = htole32(0);
3324
3325         tx_buffer->m_head = NULL;
3326         tx_buffer->eop_index = -1;
3327
3328         /* We've consumed the first desc, adjust counters */
3329         if (++ctxd == adapter->num_tx_desc)
3330                 ctxd = 0;
3331         txr->next_avail_desc = ctxd;
3332         --txr->tx_avail;
3333
3334         return (offload);
3335 }
3336
3337 /**********************************************************************
3338  *
3339  *  Setup work for hardware segmentation offload (TSO) on
3340  *  adapters using advanced tx descriptors
3341  *
3342  **********************************************************************/
3343 #if 0   /* NET_TSO */
3344 static bool
3345 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3346     u32 *olinfo_status)
3347 {
3348         struct adapter *adapter = txr->adapter;
3349         struct ixgbe_adv_tx_context_desc *TXD;
3350         struct ixgbe_tx_buf        *tx_buffer;
3351 #ifdef NET_VLAN
3352         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3353         u16 vtag = 0, eh_type;
3354 #else
3355         u16 eh_type;
3356         u32 type_tucmd_mlhl = 0;
3357 #endif
3358         u32 mss_l4len_idx = 0, len;
3359         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3360         struct ether_vlan_header *eh;
3361 #ifdef INET6
3362         struct ip6_hdr *ip6;
3363 #endif
3364 #ifdef INET
3365         struct ip *ip;
3366 #endif
3367         struct tcphdr *th;
3368
3369
3370         /*
3371          * Determine where frame payload starts.
3372          * Jump over vlan headers if already present
3373          */
3374         eh = mtod(mp, struct ether_vlan_header *);
3375         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3376                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3377                 eh_type = eh->evl_proto;
3378         } else {
3379                 ehdrlen = ETHER_HDR_LEN;
3380                 eh_type = eh->evl_encap_proto;
3381         }
3382
3383         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3384         len = ehdrlen + sizeof(struct tcphdr);
3385         switch (ntohs(eh_type)) {
3386 #ifdef INET6
3387         case ETHERTYPE_IPV6:
3388                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3389                         return FALSE;
3390                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3391                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3392                 if (ip6->ip6_nxt != IPPROTO_TCP)
3393                         return FALSE;
3394                 ip_hlen = sizeof(struct ip6_hdr);
3395                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3396                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3397                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3398                 break;
3399 #endif
3400 #ifdef INET
3401         case ETHERTYPE_IP:
3402                 if (mp->m_len < len + sizeof(struct ip))
3403                         return FALSE;
3404                 ip = (struct ip *)(mp->m_data + ehdrlen);
3405                 if (ip->ip_p != IPPROTO_TCP)
3406                         return FALSE;
3407                 ip->ip_sum = 0;
3408                 ip_hlen = ip->ip_hl << 2;
3409                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3410                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3411                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3412                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3413                 /* Tell transmit desc to also do IPv4 checksum. */
3414                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3415                 break;
3416 #endif
3417         default:
3418                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3419                     __func__, ntohs(eh_type));
3420                 break;
3421         }
3422
3423         ctxd = txr->next_avail_desc;
3424         tx_buffer = &txr->tx_buffers[ctxd];
3425         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3426
3427         tcp_hlen = th->th_off << 2;
3428
3429         /* This is used in the transmit desc in encap */
3430         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3431
3432         /* VLAN MACLEN IPLEN */
3433 #ifdef NET_VLAN
3434         if (mp->m_flags & M_VLANTAG) {
3435                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3436                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3437         }
3438
3439         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3440         vlan_macip_lens |= ip_hlen;
3441         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3442 #endif
3443
3444         /* ADV DTYPE TUCMD */
3445         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3446         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3447         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3448
3449         /* MSS L4LEN IDX */
3450         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3451         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3452         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3453
3454         TXD->seqnum_seed = htole32(0);
3455         tx_buffer->m_head = NULL;
3456         tx_buffer->eop_index = -1;
3457
3458         if (++ctxd == adapter->num_tx_desc)
3459                 ctxd = 0;
3460
3461         txr->tx_avail--;
3462         txr->next_avail_desc = ctxd;
3463         return TRUE;
3464 }
3465 #endif
3466
3467 #ifdef IXGBE_FDIR
3468 /*
3469 ** This routine parses packet headers so that Flow
3470 ** Director can make a hashed filter table entry 
3471 ** allowing traffic flows to be identified and kept
3472 ** on the same cpu.  This would be a performance
3473 ** hit, but we only do it at IXGBE_FDIR_RATE of
3474 ** packets.
3475 */
3476 static void
3477 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3478 {
3479         struct adapter                  *adapter = txr->adapter;
3480         struct ix_queue                 *que;
3481         struct ip                       *ip;
3482         struct tcphdr                   *th;
3483         struct udphdr                   *uh;
3484         struct ether_vlan_header        *eh;
3485         union ixgbe_atr_hash_dword      input = {.dword = 0}; 
3486         union ixgbe_atr_hash_dword      common = {.dword = 0}; 
3487         int                             ehdrlen, ip_hlen;
3488         u16                             etype;
3489
3490         eh = mtod(mp, struct ether_vlan_header *);
3491         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3492                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3493                 etype = eh->evl_proto;
3494         } else {
3495                 ehdrlen = ETHER_HDR_LEN;
3496                 etype = eh->evl_encap_proto;
3497         }
3498
3499         /* Only handling IPv4 */
3500         if (etype != htons(ETHERTYPE_IP))
3501                 return;
3502
3503         ip = (struct ip *)(mp->m_data + ehdrlen);
3504         ip_hlen = ip->ip_hl << 2;
3505
3506         /* check if we're UDP or TCP */
3507         switch (ip->ip_p) {
3508         case IPPROTO_TCP:
3509                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3510                 /* src and dst are inverted */
3511                 common.port.dst ^= th->th_sport;
3512                 common.port.src ^= th->th_dport;
3513                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3514                 break;
3515         case IPPROTO_UDP:
3516                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3517                 /* src and dst are inverted */
3518                 common.port.dst ^= uh->uh_sport;
3519                 common.port.src ^= uh->uh_dport;
3520                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3521                 break;
3522         default:
3523                 return;
3524         }
3525
3526         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3527         if (mp->m_pkthdr.ether_vtag)
3528                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3529         else
3530                 common.flex_bytes ^= etype;
3531         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3532
3533         que = &adapter->queues[txr->me];
3534         /*
3535         ** This assumes the Rx queue and Tx
3536         ** queue are bound to the same CPU
3537         */
3538         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3539             input, common, que->msix);
3540 }
3541 #endif /* IXGBE_FDIR */
3542
3543 /**********************************************************************
3544  *
3545  *  Examine each tx_buffer in the used queue. If the hardware is done
3546  *  processing the packet then free associated resources. The
3547  *  tx_buffer is put back on the free queue.
3548  *
3549  **********************************************************************/
3550 static bool
3551 ixgbe_txeof(struct tx_ring *txr)
3552 {
3553         struct adapter  *adapter = txr->adapter;
3554         struct ifnet    *ifp = adapter->ifp;
3555         u32     first, last, done, processed;
3556         struct ixgbe_tx_buf *tx_buffer;
3557         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3558
3559         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3560
3561 #ifdef DEV_NETMAP
3562         if (ifp->if_capenable & IFCAP_NETMAP) {
3563                 struct netmap_adapter *na = NA(ifp);
3564                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3565
3566                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3567
3568                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3569                     BUS_DMASYNC_POSTREAD);
3570                 /*
3571                  * In netmap mode, all the work is done in the context
3572                  * of the client thread. Interrupt handlers only wake up
3573                  * clients, which may be sleeping on individual rings
3574                  * or on a global resource for all rings.
3575                  * To implement tx interrupt mitigation, we wake up the client
3576                  * thread roughly every half ring, even if the NIC interrupts
3577                  * more frequently. This is implemented as follows:
3578                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3579                  *   the slot that should wake up the thread (nkr_num_slots
3580                  *   means the user thread should not be woken up);
3581                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3582                  *   or the slot has the DD bit set.
3583                  *
3584                  * When the driver has separate locks, we need to
3585                  * release and re-acquire txlock to avoid deadlocks.
3586                  * XXX see if we can find a better way.
3587                  */
3588                 if (!netmap_mitigate ||
3589                     (kring->nr_kflags < kring->nkr_num_slots &&
3590                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3591                         kring->nr_kflags = kring->nkr_num_slots;
3592                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3593                         IXGBE_TX_UNLOCK(txr);
3594                         IXGBE_CORE_LOCK(adapter);
3595                         selwakeuppri(&na->tx_si, PI_NET);
3596                         IXGBE_CORE_UNLOCK(adapter);
3597                         IXGBE_TX_LOCK(txr);
3598                 }
3599                 return FALSE;
3600         }
3601 #endif /* DEV_NETMAP */
3602
3603         if (txr->tx_avail == adapter->num_tx_desc) {
3604                 txr->queue_status = IXGBE_QUEUE_IDLE;
3605                 return FALSE;
3606         }
3607
3608         processed = 0;
3609         first = txr->next_to_clean;
3610         tx_buffer = &txr->tx_buffers[first];
3611         /* For cleanup we just use legacy struct */
3612         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3613         last = tx_buffer->eop_index;
3614         if (last == -1)
3615                 return FALSE;
3616         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3617
3618         /*
3619         ** Get the index of the first descriptor
3620         ** BEYOND the EOP and call that 'done'.
3621         ** I do this so the comparison in the
3622         ** inner while loop below can be simple
3623         */
3624         if (++last == adapter->num_tx_desc) last = 0;
3625         done = last;
3626
3627         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3628             BUS_DMASYNC_POSTREAD);
3629         /*
3630         ** Only the EOP descriptor of a packet now has the DD
3631         ** bit set, this is what we look for...
3632         */
3633         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3634                 /* We clean the range of the packet */
3635                 while (first != done) {
3636                         tx_desc->upper.data = 0;
3637                         tx_desc->lower.data = 0;
3638                         tx_desc->buffer_addr = 0;
3639                         ++txr->tx_avail;
3640                         ++processed;
3641
3642                         if (tx_buffer->m_head) {
3643                                 txr->bytes +=
3644                                     tx_buffer->m_head->m_pkthdr.len;
3645                                 bus_dmamap_sync(txr->txtag,
3646                                     tx_buffer->map,
3647                                     BUS_DMASYNC_POSTWRITE);
3648                                 bus_dmamap_unload(txr->txtag,
3649                                     tx_buffer->map);
3650                                 m_freem(tx_buffer->m_head);
3651                                 tx_buffer->m_head = NULL;
3652                                 tx_buffer->map = NULL;
3653                         }
3654                         tx_buffer->eop_index = -1;
3655                         txr->watchdog_time = ticks;
3656
3657                         if (++first == adapter->num_tx_desc)
3658                                 first = 0;
3659
3660                         tx_buffer = &txr->tx_buffers[first];
3661                         tx_desc =
3662                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3663                 }
3664                 ++txr->packets;
3665                 ++ifp->if_opackets;
3666                 /* See if there is more work now */
3667                 last = tx_buffer->eop_index;
3668                 if (last != -1) {
3669                         eop_desc =
3670                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3671                         /* Get next done point */
3672                         if (++last == adapter->num_tx_desc) last = 0;
3673                         done = last;
3674                 } else
3675                         break;
3676         }
3677         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3678             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3679
3680         txr->next_to_clean = first;
3681
3682         /*
3683         ** Watchdog calculation, we know there's
3684         ** work outstanding or the first return
3685         ** would have been taken, so none processed
3686         ** for too long indicates a hang.
3687         */
3688         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3689                 txr->queue_status = IXGBE_QUEUE_HUNG;
3690
3691         /* With a minimum free clear the depleted state bit.  */
3692         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3693                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3694
3695         if (txr->tx_avail == adapter->num_tx_desc) {
3696                 txr->queue_status = IXGBE_QUEUE_IDLE;
3697                 return (FALSE);
3698         }
3699
3700         return TRUE;
3701 }
3702
3703 /*********************************************************************
3704  *
3705  *  Refresh mbuf buffers for RX descriptor rings
3706  *   - now keeps its own state so discards due to resource
3707  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3708  *     it just returns, keeping its placeholder, thus it can simply
3709  *     be recalled to try again.
3710  *
3711  **********************************************************************/
3712 static void
3713 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3714 {
3715         struct adapter          *adapter = rxr->adapter;
3716         bus_dma_segment_t       hseg[1];
3717         bus_dma_segment_t       pseg[1];
3718         struct ixgbe_rx_buf     *rxbuf;
3719         struct mbuf             *mh, *mp;
3720         int                     i, j, nsegs, error;
3721         bool                    refreshed = FALSE;
3722
3723         i = j = rxr->next_to_refresh;
3724         /* Control the loop with one beyond */
3725         if (++j == adapter->num_rx_desc)
3726                 j = 0;
3727
3728         while (j != limit) {
3729                 rxbuf = &rxr->rx_buffers[i];
3730                 if (rxr->hdr_split == FALSE)
3731                         goto no_split;
3732
3733                 if (rxbuf->m_head == NULL) {
3734                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3735                         if (mh == NULL)
3736                                 goto update;
3737                 } else
3738                         mh = rxbuf->m_head;
3739
3740                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3741                 mh->m_len = MHLEN;
3742                 mh->m_flags |= M_PKTHDR;
3743                 /* Get the memory mapping */
3744                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3745                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3746                 if (error != 0) {
3747                         kprintf("Refresh mbufs: hdr dmamap load"
3748                             " failure - %d\n", error);
3749                         m_free(mh);
3750                         rxbuf->m_head = NULL;
3751                         goto update;
3752                 }
3753                 rxbuf->m_head = mh;
3754                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3755                     BUS_DMASYNC_PREREAD);
3756                 rxr->rx_base[i].read.hdr_addr =
3757                     htole64(hseg[0].ds_addr);
3758
3759 no_split:
3760                 if (rxbuf->m_pack == NULL) {
3761                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3762                             M_PKTHDR, adapter->rx_mbuf_sz);
3763                         if (mp == NULL)
3764                                 goto update;
3765                 } else
3766                         mp = rxbuf->m_pack;
3767
3768                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3769                 /* Get the memory mapping */
3770                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3771                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3772                 if (error != 0) {
3773                         kprintf("Refresh mbufs: payload dmamap load"
3774                             " failure - %d\n", error);
3775                         m_free(mp);
3776                         rxbuf->m_pack = NULL;
3777                         goto update;
3778                 }
3779                 rxbuf->m_pack = mp;
3780                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3781                     BUS_DMASYNC_PREREAD);
3782                 rxr->rx_base[i].read.pkt_addr =
3783                     htole64(pseg[0].ds_addr);
3784
3785                 refreshed = TRUE;
3786                 /* Next is precalculated */
3787                 i = j;
3788                 rxr->next_to_refresh = i;
3789                 if (++j == adapter->num_rx_desc)
3790                         j = 0;
3791         }
3792 update:
3793         if (refreshed) /* Update hardware tail index */
3794                 IXGBE_WRITE_REG(&adapter->hw,
3795                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3796         return;
3797 }
3798
3799 /*********************************************************************
3800  *
3801  *  Allocate memory for rx_buffer structures. Since we use one
3802  *  rx_buffer per received packet, the maximum number of rx_buffer's
3803  *  that we'll need is equal to the number of receive descriptors
3804  *  that we've allocated.
3805  *
3806  **********************************************************************/
3807 static int
3808 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3809 {
3810         struct  adapter         *adapter = rxr->adapter;
3811         device_t                dev = adapter->dev;
3812         struct ixgbe_rx_buf     *rxbuf;
3813         int                     i, bsize, error;
3814
3815         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3816         if (!(rxr->rx_buffers =
3817             (struct ixgbe_rx_buf *) kmalloc(bsize,
3818             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3819                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3820                 error = ENOMEM;
3821                 goto fail;
3822         }
3823
3824         if ((error = bus_dma_tag_create(NULL,   /* parent */
3825                                    1, 0,        /* alignment, bounds */
3826                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3827                                    BUS_SPACE_MAXADDR,   /* highaddr */
3828                                    NULL, NULL,          /* filter, filterarg */
3829                                    MSIZE,               /* maxsize */
3830                                    1,                   /* nsegments */
3831                                    MSIZE,               /* maxsegsize */
3832                                    0,                   /* flags */
3833                                    &rxr->htag))) {
3834                 device_printf(dev, "Unable to create RX DMA tag\n");
3835                 goto fail;
3836         }
3837
3838         if ((error = bus_dma_tag_create(NULL,   /* parent */
3839                                    1, 0,        /* alignment, bounds */
3840                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3841                                    BUS_SPACE_MAXADDR,   /* highaddr */
3842                                    NULL, NULL,          /* filter, filterarg */
3843                                    MJUM16BYTES,         /* maxsize */
3844                                    1,                   /* nsegments */
3845                                    MJUM16BYTES,         /* maxsegsize */
3846                                    0,                   /* flags */
3847                                    &rxr->ptag))) {
3848                 device_printf(dev, "Unable to create RX DMA tag\n");
3849                 goto fail;
3850         }
3851
3852         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3853                 rxbuf = &rxr->rx_buffers[i];
3854                 error = bus_dmamap_create(rxr->htag,
3855                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3856                 if (error) {
3857                         device_printf(dev, "Unable to create RX head map\n");
3858                         goto fail;
3859                 }
3860                 error = bus_dmamap_create(rxr->ptag,
3861                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3862                 if (error) {
3863                         device_printf(dev, "Unable to create RX pkt map\n");
3864                         goto fail;
3865                 }
3866         }
3867
3868         return (0);
3869
3870 fail:
3871         /* Frees all, but can handle partial completion */
3872         ixgbe_free_receive_structures(adapter);
3873         return (error);
3874 }
3875
3876 /*
3877 ** Used to detect a descriptor that has
3878 ** been merged by Hardware RSC.
3879 */
3880 static inline u32
3881 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3882 {
3883         return (le32toh(rx->wb.lower.lo_dword.data) &
3884             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3885 }
3886