94aad2023def012ba260543725d71c37f798cbbc
[dragonfly.git] / sys / dev / netif / ixgbe / ixgbe.c
1 /******************************************************************************
2
3   Copyright (c) 2001-2012, Intel Corporation 
4   All rights reserved.
5   
6   Redistribution and use in source and binary forms, with or without 
7   modification, are permitted provided that the following conditions are met:
8   
9    1. Redistributions of source code must retain the above copyright notice, 
10       this list of conditions and the following disclaimer.
11   
12    2. Redistributions in binary form must reproduce the above copyright 
13       notice, this list of conditions and the following disclaimer in the 
14       documentation and/or other materials provided with the distribution.
15   
16    3. Neither the name of the Intel Corporation nor the names of its 
17       contributors may be used to endorse or promote products derived from 
18       this software without specific prior written permission.
19   
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
34
35 #include "opt_inet.h"
36 #include "opt_inet6.h"
37
38 #include "ixgbe.h"
39
40 /*********************************************************************
41  *  Set this to one to display debug statistics
42  *********************************************************************/
43 int             ixgbe_display_debug_stats = 0;
44
45 /*********************************************************************
46  *  Driver version
47  *********************************************************************/
48 char ixgbe_driver_version[] = "2.4.8";
49
50 /*********************************************************************
51  *  PCI Device ID Table
52  *
53  *  Used by probe to select devices to load on
54  *  Last field stores an index into ixgbe_strings
55  *  Last entry must be all 0s
56  *
57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58  *********************************************************************/
59
60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
61 {
62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86         /* required last entry */
87         {0, 0, 0, 0, 0}
88 };
89
90 /*********************************************************************
91  *  Table of branding strings
92  *********************************************************************/
93
94 static char    *ixgbe_strings[] = {
95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
96 };
97
98 /*********************************************************************
99  *  Function prototypes
100  *********************************************************************/
101 static int      ixgbe_probe(device_t);
102 static int      ixgbe_attach(device_t);
103 static int      ixgbe_detach(device_t);
104 static int      ixgbe_shutdown(device_t);
105 static void     ixgbe_start(struct ifnet *);
106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107 #if 0 /* __FreeBSD_version >= 800000 */
108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
109 static int      ixgbe_mq_start_locked(struct ifnet *,
110                     struct tx_ring *, struct mbuf *);
111 static void     ixgbe_qflush(struct ifnet *);
112 #endif
113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114 static void     ixgbe_init(void *);
115 static void     ixgbe_init_locked(struct adapter *);
116 static void     ixgbe_stop(void *);
117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118 static int      ixgbe_media_change(struct ifnet *);
119 static void     ixgbe_identify_hardware(struct adapter *);
120 static int      ixgbe_allocate_pci_resources(struct adapter *);
121 static int      ixgbe_allocate_msix(struct adapter *);
122 static int      ixgbe_allocate_legacy(struct adapter *);
123 static int      ixgbe_allocate_queues(struct adapter *);
124 #if 0   /* HAVE_MSIX */
125 static int      ixgbe_setup_msix(struct adapter *);
126 #endif
127 static void     ixgbe_free_pci_resources(struct adapter *);
128 static void     ixgbe_local_timer(void *);
129 static int      ixgbe_setup_interface(device_t, struct adapter *);
130 static void     ixgbe_config_link(struct adapter *);
131
132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
133 static int      ixgbe_setup_transmit_structures(struct adapter *);
134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
135 static void     ixgbe_initialize_transmit_units(struct adapter *);
136 static void     ixgbe_free_transmit_structures(struct adapter *);
137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
138
139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
140 static int      ixgbe_setup_receive_structures(struct adapter *);
141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
142 static void     ixgbe_initialize_receive_units(struct adapter *);
143 static void     ixgbe_free_receive_structures(struct adapter *);
144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
145 #if 0   /* NET_LRO */
146 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
147 #endif
148
149 static void     ixgbe_enable_intr(struct adapter *);
150 static void     ixgbe_disable_intr(struct adapter *);
151 static void     ixgbe_update_stats_counters(struct adapter *);
152 static bool     ixgbe_txeof(struct tx_ring *);
153 static bool     ixgbe_rxeof(struct ix_queue *, int);
154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
155 static void     ixgbe_set_promisc(struct adapter *);
156 static void     ixgbe_set_multi(struct adapter *);
157 static void     ixgbe_update_link_status(struct adapter *);
158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
164                     struct ixgbe_dma_alloc *, int);
165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
166 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
167                     const char *, int *, int);
168 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
169 #if 0   /* NET_TSO */
170 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
171 #endif
172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173 static void     ixgbe_configure_ivars(struct adapter *);
174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
175
176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
179
180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
181
182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
184                     struct mbuf *, u32);
185
186 /* Support for pluggable optic modules */
187 static bool     ixgbe_sfp_probe(struct adapter *);
188 static void     ixgbe_setup_optics(struct adapter *);
189
190 /* Legacy (single vector interrupt handler */
191 static void     ixgbe_legacy_irq(void *);
192
193 /* The MSI/X Interrupt handlers */
194 static void     ixgbe_msix_que(void *);
195 static void     ixgbe_msix_link(void *);
196
197 /* Deferred interrupt tasklets */
198 static void     ixgbe_handle_que(void *, int);
199 static void     ixgbe_handle_link(void *, int);
200 static void     ixgbe_handle_msf(void *, int);
201 static void     ixgbe_handle_mod(void *, int);
202
203 #ifdef IXGBE_FDIR
204 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
205 static void     ixgbe_reinit_fdir(void *, int);
206 #endif
207
208 /*********************************************************************
209  *  FreeBSD Device Interface Entry Points
210  *********************************************************************/
211
212 static device_method_t ixgbe_methods[] = {
213         /* Device interface */
214         DEVMETHOD(device_probe, ixgbe_probe),
215         DEVMETHOD(device_attach, ixgbe_attach),
216         DEVMETHOD(device_detach, ixgbe_detach),
217         DEVMETHOD(device_shutdown, ixgbe_shutdown),
218         {0, 0}
219 };
220
221 static driver_t ixgbe_driver = {
222         "ix", ixgbe_methods, sizeof(struct adapter),
223 };
224
225 devclass_t ixgbe_devclass;
226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
227
228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230
231 /*
232 ** TUNEABLE PARAMETERS:
233 */
234
235 /*
236 ** AIM: Adaptive Interrupt Moderation
237 ** which means that the interrupt rate
238 ** is varied over time based on the
239 ** traffic for that interrupt vector
240 */
241 static int ixgbe_enable_aim = TRUE;
242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
243
244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
246
247 /* How many packets rxeof tries to clean at a time */
248 static int ixgbe_rx_process_limit = 128;
249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
250
251 /*
252 ** Smart speed setting, default to on
253 ** this only works as a compile option
254 ** right now as its during attach, set
255 ** this to 'ixgbe_smart_speed_off' to
256 ** disable.
257 */
258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
259
260 static int ixgbe_msi_enable = 1;
261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
262
263 /*
264  * MSIX should be the default for best performance,
265  * but this allows it to be forced off for testing.
266  */
267 static int ixgbe_enable_msix = 1;
268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
269
270 /*
271  * Header split: this causes the hardware to DMA
272  * the header into a separate mbuf from the payload,
273  * it can be a performance win in some workloads, but
274  * in others it actually hurts, its off by default. 
275  */
276 static int ixgbe_header_split = FALSE;
277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
278
279 /*
280  * Number of Queues, can be set to 0,
281  * it then autoconfigures based on the
282  * number of cpus with a max of 8. This
283  * can be overriden manually here.
284  */
285 static int ixgbe_num_queues = 0;
286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
287
288 /*
289 ** Number of TX descriptors per ring,
290 ** setting higher than RX as this seems
291 ** the better performing choice.
292 */
293 static int ixgbe_txd = PERFORM_TXD;
294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
295
296 /* Number of RX descriptors per ring */
297 static int ixgbe_rxd = PERFORM_RXD;
298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
299
300 /* Keep running tab on them for sanity check */
301 static int ixgbe_total_ports;
302
303 #ifdef IXGBE_FDIR
304 /*
305 ** For Flow Director: this is the
306 ** number of TX packets we sample
307 ** for the filter pool, this means
308 ** every 20th packet will be probed.
309 **
310 ** This feature can be disabled by 
311 ** setting this to 0.
312 */
313 static int atr_sample_rate = 20;
314 /* 
315 ** Flow Director actually 'steals'
316 ** part of the packet buffer as its
317 ** filter pool, this variable controls
318 ** how much it uses:
319 **  0 = 64K, 1 = 128K, 2 = 256K
320 */
321 static int fdir_pballoc = 1;
322 #endif
323
324 #ifdef DEV_NETMAP
325 /*
326  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
327  * be a reference on how to implement netmap support in a driver.
328  * Additional comments are in ixgbe_netmap.h .
329  *
330  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
331  * that extend the standard driver.
332  */
333 #include <dev/netmap/ixgbe_netmap.h>
334 #endif /* DEV_NETMAP */
335
336 /*********************************************************************
337  *  Device identification routine
338  *
339  *  ixgbe_probe determines if the driver should be loaded on
340  *  adapter based on PCI vendor/device id of the adapter.
341  *
342  *  return BUS_PROBE_DEFAULT on success, positive on failure
343  *********************************************************************/
344
345 static int
346 ixgbe_probe(device_t dev)
347 {
348         ixgbe_vendor_info_t *ent;
349
350         u16     pci_vendor_id = 0;
351         u16     pci_device_id = 0;
352         u16     pci_subvendor_id = 0;
353         u16     pci_subdevice_id = 0;
354         char    adapter_name[256];
355
356         INIT_DEBUGOUT("ixgbe_probe: begin");
357
358         pci_vendor_id = pci_get_vendor(dev);
359         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
360                 return (ENXIO);
361
362         pci_device_id = pci_get_device(dev);
363         pci_subvendor_id = pci_get_subvendor(dev);
364         pci_subdevice_id = pci_get_subdevice(dev);
365
366         ent = ixgbe_vendor_info_array;
367         while (ent->vendor_id != 0) {
368                 if ((pci_vendor_id == ent->vendor_id) &&
369                     (pci_device_id == ent->device_id) &&
370
371                     ((pci_subvendor_id == ent->subvendor_id) ||
372                      (ent->subvendor_id == 0)) &&
373
374                     ((pci_subdevice_id == ent->subdevice_id) ||
375                      (ent->subdevice_id == 0))) {
376                         ksprintf(adapter_name, "%s, Version - %s",
377                                 ixgbe_strings[ent->index],
378                                 ixgbe_driver_version);
379                         device_set_desc_copy(dev, adapter_name);
380                         ++ixgbe_total_ports;
381                         return (BUS_PROBE_DEFAULT);
382                 }
383                 ent++;
384         }
385         return (ENXIO);
386 }
387
388 /*********************************************************************
389  *  Device initialization routine
390  *
391  *  The attach entry point is called when the driver is being loaded.
392  *  This routine identifies the type of hardware, allocates all resources
393  *  and initializes the hardware.
394  *
395  *  return 0 on success, positive on failure
396  *********************************************************************/
397
398 static int
399 ixgbe_attach(device_t dev)
400 {
401         struct adapter *adapter;
402         struct ixgbe_hw *hw;
403         int             error = 0;
404         u16             csum;
405         u32             ctrl_ext;
406
407         INIT_DEBUGOUT("ixgbe_attach: begin");
408
409         if (resource_disabled("ixgbe", device_get_unit(dev))) {
410                 device_printf(dev, "Disabled by device hint\n");
411                 return (ENXIO);
412         }
413
414         /* Allocate, clear, and link in our adapter structure */
415         adapter = device_get_softc(dev);
416         adapter->dev = adapter->osdep.dev = dev;
417         hw = &adapter->hw;
418
419         /* Core Lock Init*/
420         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
421
422         /* SYSCTL APIs */
423
424         sysctl_ctx_init(&adapter->sysctl_ctx);
425         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
426             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
427             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
428         if (adapter->sysctl_tree == NULL) {
429                 device_printf(adapter->dev, "can't add sysctl node\n");
430                 return (EINVAL);
431         }
432         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
433                         SYSCTL_CHILDREN(adapter->sysctl_tree),
434                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
435                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
436
437         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
438                         SYSCTL_CHILDREN(adapter->sysctl_tree),
439                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
440                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
441
442         /*
443         ** Allow a kind of speed control by forcing the autoneg
444         ** advertised speed list to only a certain value, this
445         ** supports 1G on 82599 devices, and 100Mb on x540.
446         */
447         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
448                         SYSCTL_CHILDREN(adapter->sysctl_tree),
449                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
450                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
451
452         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
453                         SYSCTL_CHILDREN(adapter->sysctl_tree),
454                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
455                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
456
457         /* Set up the timer callout */
458         callout_init_mp(&adapter->timer);
459
460         /* Determine hardware revision */
461         ixgbe_identify_hardware(adapter);
462
463         /* Do base PCI setup - map BAR0 */
464         if (ixgbe_allocate_pci_resources(adapter)) {
465                 device_printf(dev, "Allocation of PCI resources failed\n");
466                 error = ENXIO;
467                 goto err_out;
468         }
469
470         /* Do descriptor calc and sanity checks */
471         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
472             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
473                 device_printf(dev, "TXD config issue, using default!\n");
474                 adapter->num_tx_desc = DEFAULT_TXD;
475         } else
476                 adapter->num_tx_desc = ixgbe_txd;
477
478         /*
479         ** With many RX rings it is easy to exceed the
480         ** system mbuf allocation. Tuning nmbclusters
481         ** can alleviate this.
482         */
483         if (nmbclusters > 0 ) {
484                 int s;
485                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
486                 if (s > nmbclusters) {
487                         device_printf(dev, "RX Descriptors exceed "
488                             "system mbuf max, using default instead!\n");
489                         ixgbe_rxd = DEFAULT_RXD;
490                 }
491         }
492
493         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
494             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
495                 device_printf(dev, "RXD config issue, using default!\n");
496                 adapter->num_rx_desc = DEFAULT_RXD;
497         } else
498                 adapter->num_rx_desc = ixgbe_rxd;
499
500         /* Allocate our TX/RX Queues */
501         if (ixgbe_allocate_queues(adapter)) {
502                 error = ENOMEM;
503                 goto err_out;
504         }
505
506         /* Allocate multicast array memory. */
507         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
508             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
509         if (adapter->mta == NULL) {
510                 device_printf(dev, "Can not allocate multicast setup array\n");
511                 error = ENOMEM;
512                 goto err_late;
513         }
514
515         /* Initialize the shared code */
516         error = ixgbe_init_shared_code(hw);
517         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
518                 /*
519                 ** No optics in this port, set up
520                 ** so the timer routine will probe 
521                 ** for later insertion.
522                 */
523                 adapter->sfp_probe = TRUE;
524                 error = 0;
525         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
526                 device_printf(dev,"Unsupported SFP+ module detected!\n");
527                 error = EIO;
528                 goto err_late;
529         } else if (error) {
530                 device_printf(dev,"Unable to initialize the shared code\n");
531                 error = EIO;
532                 goto err_late;
533         }
534
535         /* Make sure we have a good EEPROM before we read from it */
536         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
537                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
538                 error = EIO;
539                 goto err_late;
540         }
541
542         error = ixgbe_init_hw(hw);
543         switch (error) {
544         case IXGBE_ERR_EEPROM_VERSION:
545                 device_printf(dev, "This device is a pre-production adapter/"
546                     "LOM.  Please be aware there may be issues associated "
547                     "with your hardware.\n If you are experiencing problems "
548                     "please contact your Intel or hardware representative "
549                     "who provided you with this hardware.\n");
550                 break;
551         case IXGBE_ERR_SFP_NOT_SUPPORTED:
552                 device_printf(dev,"Unsupported SFP+ Module\n");
553                 error = EIO;
554                 device_printf(dev,"Hardware Initialization Failure\n");
555                 goto err_late;
556         case IXGBE_ERR_SFP_NOT_PRESENT:
557                 device_printf(dev,"No SFP+ Module found\n");
558                 /* falls thru */
559         default:
560                 break;
561         }
562
563         /* Detect and set physical type */
564         ixgbe_setup_optics(adapter);
565
566         if ((adapter->msix > 1) && (ixgbe_enable_msix))
567                 error = ixgbe_allocate_msix(adapter); 
568         else
569                 error = ixgbe_allocate_legacy(adapter); 
570         if (error) 
571                 goto err_late;
572
573         /* Setup OS specific network interface */
574         if (ixgbe_setup_interface(dev, adapter) != 0)
575                 goto err_late;
576
577         /* Sysctl for limiting the amount of work done in the taskqueue */
578         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
579             "max number of rx packets to process", &adapter->rx_process_limit,
580             ixgbe_rx_process_limit);
581
582         /* Initialize statistics */
583         ixgbe_update_stats_counters(adapter);
584
585         /* Register for VLAN events */
586         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
587             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
588         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
589             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
590
591         /* Print PCIE bus type/speed/width info */
592         ixgbe_get_bus_info(hw);
593         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
594             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
595             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
596             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
597             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
598             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
599             ("Unknown"));
600
601         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
602             (hw->bus.speed == ixgbe_bus_speed_2500)) {
603                 device_printf(dev, "PCI-Express bandwidth available"
604                     " for this card\n     is not sufficient for"
605                     " optimal performance.\n");
606                 device_printf(dev, "For optimal performance a x8 "
607                     "PCIE, or x4 PCIE 2 slot is required.\n");
608         }
609
610         /* let hardware know driver is loaded */
611         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
612         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
613         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
614
615         ixgbe_add_hw_stats(adapter);
616
617 #ifdef DEV_NETMAP
618         ixgbe_netmap_attach(adapter);
619 #endif /* DEV_NETMAP */
620         INIT_DEBUGOUT("ixgbe_attach: end");
621         return (0);
622 err_late:
623         ixgbe_free_transmit_structures(adapter);
624         ixgbe_free_receive_structures(adapter);
625 err_out:
626         if (adapter->ifp != NULL)
627                 if_free(adapter->ifp);
628         ixgbe_free_pci_resources(adapter);
629         kfree(adapter->mta, M_DEVBUF);
630         return (error);
631
632 }
633
634 /*********************************************************************
635  *  Device removal routine
636  *
637  *  The detach entry point is called when the driver is being removed.
638  *  This routine stops the adapter and deallocates all the resources
639  *  that were allocated for driver operation.
640  *
641  *  return 0 on success, positive on failure
642  *********************************************************************/
643
644 static int
645 ixgbe_detach(device_t dev)
646 {
647         struct adapter *adapter = device_get_softc(dev);
648         struct ix_queue *que = adapter->queues;
649         u32     ctrl_ext;
650
651         INIT_DEBUGOUT("ixgbe_detach: begin");
652
653 #ifdef NET_VLAN
654         /* Make sure VLANS are not using driver */
655         if (adapter->ifp->if_vlantrunk != NULL) {
656                 device_printf(dev,"Vlan in use, detach first\n");
657                 return (EBUSY);
658         }
659 #endif
660
661         IXGBE_CORE_LOCK(adapter);
662         ixgbe_stop(adapter);
663         IXGBE_CORE_UNLOCK(adapter);
664
665         for (int i = 0; i < adapter->num_queues; i++, que++) {
666                 if (que->tq) {
667                         taskqueue_drain(que->tq, &que->que_task);
668                         taskqueue_free(que->tq);
669                 }
670         }
671
672         /* Drain the Link queue */
673         if (adapter->tq) {
674                 taskqueue_drain(adapter->tq, &adapter->link_task);
675                 taskqueue_drain(adapter->tq, &adapter->mod_task);
676                 taskqueue_drain(adapter->tq, &adapter->msf_task);
677 #ifdef IXGBE_FDIR
678                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
679 #endif
680                 taskqueue_free(adapter->tq);
681         }
682
683         /* let hardware know driver is unloading */
684         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
685         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
686         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
687
688         /* Unregister VLAN events */
689 #ifdef NET_VLAN
690         if (adapter->vlan_attach != NULL)
691                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
692         if (adapter->vlan_detach != NULL)
693                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
694 #endif
695
696         ether_ifdetach(adapter->ifp);
697         callout_stop(&adapter->timer);
698 #ifdef DEV_NETMAP
699         netmap_detach(adapter->ifp);
700 #endif /* DEV_NETMAP */
701         ixgbe_free_pci_resources(adapter);
702         bus_generic_detach(dev);
703         if_free(adapter->ifp);
704
705         ixgbe_free_transmit_structures(adapter);
706         ixgbe_free_receive_structures(adapter);
707         kfree(adapter->mta, M_DEVBUF);
708         sysctl_ctx_free(&adapter->sysctl_ctx);
709         
710         IXGBE_CORE_LOCK_DESTROY(adapter);
711         return (0);
712 }
713
714 /*********************************************************************
715  *
716  *  Shutdown entry point
717  *
718  **********************************************************************/
719
720 static int
721 ixgbe_shutdown(device_t dev)
722 {
723         struct adapter *adapter = device_get_softc(dev);
724         IXGBE_CORE_LOCK(adapter);
725         ixgbe_stop(adapter);
726         IXGBE_CORE_UNLOCK(adapter);
727         return (0);
728 }
729
730
731 /*********************************************************************
732  *  Transmit entry point
733  *
734  *  ixgbe_start is called by the stack to initiate a transmit.
735  *  The driver will remain in this routine as long as there are
736  *  packets to transmit and transmit resources are available.
737  *  In case resources are not available stack is notified and
738  *  the packet is requeued.
739  **********************************************************************/
740
741 static void
742 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
743 {
744         struct mbuf    *m_head;
745         struct adapter *adapter = txr->adapter;
746
747         IXGBE_TX_LOCK_ASSERT(txr);
748
749         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
750                 return;
751         if (!adapter->link_active)
752                 return;
753
754         while (!ifq_is_empty(&ifp->if_snd)) {
755                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
756                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
757                         break;
758                 }
759
760                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
761                 if (m_head == NULL)
762                         break;
763
764                 if (ixgbe_xmit(txr, &m_head)) {
765 #if 0 /* XXX: prepend to an ALTQ queue ? */
766                         if (m_head != NULL)
767                                 IF_PREPEND(&ifp->if_snd, m_head);
768 #endif
769                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
770                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
771                         break;
772                 }
773                 /* Send a copy of the frame to the BPF listener */
774                 ETHER_BPF_MTAP(ifp, m_head);
775
776                 /* Set watchdog on */
777                 txr->watchdog_time = ticks;
778                 txr->queue_status = IXGBE_QUEUE_WORKING;
779
780         }
781         return;
782 }
783
784 /*
785  * Legacy TX start - called by the stack, this
786  * always uses the first tx ring, and should
787  * not be used with multiqueue tx enabled.
788  */
789 static void
790 ixgbe_start(struct ifnet *ifp)
791 {
792         struct adapter *adapter = ifp->if_softc;
793         struct tx_ring  *txr = adapter->tx_rings;
794
795         if (ifp->if_flags & IFF_RUNNING) {
796                 IXGBE_TX_LOCK(txr);
797                 ixgbe_start_locked(txr, ifp);
798                 IXGBE_TX_UNLOCK(txr);
799         }
800         return;
801 }
802
803 #if 0 /* __FreeBSD_version >= 800000 */
804 /*
805 ** Multiqueue Transmit driver
806 **
807 */
808 static int
809 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
810 {
811         struct adapter  *adapter = ifp->if_softc;
812         struct ix_queue *que;
813         struct tx_ring  *txr;
814         int             i = 0, err = 0;
815
816         /* Which queue to use */
817         if ((m->m_flags & M_FLOWID) != 0)
818                 i = m->m_pkthdr.flowid % adapter->num_queues;
819         else
820                 i = curcpu % adapter->num_queues;
821
822         txr = &adapter->tx_rings[i];
823         que = &adapter->queues[i];
824
825         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
826             IXGBE_TX_TRYLOCK(txr)) {
827                 err = ixgbe_mq_start_locked(ifp, txr, m);
828                 IXGBE_TX_UNLOCK(txr);
829         } else {
830                 err = drbr_enqueue(ifp, txr->br, m);
831                 taskqueue_enqueue(que->tq, &que->que_task);
832         }
833
834         return (err);
835 }
836
837 static int
838 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
839 {
840         struct adapter  *adapter = txr->adapter;
841         struct mbuf     *next;
842         int             enqueued, err = 0;
843
844         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
845             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
846             adapter->link_active == 0) {
847                 if (m != NULL)
848                         err = drbr_enqueue(ifp, txr->br, m);
849                 return (err);
850         }
851
852         enqueued = 0;
853         if (m == NULL) {
854                 next = drbr_dequeue(ifp, txr->br);
855         } else if (drbr_needs_enqueue(ifp, txr->br)) {
856                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
857                         return (err);
858                 next = drbr_dequeue(ifp, txr->br);
859         } else
860                 next = m;
861
862         /* Process the queue */
863         while (next != NULL) {
864                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
865                         if (next != NULL)
866                                 err = drbr_enqueue(ifp, txr->br, next);
867                         break;
868                 }
869                 enqueued++;
870                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
871                 /* Send a copy of the frame to the BPF listener */
872                 ETHER_BPF_MTAP(ifp, next);
873                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
874                         break;
875                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
876                         ixgbe_txeof(txr);
877                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
878                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
879                         break;
880                 }
881                 next = drbr_dequeue(ifp, txr->br);
882         }
883
884         if (enqueued > 0) {
885                 /* Set watchdog on */
886                 txr->queue_status |= IXGBE_QUEUE_WORKING;
887                 txr->watchdog_time = ticks;
888         }
889
890         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
891                 ixgbe_txeof(txr);
892
893         return (err);
894 }
895
896 /*
897 ** Flush all ring buffers
898 */
899 static void
900 ixgbe_qflush(struct ifnet *ifp)
901 {
902         struct adapter  *adapter = ifp->if_softc;
903         struct tx_ring  *txr = adapter->tx_rings;
904         struct mbuf     *m;
905
906         for (int i = 0; i < adapter->num_queues; i++, txr++) {
907                 IXGBE_TX_LOCK(txr);
908                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
909                         m_freem(m);
910                 IXGBE_TX_UNLOCK(txr);
911         }
912         if_qflush(ifp);
913 }
914 #endif /* __FreeBSD_version >= 800000 */
915
916 /*********************************************************************
917  *  Ioctl entry point
918  *
919  *  ixgbe_ioctl is called when the user wants to configure the
920  *  interface.
921  *
922  *  return 0 on success, positive on failure
923  **********************************************************************/
924
925 static int
926 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
927 {
928         struct adapter  *adapter = ifp->if_softc;
929         struct ifreq    *ifr = (struct ifreq *) data;
930 #if defined(INET) || defined(INET6)
931         struct ifaddr *ifa = (struct ifaddr *)data;
932         bool            avoid_reset = FALSE;
933 #endif
934         int             error = 0;
935
936         switch (command) {
937
938         case SIOCSIFADDR:
939 #ifdef INET
940                 if (ifa->ifa_addr->sa_family == AF_INET)
941                         avoid_reset = TRUE;
942 #endif
943 #ifdef INET6
944                 if (ifa->ifa_addr->sa_family == AF_INET6)
945                         avoid_reset = TRUE;
946 #endif
947 #if defined(INET) || defined(INET6)
948                 /*
949                 ** Calling init results in link renegotiation,
950                 ** so we avoid doing it when possible.
951                 */
952                 if (avoid_reset) {
953                         ifp->if_flags |= IFF_UP;
954                         if (!(ifp->if_flags & IFF_RUNNING))
955                                 ixgbe_init(adapter);
956                         if (!(ifp->if_flags & IFF_NOARP))
957                                 arp_ifinit(ifp, ifa);
958                 } else
959                         error = ether_ioctl(ifp, command, data);
960 #endif
961                 break;
962         case SIOCSIFMTU:
963                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
964                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
965                         error = EINVAL;
966                 } else {
967                         IXGBE_CORE_LOCK(adapter);
968                         ifp->if_mtu = ifr->ifr_mtu;
969                         adapter->max_frame_size =
970                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
971                         ixgbe_init_locked(adapter);
972                         IXGBE_CORE_UNLOCK(adapter);
973                 }
974                 break;
975         case SIOCSIFFLAGS:
976                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
977                 IXGBE_CORE_LOCK(adapter);
978                 if (ifp->if_flags & IFF_UP) {
979                         if ((ifp->if_flags & IFF_RUNNING)) {
980                                 if ((ifp->if_flags ^ adapter->if_flags) &
981                                     (IFF_PROMISC | IFF_ALLMULTI)) {
982                                         ixgbe_set_promisc(adapter);
983                                 }
984                         } else
985                                 ixgbe_init_locked(adapter);
986                 } else
987                         if (ifp->if_flags & IFF_RUNNING)
988                                 ixgbe_stop(adapter);
989                 adapter->if_flags = ifp->if_flags;
990                 IXGBE_CORE_UNLOCK(adapter);
991                 break;
992         case SIOCADDMULTI:
993         case SIOCDELMULTI:
994                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
995                 if (ifp->if_flags & IFF_RUNNING) {
996                         IXGBE_CORE_LOCK(adapter);
997                         ixgbe_disable_intr(adapter);
998                         ixgbe_set_multi(adapter);
999                         ixgbe_enable_intr(adapter);
1000                         IXGBE_CORE_UNLOCK(adapter);
1001                 }
1002                 break;
1003         case SIOCSIFMEDIA:
1004         case SIOCGIFMEDIA:
1005                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1006                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1007                 break;
1008         case SIOCSIFCAP:
1009         {
1010                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1011                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1012                 if (mask & IFCAP_HWCSUM)
1013                         ifp->if_capenable ^= IFCAP_HWCSUM;
1014 #if 0 /* NET_TSO */
1015                 if (mask & IFCAP_TSO4)
1016                         ifp->if_capenable ^= IFCAP_TSO4;
1017                 if (mask & IFCAP_TSO6)
1018                         ifp->if_capenable ^= IFCAP_TSO6;
1019 #endif
1020 #if 0 /* NET_LRO */
1021                 if (mask & IFCAP_LRO)
1022                         ifp->if_capenable ^= IFCAP_LRO;
1023 #endif
1024                 if (mask & IFCAP_VLAN_HWTAGGING)
1025                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1026                 if (mask & IFCAP_VLAN_HWFILTER)
1027                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1028 #if 0 /* NET_TSO */
1029                 if (mask & IFCAP_VLAN_HWTSO)
1030                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1031 #endif
1032                 if (ifp->if_flags & IFF_RUNNING) {
1033                         IXGBE_CORE_LOCK(adapter);
1034                         ixgbe_init_locked(adapter);
1035                         IXGBE_CORE_UNLOCK(adapter);
1036                 }
1037 #if 0
1038                 VLAN_CAPABILITIES(ifp);
1039 #endif
1040                 break;
1041         }
1042
1043         default:
1044                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1045                 error = ether_ioctl(ifp, command, data);
1046                 break;
1047         }
1048
1049         return (error);
1050 }
1051
1052 /*********************************************************************
1053  *  Init entry point
1054  *
1055  *  This routine is used in two ways. It is used by the stack as
1056  *  init entry point in network interface structure. It is also used
1057  *  by the driver as a hw/sw initialization routine to get to a
1058  *  consistent state.
1059  *
1060  *  return 0 on success, positive on failure
1061  **********************************************************************/
1062 #define IXGBE_MHADD_MFS_SHIFT 16
1063
1064 static void
1065 ixgbe_init_locked(struct adapter *adapter)
1066 {
1067         struct ifnet   *ifp = adapter->ifp;
1068         device_t        dev = adapter->dev;
1069         struct ixgbe_hw *hw = &adapter->hw;
1070         u32             k, txdctl, mhadd, gpie;
1071         u32             rxdctl, rxctrl;
1072
1073         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1074         INIT_DEBUGOUT("ixgbe_init: begin");
1075         hw->adapter_stopped = FALSE;
1076         ixgbe_stop_adapter(hw);
1077         callout_stop(&adapter->timer);
1078
1079         /* reprogram the RAR[0] in case user changed it. */
1080         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1081
1082         /* Get the latest mac address, User can use a LAA */
1083         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1084               IXGBE_ETH_LENGTH_OF_ADDRESS);
1085         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1086         hw->addr_ctrl.rar_used_count = 1;
1087
1088         /* Set the various hardware offload abilities */
1089         ifp->if_hwassist = 0;
1090 #if 0 /* NET_TSO */
1091         if (ifp->if_capenable & IFCAP_TSO)
1092                 ifp->if_hwassist |= CSUM_TSO;
1093 #endif
1094         if (ifp->if_capenable & IFCAP_TXCSUM) {
1095                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1096 #if 0
1097                 if (hw->mac.type != ixgbe_mac_82598EB)
1098                         ifp->if_hwassist |= CSUM_SCTP;
1099 #endif
1100         }
1101
1102         /* Prepare transmit descriptors and buffers */
1103         if (ixgbe_setup_transmit_structures(adapter)) {
1104                 device_printf(dev,"Could not setup transmit structures\n");
1105                 ixgbe_stop(adapter);
1106                 return;
1107         }
1108
1109         ixgbe_init_hw(hw);
1110         ixgbe_initialize_transmit_units(adapter);
1111
1112         /* Setup Multicast table */
1113         ixgbe_set_multi(adapter);
1114
1115         /*
1116         ** Determine the correct mbuf pool
1117         ** for doing jumbo/headersplit
1118         */
1119         if (adapter->max_frame_size <= 2048)
1120                 adapter->rx_mbuf_sz = MCLBYTES;
1121         else if (adapter->max_frame_size <= 4096)
1122                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1123         else if (adapter->max_frame_size <= 9216)
1124                 adapter->rx_mbuf_sz = MJUM9BYTES;
1125         else
1126                 adapter->rx_mbuf_sz = MJUM16BYTES;
1127
1128         /* Prepare receive descriptors and buffers */
1129         if (ixgbe_setup_receive_structures(adapter)) {
1130                 device_printf(dev,"Could not setup receive structures\n");
1131                 ixgbe_stop(adapter);
1132                 return;
1133         }
1134
1135         /* Configure RX settings */
1136         ixgbe_initialize_receive_units(adapter);
1137
1138         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1139
1140         /* Enable Fan Failure Interrupt */
1141         gpie |= IXGBE_SDP1_GPIEN;
1142
1143         /* Add for Module detection */
1144         if (hw->mac.type == ixgbe_mac_82599EB)
1145                 gpie |= IXGBE_SDP2_GPIEN;
1146
1147         /* Thermal Failure Detection */
1148         if (hw->mac.type == ixgbe_mac_X540)
1149                 gpie |= IXGBE_SDP0_GPIEN;
1150
1151         if (adapter->msix > 1) {
1152                 /* Enable Enhanced MSIX mode */
1153                 gpie |= IXGBE_GPIE_MSIX_MODE;
1154                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1155                     IXGBE_GPIE_OCD;
1156         }
1157         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1158
1159         /* Set MTU size */
1160         if (ifp->if_mtu > ETHERMTU) {
1161                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1162                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1163                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1164                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1165         }
1166         
1167         /* Now enable all the queues */
1168
1169         for (int i = 0; i < adapter->num_queues; i++) {
1170                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1171                 txdctl |= IXGBE_TXDCTL_ENABLE;
1172                 /* Set WTHRESH to 8, burst writeback */
1173                 txdctl |= (8 << 16);
1174                 /*
1175                  * When the internal queue falls below PTHRESH (32),
1176                  * start prefetching as long as there are at least
1177                  * HTHRESH (1) buffers ready. The values are taken
1178                  * from the Intel linux driver 3.8.21.
1179                  * Prefetching enables tx line rate even with 1 queue.
1180                  */
1181                 txdctl |= (32 << 0) | (1 << 8);
1182                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1183         }
1184
1185         for (int i = 0; i < adapter->num_queues; i++) {
1186                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1187                 if (hw->mac.type == ixgbe_mac_82598EB) {
1188                         /*
1189                         ** PTHRESH = 21
1190                         ** HTHRESH = 4
1191                         ** WTHRESH = 8
1192                         */
1193                         rxdctl &= ~0x3FFFFF;
1194                         rxdctl |= 0x080420;
1195                 }
1196                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1197                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1198                 for (k = 0; k < 10; k++) {
1199                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1200                             IXGBE_RXDCTL_ENABLE)
1201                                 break;
1202                         else
1203                                 msec_delay(1);
1204                 }
1205                 wmb();
1206 #ifdef DEV_NETMAP
1207                 /*
1208                  * In netmap mode, we must preserve the buffers made
1209                  * available to userspace before the if_init()
1210                  * (this is true by default on the TX side, because
1211                  * init makes all buffers available to userspace).
1212                  *
1213                  * netmap_reset() and the device specific routines
1214                  * (e.g. ixgbe_setup_receive_rings()) map these
1215                  * buffers at the end of the NIC ring, so here we
1216                  * must set the RDT (tail) register to make sure
1217                  * they are not overwritten.
1218                  *
1219                  * In this driver the NIC ring starts at RDH = 0,
1220                  * RDT points to the last slot available for reception (?),
1221                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1222                  */
1223                 if (ifp->if_capenable & IFCAP_NETMAP) {
1224                         struct netmap_adapter *na = NA(adapter->ifp);
1225                         struct netmap_kring *kring = &na->rx_rings[i];
1226                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1227
1228                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1229                 } else
1230 #endif /* DEV_NETMAP */
1231                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1232         }
1233
1234         /* Set up VLAN support and filter */
1235         ixgbe_setup_vlan_hw_support(adapter);
1236
1237         /* Enable Receive engine */
1238         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1239         if (hw->mac.type == ixgbe_mac_82598EB)
1240                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1241         rxctrl |= IXGBE_RXCTRL_RXEN;
1242         ixgbe_enable_rx_dma(hw, rxctrl);
1243
1244         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1245
1246         /* Set up MSI/X routing */
1247         if (ixgbe_enable_msix)  {
1248                 ixgbe_configure_ivars(adapter);
1249                 /* Set up auto-mask */
1250                 if (hw->mac.type == ixgbe_mac_82598EB)
1251                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1252                 else {
1253                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1254                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1255                 }
1256         } else {  /* Simple settings for Legacy/MSI */
1257                 ixgbe_set_ivar(adapter, 0, 0, 0);
1258                 ixgbe_set_ivar(adapter, 0, 0, 1);
1259                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1260         }
1261
1262 #ifdef IXGBE_FDIR
1263         /* Init Flow director */
1264         if (hw->mac.type != ixgbe_mac_82598EB) {
1265                 u32 hdrm = 32 << fdir_pballoc;
1266
1267                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1268                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1269         }
1270 #endif
1271
1272         /*
1273         ** Check on any SFP devices that
1274         ** need to be kick-started
1275         */
1276         if (hw->phy.type == ixgbe_phy_none) {
1277                 int err = hw->phy.ops.identify(hw);
1278                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1279                         device_printf(dev,
1280                             "Unsupported SFP+ module type was detected.\n");
1281                         return;
1282                 }
1283         }
1284
1285         /* Set moderation on the Link interrupt */
1286         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1287
1288         /* Config/Enable Link */
1289         ixgbe_config_link(adapter);
1290
1291         /* Hardware Packet Buffer & Flow Control setup */
1292         {
1293                 u32 rxpb, frame, size, tmp;
1294
1295                 frame = adapter->max_frame_size;
1296
1297                 /* Calculate High Water */
1298                 if (hw->mac.type == ixgbe_mac_X540)
1299                         tmp = IXGBE_DV_X540(frame, frame);
1300                 else
1301                         tmp = IXGBE_DV(frame, frame);
1302                 size = IXGBE_BT2KB(tmp);
1303                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1304                 hw->fc.high_water[0] = rxpb - size;
1305
1306                 /* Now calculate Low Water */
1307                 if (hw->mac.type == ixgbe_mac_X540)
1308                         tmp = IXGBE_LOW_DV_X540(frame);
1309                 else
1310                         tmp = IXGBE_LOW_DV(frame);
1311                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1312                 
1313                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1314                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1315                 hw->fc.send_xon = TRUE;
1316         }
1317         /* Initialize the FC settings */
1318         ixgbe_start_hw(hw);
1319
1320         /* And now turn on interrupts */
1321         ixgbe_enable_intr(adapter);
1322
1323         /* Now inform the stack we're ready */
1324         ifp->if_flags |= IFF_RUNNING;
1325         ifp->if_flags &= ~IFF_OACTIVE;
1326
1327         return;
1328 }
1329
1330 static void
1331 ixgbe_init(void *arg)
1332 {
1333         struct adapter *adapter = arg;
1334
1335         IXGBE_CORE_LOCK(adapter);
1336         ixgbe_init_locked(adapter);
1337         IXGBE_CORE_UNLOCK(adapter);
1338         return;
1339 }
1340
1341
1342 /*
1343 **
1344 ** MSIX Interrupt Handlers and Tasklets
1345 **
1346 */
1347
1348 static inline void
1349 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1350 {
1351         struct ixgbe_hw *hw = &adapter->hw;
1352         u64     queue = (u64)(1 << vector);
1353         u32     mask;
1354
1355         if (hw->mac.type == ixgbe_mac_82598EB) {
1356                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1357                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1358         } else {
1359                 mask = (queue & 0xFFFFFFFF);
1360                 if (mask)
1361                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1362                 mask = (queue >> 32);
1363                 if (mask)
1364                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1365         }
1366 }
1367
1368 static inline void
1369 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1370 {
1371         struct ixgbe_hw *hw = &adapter->hw;
1372         u64     queue = (u64)(1 << vector);
1373         u32     mask;
1374
1375         if (hw->mac.type == ixgbe_mac_82598EB) {
1376                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1377                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1378         } else {
1379                 mask = (queue & 0xFFFFFFFF);
1380                 if (mask)
1381                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1382                 mask = (queue >> 32);
1383                 if (mask)
1384                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1385         }
1386 }
1387
1388 static inline void
1389 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1390 {
1391         u32 mask;
1392
1393         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1394                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1395                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1396         } else {
1397                 mask = (queues & 0xFFFFFFFF);
1398                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1399                 mask = (queues >> 32);
1400                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1401         }
1402 }
1403
1404
1405 static void
1406 ixgbe_handle_que(void *context, int pending)
1407 {
1408         struct ix_queue *que = context;
1409         struct adapter  *adapter = que->adapter;
1410         struct tx_ring  *txr = que->txr;
1411         struct ifnet    *ifp = adapter->ifp;
1412         bool            more;
1413
1414         if (ifp->if_flags & IFF_RUNNING) {
1415                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1416                 IXGBE_TX_LOCK(txr);
1417                 ixgbe_txeof(txr);
1418 #if 0 /*__FreeBSD_version >= 800000*/
1419                 if (!drbr_empty(ifp, txr->br))
1420                         ixgbe_mq_start_locked(ifp, txr, NULL);
1421 #else
1422                 if (!ifq_is_empty(&ifp->if_snd))
1423                         ixgbe_start_locked(txr, ifp);
1424 #endif
1425                 IXGBE_TX_UNLOCK(txr);
1426                 if (more) {
1427                         taskqueue_enqueue(que->tq, &que->que_task);
1428                         return;
1429                 }
1430         }
1431
1432         /* Reenable this interrupt */
1433         ixgbe_enable_queue(adapter, que->msix);
1434         return;
1435 }
1436
1437
1438 /*********************************************************************
1439  *
1440  *  Legacy Interrupt Service routine
1441  *
1442  **********************************************************************/
1443
1444 static void
1445 ixgbe_legacy_irq(void *arg)
1446 {
1447         struct ix_queue *que = arg;
1448         struct adapter  *adapter = que->adapter;
1449         struct ixgbe_hw *hw = &adapter->hw;
1450         struct          tx_ring *txr = adapter->tx_rings;
1451         bool            more_tx, more_rx;
1452         u32             reg_eicr, loop = MAX_LOOP;
1453
1454
1455         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1456
1457         ++que->irqs;
1458         if (reg_eicr == 0) {
1459                 ixgbe_enable_intr(adapter);
1460                 return;
1461         }
1462
1463         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1464
1465         IXGBE_TX_LOCK(txr);
1466         do {
1467                 more_tx = ixgbe_txeof(txr);
1468         } while (loop-- && more_tx);
1469         IXGBE_TX_UNLOCK(txr);
1470
1471         if (more_rx || more_tx)
1472                 taskqueue_enqueue(que->tq, &que->que_task);
1473
1474         /* Check for fan failure */
1475         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1476             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1477                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1478                     "REPLACE IMMEDIATELY!!\n");
1479                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1480         }
1481
1482         /* Link status change */
1483         if (reg_eicr & IXGBE_EICR_LSC)
1484                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1485
1486         ixgbe_enable_intr(adapter);
1487         return;
1488 }
1489
1490
1491 /*********************************************************************
1492  *
1493  *  MSIX Queue Interrupt Service routine
1494  *
1495  **********************************************************************/
1496 void
1497 ixgbe_msix_que(void *arg)
1498 {
1499         struct ix_queue *que = arg;
1500         struct adapter  *adapter = que->adapter;
1501         struct tx_ring  *txr = que->txr;
1502         struct rx_ring  *rxr = que->rxr;
1503         bool            more_tx, more_rx;
1504         u32             newitr = 0;
1505
1506         ixgbe_disable_queue(adapter, que->msix);
1507         ++que->irqs;
1508
1509         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1510
1511         IXGBE_TX_LOCK(txr);
1512         more_tx = ixgbe_txeof(txr);
1513         /*
1514         ** Make certain that if the stack 
1515         ** has anything queued the task gets
1516         ** scheduled to handle it.
1517         */
1518 #if 0
1519 #if __FreeBSD_version < 800000
1520         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1521 #else
1522         if (!drbr_empty(adapter->ifp, txr->br))
1523 #endif
1524 #endif
1525         if (!ifq_is_empty(&adapter->ifp->if_snd))
1526                 more_tx = 1;
1527         IXGBE_TX_UNLOCK(txr);
1528
1529         /* Do AIM now? */
1530
1531         if (ixgbe_enable_aim == FALSE)
1532                 goto no_calc;
1533         /*
1534         ** Do Adaptive Interrupt Moderation:
1535         **  - Write out last calculated setting
1536         **  - Calculate based on average size over
1537         **    the last interval.
1538         */
1539         if (que->eitr_setting)
1540                 IXGBE_WRITE_REG(&adapter->hw,
1541                     IXGBE_EITR(que->msix), que->eitr_setting);
1542  
1543         que->eitr_setting = 0;
1544
1545         /* Idle, do nothing */
1546         if ((txr->bytes == 0) && (rxr->bytes == 0))
1547                 goto no_calc;
1548                                 
1549         if ((txr->bytes) && (txr->packets))
1550                 newitr = txr->bytes/txr->packets;
1551         if ((rxr->bytes) && (rxr->packets))
1552                 newitr = max(newitr,
1553                     (rxr->bytes / rxr->packets));
1554         newitr += 24; /* account for hardware frame, crc */
1555
1556         /* set an upper boundary */
1557         newitr = min(newitr, 3000);
1558
1559         /* Be nice to the mid range */
1560         if ((newitr > 300) && (newitr < 1200))
1561                 newitr = (newitr / 3);
1562         else
1563                 newitr = (newitr / 2);
1564
1565         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1566                 newitr |= newitr << 16;
1567         else
1568                 newitr |= IXGBE_EITR_CNT_WDIS;
1569                  
1570         /* save for next interrupt */
1571         que->eitr_setting = newitr;
1572
1573         /* Reset state */
1574         txr->bytes = 0;
1575         txr->packets = 0;
1576         rxr->bytes = 0;
1577         rxr->packets = 0;
1578
1579 no_calc:
1580         if (more_tx || more_rx)
1581                 taskqueue_enqueue(que->tq, &que->que_task);
1582         else /* Reenable this interrupt */
1583                 ixgbe_enable_queue(adapter, que->msix);
1584         return;
1585 }
1586
1587
1588 static void
1589 ixgbe_msix_link(void *arg)
1590 {
1591         struct adapter  *adapter = arg;
1592         struct ixgbe_hw *hw = &adapter->hw;
1593         u32             reg_eicr;
1594
1595         ++adapter->link_irq;
1596
1597         /* First get the cause */
1598         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1599         /* Clear interrupt with write */
1600         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1601
1602         /* Link status change */
1603         if (reg_eicr & IXGBE_EICR_LSC)
1604                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1605
1606         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1607 #ifdef IXGBE_FDIR
1608                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1609                         /* This is probably overkill :) */
1610                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1611                                 return;
1612                         /* Disable the interrupt */
1613                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1614                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1615                 } else
1616 #endif
1617                 if (reg_eicr & IXGBE_EICR_ECC) {
1618                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1619                             "Please Reboot!!\n");
1620                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1621                 } else
1622
1623                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1624                         /* Clear the interrupt */
1625                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1626                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1627                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1628                         /* Clear the interrupt */
1629                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1630                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1631                 }
1632         } 
1633
1634         /* Check for fan failure */
1635         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1636             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1637                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1638                     "REPLACE IMMEDIATELY!!\n");
1639                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1640         }
1641
1642         /* Check for over temp condition */
1643         if ((hw->mac.type == ixgbe_mac_X540) &&
1644             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1645                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1646                     "PHY IS SHUT DOWN!!\n");
1647                 device_printf(adapter->dev, "System shutdown required\n");
1648                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1649         }
1650
1651         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1652         return;
1653 }
1654
1655 /*********************************************************************
1656  *
1657  *  Media Ioctl callback
1658  *
1659  *  This routine is called whenever the user queries the status of
1660  *  the interface using ifconfig.
1661  *
1662  **********************************************************************/
1663 static void
1664 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1665 {
1666         struct adapter *adapter = ifp->if_softc;
1667
1668         INIT_DEBUGOUT("ixgbe_media_status: begin");
1669         IXGBE_CORE_LOCK(adapter);
1670         ixgbe_update_link_status(adapter);
1671
1672         ifmr->ifm_status = IFM_AVALID;
1673         ifmr->ifm_active = IFM_ETHER;
1674
1675         if (!adapter->link_active) {
1676                 IXGBE_CORE_UNLOCK(adapter);
1677                 return;
1678         }
1679
1680         ifmr->ifm_status |= IFM_ACTIVE;
1681
1682         switch (adapter->link_speed) {
1683                 case IXGBE_LINK_SPEED_100_FULL:
1684                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1685                         break;
1686                 case IXGBE_LINK_SPEED_1GB_FULL:
1687                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1688                         break;
1689                 case IXGBE_LINK_SPEED_10GB_FULL:
1690                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1691                         break;
1692         }
1693
1694         IXGBE_CORE_UNLOCK(adapter);
1695
1696         return;
1697 }
1698
1699 /*********************************************************************
1700  *
1701  *  Media Ioctl callback
1702  *
1703  *  This routine is called when the user changes speed/duplex using
1704  *  media/mediopt option with ifconfig.
1705  *
1706  **********************************************************************/
1707 static int
1708 ixgbe_media_change(struct ifnet * ifp)
1709 {
1710         struct adapter *adapter = ifp->if_softc;
1711         struct ifmedia *ifm = &adapter->media;
1712
1713         INIT_DEBUGOUT("ixgbe_media_change: begin");
1714
1715         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1716                 return (EINVAL);
1717
1718         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1719         case IFM_AUTO:
1720                 adapter->hw.phy.autoneg_advertised =
1721                     IXGBE_LINK_SPEED_100_FULL |
1722                     IXGBE_LINK_SPEED_1GB_FULL |
1723                     IXGBE_LINK_SPEED_10GB_FULL;
1724                 break;
1725         default:
1726                 device_printf(adapter->dev, "Only auto media type\n");
1727                 return (EINVAL);
1728         }
1729
1730         return (0);
1731 }
1732
1733 /*********************************************************************
1734  *
1735  *  This routine maps the mbufs to tx descriptors, allowing the
1736  *  TX engine to transmit the packets. 
1737  *      - return 0 on success, positive on failure
1738  *
1739  **********************************************************************/
1740
1741 static int
1742 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1743 {
1744         struct adapter  *adapter = txr->adapter;
1745         u32             olinfo_status = 0, cmd_type_len;
1746         u32             paylen = 0;
1747         int             i, j, error, nsegs, maxsegs;
1748         int             first, last = 0;
1749         struct mbuf     *m_head;
1750         bus_dma_segment_t segs[adapter->num_segs];
1751         bus_dmamap_t    map;
1752         struct ixgbe_tx_buf *txbuf;
1753         union ixgbe_adv_tx_desc *txd = NULL;
1754
1755         m_head = *m_headp;
1756
1757         /* Basic descriptor defines */
1758         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1759             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1760
1761         if (m_head->m_flags & M_VLANTAG)
1762                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1763
1764         /*
1765          * Important to capture the first descriptor
1766          * used because it will contain the index of
1767          * the one we tell the hardware to report back
1768          */
1769         first = txr->next_avail_desc;
1770         txbuf = &txr->tx_buffers[first];
1771         map = txbuf->map;
1772
1773         /*
1774          * Map the packet for DMA.
1775          */
1776         maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1777         if (maxsegs > adapter->num_segs)
1778                 maxsegs = adapter->num_segs;
1779
1780         error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1781             segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1782         if (error) {
1783                 if (error == ENOBUFS)
1784                         adapter->mbuf_defrag_failed++;
1785                 else
1786                         adapter->no_tx_dma_setup++;
1787
1788                 m_freem(*m_headp);
1789                 *m_headp = NULL;
1790                 return (error);
1791         }
1792
1793         /* Make certain there are enough descriptors */
1794         if (nsegs > txr->tx_avail - 2) {
1795                 txr->no_desc_avail++;
1796                 error = ENOBUFS;
1797                 goto xmit_fail;
1798         }
1799         m_head = *m_headp;
1800
1801         /*
1802         ** Set up the appropriate offload context
1803         ** this becomes the first descriptor of 
1804         ** a packet.
1805         */
1806 #if 0 /* NET_TSO */
1807         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1808                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1809                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1810                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1811                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1812                         ++adapter->tso_tx;
1813                 } else
1814                         return (ENXIO);
1815         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1816 #endif
1817         if (ixgbe_tx_ctx_setup(txr, m_head))
1818                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1819
1820 #ifdef IXGBE_IEEE1588
1821         /* This is changing soon to an mtag detection */
1822         if (we detect this mbuf has a TSTAMP mtag)
1823                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1824 #endif
1825
1826 #ifdef IXGBE_FDIR
1827         /* Do the flow director magic */
1828         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1829                 ++txr->atr_count;
1830                 if (txr->atr_count >= atr_sample_rate) {
1831                         ixgbe_atr(txr, m_head);
1832                         txr->atr_count = 0;
1833                 }
1834         }
1835 #endif
1836         /* Record payload length */
1837         if (paylen == 0)
1838                 olinfo_status |= m_head->m_pkthdr.len <<
1839                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1840
1841         i = txr->next_avail_desc;
1842         for (j = 0; j < nsegs; j++) {
1843                 bus_size_t seglen;
1844                 bus_addr_t segaddr;
1845
1846                 txbuf = &txr->tx_buffers[i];
1847                 txd = &txr->tx_base[i];
1848                 seglen = segs[j].ds_len;
1849                 segaddr = htole64(segs[j].ds_addr);
1850
1851                 txd->read.buffer_addr = segaddr;
1852                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1853                     cmd_type_len |seglen);
1854                 txd->read.olinfo_status = htole32(olinfo_status);
1855                 last = i; /* descriptor that will get completion IRQ */
1856
1857                 if (++i == adapter->num_tx_desc)
1858                         i = 0;
1859
1860                 txbuf->m_head = NULL;
1861                 txbuf->eop_index = -1;
1862         }
1863
1864         txd->read.cmd_type_len |=
1865             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1866         txr->tx_avail -= nsegs;
1867         txr->next_avail_desc = i;
1868
1869         txbuf->m_head = m_head;
1870         /* Swap the dma map between the first and last descriptor */
1871         txr->tx_buffers[first].map = txbuf->map;
1872         txbuf->map = map;
1873         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1874
1875         /* Set the index of the descriptor that will be marked done */
1876         txbuf = &txr->tx_buffers[first];
1877         txbuf->eop_index = last;
1878
1879         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1880             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1881         /*
1882          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1883          * hardware that this frame is available to transmit.
1884          */
1885         ++txr->total_packets;
1886         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1887
1888         return (0);
1889
1890 xmit_fail:
1891         bus_dmamap_unload(txr->txtag, txbuf->map);
1892         return (error);
1893
1894 }
1895
1896 static void
1897 ixgbe_set_promisc(struct adapter *adapter)
1898 {
1899         u_int32_t       reg_rctl;
1900         struct ifnet   *ifp = adapter->ifp;
1901
1902         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1903         reg_rctl &= (~IXGBE_FCTRL_UPE);
1904         reg_rctl &= (~IXGBE_FCTRL_MPE);
1905         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1906
1907         if (ifp->if_flags & IFF_PROMISC) {
1908                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1909                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1910         } else if (ifp->if_flags & IFF_ALLMULTI) {
1911                 reg_rctl |= IXGBE_FCTRL_MPE;
1912                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1913                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1914         }
1915         return;
1916 }
1917
1918
1919 /*********************************************************************
1920  *  Multicast Update
1921  *
1922  *  This routine is called whenever multicast address list is updated.
1923  *
1924  **********************************************************************/
1925 #define IXGBE_RAR_ENTRIES 16
1926
1927 static void
1928 ixgbe_set_multi(struct adapter *adapter)
1929 {
1930         u32     fctrl;
1931         u8      *mta;
1932         u8      *update_ptr;
1933         struct  ifmultiaddr *ifma;
1934         int     mcnt = 0;
1935         struct ifnet   *ifp = adapter->ifp;
1936
1937         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1938
1939         mta = adapter->mta;
1940         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1941             MAX_NUM_MULTICAST_ADDRESSES);
1942
1943         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1944         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1945         if (ifp->if_flags & IFF_PROMISC)
1946                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1947         else if (ifp->if_flags & IFF_ALLMULTI) {
1948                 fctrl |= IXGBE_FCTRL_MPE;
1949                 fctrl &= ~IXGBE_FCTRL_UPE;
1950         } else
1951                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1952         
1953         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1954
1955         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1956                 if (ifma->ifma_addr->sa_family != AF_LINK)
1957                         continue;
1958                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1959                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1960                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1961                 mcnt++;
1962         }
1963
1964         update_ptr = mta;
1965         ixgbe_update_mc_addr_list(&adapter->hw,
1966             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1967
1968         return;
1969 }
1970
1971 /*
1972  * This is an iterator function now needed by the multicast
1973  * shared code. It simply feeds the shared code routine the
1974  * addresses in the array of ixgbe_set_multi() one by one.
1975  */
1976 static u8 *
1977 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1978 {
1979         u8 *addr = *update_ptr;
1980         u8 *newptr;
1981         *vmdq = 0;
1982
1983         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1984         *update_ptr = newptr;
1985         return addr;
1986 }
1987
1988
1989 /*********************************************************************
1990  *  Timer routine
1991  *
1992  *  This routine checks for link status,updates statistics,
1993  *  and runs the watchdog check.
1994  *
1995  **********************************************************************/
1996
1997 static void
1998 ixgbe_local_timer(void *arg)
1999 {
2000         struct adapter  *adapter = arg;
2001         device_t        dev = adapter->dev;
2002         struct ifnet    *ifp = adapter->ifp;
2003         struct ix_queue *que = adapter->queues;
2004         struct tx_ring  *txr = adapter->tx_rings;
2005         int             hung, busy, paused;
2006
2007         IXGBE_CORE_LOCK(adapter);
2008         hung = busy = paused = 0;
2009
2010         /* Check for pluggable optics */
2011         if (adapter->sfp_probe)
2012                 if (!ixgbe_sfp_probe(adapter))
2013                         goto out; /* Nothing to do */
2014
2015         ixgbe_update_link_status(adapter);
2016         ixgbe_update_stats_counters(adapter);
2017
2018         /*
2019          * If the interface has been paused
2020          * then don't do the watchdog check
2021          */
2022         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2023                 paused = 1;
2024
2025         /*
2026         ** Check the TX queues status
2027         **      - central locked handling of OACTIVE
2028         **      - watchdog only if all queues show hung
2029         */          
2030         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2031                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2032                     (paused == 0))
2033                         ++hung;
2034                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2035                         ++busy;
2036                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2037                         taskqueue_enqueue(que->tq, &que->que_task);
2038         }
2039         /* Only truely watchdog if all queues show hung */
2040         if (hung == adapter->num_queues)
2041                 goto watchdog;
2042         /* Only turn off the stack flow when ALL are depleted */
2043         if (busy == adapter->num_queues)
2044                 ifp->if_flags |= IFF_OACTIVE;
2045         else if ((ifp->if_flags & IFF_OACTIVE) &&
2046             (busy < adapter->num_queues))
2047                 ifp->if_flags &= ~IFF_OACTIVE;
2048
2049 out:
2050         ixgbe_rearm_queues(adapter, adapter->que_mask);
2051         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2052         IXGBE_CORE_UNLOCK(adapter);
2053         return;
2054
2055 watchdog:
2056         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2057         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2058             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2059             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2060         device_printf(dev,"TX(%d) desc avail = %d,"
2061             "Next TX to Clean = %d\n",
2062             txr->me, txr->tx_avail, txr->next_to_clean);
2063         adapter->ifp->if_flags &= ~IFF_RUNNING;
2064         adapter->watchdog_events++;
2065         ixgbe_init_locked(adapter);
2066
2067         IXGBE_CORE_UNLOCK(adapter);
2068 }
2069
2070 /*
2071 ** Note: this routine updates the OS on the link state
2072 **      the real check of the hardware only happens with
2073 **      a link interrupt.
2074 */
2075 static void
2076 ixgbe_update_link_status(struct adapter *adapter)
2077 {
2078         struct ifnet    *ifp = adapter->ifp;
2079         struct tx_ring *txr = adapter->tx_rings;
2080         device_t dev = adapter->dev;
2081
2082
2083         if (adapter->link_up){ 
2084                 if (adapter->link_active == FALSE) {
2085                         if (bootverbose)
2086                                 device_printf(dev,"Link is up %d Gbps %s \n",
2087                                     ((adapter->link_speed == 128)? 10:1),
2088                                     "Full Duplex");
2089                         adapter->link_active = TRUE;
2090                         /* Update any Flow Control changes */
2091                         ixgbe_fc_enable(&adapter->hw);
2092                         ifp->if_link_state = LINK_STATE_UP;
2093                         if_link_state_change(ifp);
2094                 }
2095         } else { /* Link down */
2096                 if (adapter->link_active == TRUE) {
2097                         if (bootverbose)
2098                                 device_printf(dev,"Link is Down\n");
2099                         ifp->if_link_state = LINK_STATE_DOWN;
2100                         if_link_state_change(ifp);
2101                         adapter->link_active = FALSE;
2102                         for (int i = 0; i < adapter->num_queues;
2103                             i++, txr++)
2104                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2105                 }
2106         }
2107
2108         return;
2109 }
2110
2111
2112 /*********************************************************************
2113  *
2114  *  This routine disables all traffic on the adapter by issuing a
2115  *  global reset on the MAC and deallocates TX/RX buffers.
2116  *
2117  **********************************************************************/
2118
2119 static void
2120 ixgbe_stop(void *arg)
2121 {
2122         struct ifnet   *ifp;
2123         struct adapter *adapter = arg;
2124         struct ixgbe_hw *hw = &adapter->hw;
2125         ifp = adapter->ifp;
2126
2127         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2128
2129         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2130         ixgbe_disable_intr(adapter);
2131         callout_stop(&adapter->timer);
2132
2133         /* Let the stack know...*/
2134         ifp->if_flags &= ~IFF_RUNNING;
2135         ifp->if_flags |= IFF_OACTIVE;
2136
2137         ixgbe_reset_hw(hw);
2138         hw->adapter_stopped = FALSE;
2139         ixgbe_stop_adapter(hw);
2140         /* Turn off the laser */
2141         if (hw->phy.multispeed_fiber)
2142                 ixgbe_disable_tx_laser(hw);
2143
2144         /* reprogram the RAR[0] in case user changed it. */
2145         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2146
2147         return;
2148 }
2149
2150
2151 /*********************************************************************
2152  *
2153  *  Determine hardware revision.
2154  *
2155  **********************************************************************/
2156 static void
2157 ixgbe_identify_hardware(struct adapter *adapter)
2158 {
2159         device_t        dev = adapter->dev;
2160         struct ixgbe_hw *hw = &adapter->hw;
2161
2162         /* Save off the information about this board */
2163         hw->vendor_id = pci_get_vendor(dev);
2164         hw->device_id = pci_get_device(dev);
2165         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2166         hw->subsystem_vendor_id =
2167             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2168         hw->subsystem_device_id =
2169             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2170
2171         /* We need this here to set the num_segs below */
2172         ixgbe_set_mac_type(hw);
2173
2174         /* Pick up the 82599 and VF settings */
2175         if (hw->mac.type != ixgbe_mac_82598EB) {
2176                 hw->phy.smart_speed = ixgbe_smart_speed;
2177                 adapter->num_segs = IXGBE_82599_SCATTER;
2178         } else
2179                 adapter->num_segs = IXGBE_82598_SCATTER;
2180
2181         return;
2182 }
2183
2184 /*********************************************************************
2185  *
2186  *  Determine optic type
2187  *
2188  **********************************************************************/
2189 static void
2190 ixgbe_setup_optics(struct adapter *adapter)
2191 {
2192         struct ixgbe_hw *hw = &adapter->hw;
2193         int             layer;
2194         
2195         layer = ixgbe_get_supported_physical_layer(hw);
2196
2197         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2198                 adapter->optics = IFM_10G_T;
2199                 return;
2200         }
2201
2202         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2203                 adapter->optics = IFM_1000_T;
2204                 return;
2205         }
2206
2207         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2208             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2209                 adapter->optics = IFM_10G_LR;
2210                 return;
2211         }
2212
2213         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2214                 adapter->optics = IFM_10G_SR;
2215                 return;
2216         }
2217
2218         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2219                 adapter->optics = IFM_10G_TWINAX;
2220                 return;
2221         }
2222
2223         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2224             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2225                 adapter->optics = IFM_10G_CX4;
2226                 return;
2227         }
2228
2229         /* If we get here just set the default */
2230         adapter->optics = IFM_ETHER | IFM_AUTO;
2231         return;
2232 }
2233
2234 /*********************************************************************
2235  *
2236  *  Setup the Legacy or MSI Interrupt handler
2237  *
2238  **********************************************************************/
2239 static int
2240 ixgbe_allocate_legacy(struct adapter *adapter)
2241 {
2242         device_t dev = adapter->dev;
2243         struct          ix_queue *que = adapter->queues;
2244         int error, rid = 0;
2245         unsigned int intr_flags;
2246
2247         /* MSI RID at 1 */
2248         if (adapter->msix == 1)
2249                 rid = 1;
2250
2251         /* Try allocating a MSI interrupt first */
2252         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2253                 &rid, &intr_flags);
2254
2255         /* We allocate a single interrupt resource */
2256         adapter->res = bus_alloc_resource_any(dev,
2257             SYS_RES_IRQ, &rid, intr_flags);
2258         if (adapter->res == NULL) {
2259                 device_printf(dev, "Unable to allocate bus resource: "
2260                     "interrupt\n");
2261                 return (ENXIO);
2262         }
2263
2264         /*
2265          * Try allocating a fast interrupt and the associated deferred
2266          * processing contexts.
2267          */
2268         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2269         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2270             taskqueue_thread_enqueue, &que->tq);
2271         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2272             device_get_nameunit(adapter->dev));
2273
2274         /* Tasklets for Link, SFP and Multispeed Fiber */
2275         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2276         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2277         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2278 #ifdef IXGBE_FDIR
2279         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2280 #endif
2281         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2282             taskqueue_thread_enqueue, &adapter->tq);
2283         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2284             device_get_nameunit(adapter->dev));
2285
2286         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2287             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2288                 device_printf(dev, "Failed to register fast interrupt "
2289                     "handler: %d\n", error);
2290                 taskqueue_free(que->tq);
2291                 taskqueue_free(adapter->tq);
2292                 que->tq = NULL;
2293                 adapter->tq = NULL;
2294                 return (error);
2295         }
2296         /* For simplicity in the handlers */
2297         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2298
2299         return (0);
2300 }
2301
2302
2303 /*********************************************************************
2304  *
2305  *  Setup MSIX Interrupt resources and handlers 
2306  *
2307  **********************************************************************/
2308 static int
2309 ixgbe_allocate_msix(struct adapter *adapter)
2310 {
2311         device_t        dev = adapter->dev;
2312         struct          ix_queue *que = adapter->queues;
2313         int             error, rid, vector = 0;
2314
2315         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2316                 rid = vector + 1;
2317                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2318                     RF_SHAREABLE | RF_ACTIVE);
2319                 if (que->res == NULL) {
2320                         device_printf(dev,"Unable to allocate"
2321                             " bus resource: que interrupt [%d]\n", vector);
2322                         return (ENXIO);
2323                 }
2324                 /* Set the handler function */
2325                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2326                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2327                 if (error) {
2328                         que->res = NULL;
2329                         device_printf(dev, "Failed to register QUE handler");
2330                         return (error);
2331                 }
2332 #if 0 /* __FreeBSD_version >= 800504 */
2333                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2334 #endif
2335                 que->msix = vector;
2336                 adapter->que_mask |= (u64)(1 << que->msix);
2337                 /*
2338                 ** Bind the msix vector, and thus the
2339                 ** ring to the corresponding cpu.
2340                 */
2341 #if 0 /* XXX */
2342                 if (adapter->num_queues > 1)
2343                         bus_bind_intr(dev, que->res, i);
2344 #endif
2345
2346                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2347                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2348                     taskqueue_thread_enqueue, &que->tq);
2349                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2350                     device_get_nameunit(adapter->dev));
2351         }
2352
2353         /* and Link */
2354         rid = vector + 1;
2355         adapter->res = bus_alloc_resource_any(dev,
2356             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2357         if (!adapter->res) {
2358                 device_printf(dev,"Unable to allocate"
2359             " bus resource: Link interrupt [%d]\n", rid);
2360                 return (ENXIO);
2361         }
2362         /* Set the link handler function */
2363         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2364             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2365         if (error) {
2366                 adapter->res = NULL;
2367                 device_printf(dev, "Failed to register LINK handler");
2368                 return (error);
2369         }
2370 #if 0 /* __FreeBSD_version >= 800504 */
2371         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2372 #endif
2373         adapter->linkvec = vector;
2374         /* Tasklets for Link, SFP and Multispeed Fiber */
2375         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2376         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2377         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2378 #ifdef IXGBE_FDIR
2379         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2380 #endif
2381         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2382             taskqueue_thread_enqueue, &adapter->tq);
2383         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2384             device_get_nameunit(adapter->dev));
2385
2386         return (0);
2387 }
2388
2389 #if 0   /* HAVE_MSIX */
2390 /*
2391  * Setup Either MSI/X or MSI
2392  */
2393 static int
2394 ixgbe_setup_msix(struct adapter *adapter)
2395 {
2396         device_t dev = adapter->dev;
2397         int rid, want, queues, msgs;
2398
2399         /* Override by tuneable */
2400         if (ixgbe_enable_msix == 0)
2401                 goto msi;
2402
2403         /* First try MSI/X */
2404         rid = PCIR_BAR(MSIX_82598_BAR);
2405         adapter->msix_mem = bus_alloc_resource_any(dev,
2406             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2407         if (!adapter->msix_mem) {
2408                 rid += 4;       /* 82599 maps in higher BAR */
2409                 adapter->msix_mem = bus_alloc_resource_any(dev,
2410                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2411         }
2412         if (!adapter->msix_mem) {
2413                 /* May not be enabled */
2414                 device_printf(adapter->dev,
2415                     "Unable to map MSIX table \n");
2416                 goto msi;
2417         }
2418
2419         msgs = pci_msix_count(dev); 
2420         if (msgs == 0) { /* system has msix disabled */
2421                 bus_release_resource(dev, SYS_RES_MEMORY,
2422                     rid, adapter->msix_mem);
2423                 adapter->msix_mem = NULL;
2424                 goto msi;
2425         }
2426
2427         /* Figure out a reasonable auto config value */
2428         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2429
2430         if (ixgbe_num_queues != 0)
2431                 queues = ixgbe_num_queues;
2432         /* Set max queues to 8 when autoconfiguring */
2433         else if ((ixgbe_num_queues == 0) && (queues > 8))
2434                 queues = 8;
2435
2436         /*
2437         ** Want one vector (RX/TX pair) per queue
2438         ** plus an additional for Link.
2439         */
2440         want = queues + 1;
2441         if (msgs >= want)
2442                 msgs = want;
2443         else {
2444                 device_printf(adapter->dev,
2445                     "MSIX Configuration Problem, "
2446                     "%d vectors but %d queues wanted!\n",
2447                     msgs, want);
2448                 return (0); /* Will go to Legacy setup */
2449         }
2450         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2451                 device_printf(adapter->dev,
2452                     "Using MSIX interrupts with %d vectors\n", msgs);
2453                 adapter->num_queues = queues;
2454                 return (msgs);
2455         }
2456 msi:
2457         msgs = pci_msi_count(dev);
2458         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2459                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2460         else
2461                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2462         return (msgs);
2463 }
2464 #endif
2465
2466
2467 static int
2468 ixgbe_allocate_pci_resources(struct adapter *adapter)
2469 {
2470         int             rid;
2471         device_t        dev = adapter->dev;
2472
2473         rid = PCIR_BAR(0);
2474         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2475             &rid, RF_ACTIVE);
2476
2477         if (!(adapter->pci_mem)) {
2478                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2479                 return (ENXIO);
2480         }
2481
2482         adapter->osdep.mem_bus_space_tag =
2483                 rman_get_bustag(adapter->pci_mem);
2484         adapter->osdep.mem_bus_space_handle =
2485                 rman_get_bushandle(adapter->pci_mem);
2486         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2487
2488         /* Legacy defaults */
2489         adapter->num_queues = 1;
2490         adapter->hw.back = &adapter->osdep;
2491
2492         /*
2493         ** Now setup MSI or MSI/X, should
2494         ** return us the number of supported
2495         ** vectors. (Will be 1 for MSI)
2496         */
2497 #if 0   /* HAVE_MSIX */
2498         adapter->msix = ixgbe_setup_msix(adapter);
2499 #endif
2500         return (0);
2501 }
2502
2503 static void
2504 ixgbe_free_pci_resources(struct adapter * adapter)
2505 {
2506         struct          ix_queue *que = adapter->queues;
2507         device_t        dev = adapter->dev;
2508         int             rid, memrid;
2509
2510         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2511                 memrid = PCIR_BAR(MSIX_82598_BAR);
2512         else
2513                 memrid = PCIR_BAR(MSIX_82599_BAR);
2514
2515         /*
2516         ** There is a slight possibility of a failure mode
2517         ** in attach that will result in entering this function
2518         ** before interrupt resources have been initialized, and
2519         ** in that case we do not want to execute the loops below
2520         ** We can detect this reliably by the state of the adapter
2521         ** res pointer.
2522         */
2523         if (adapter->res == NULL)
2524                 goto mem;
2525
2526         /*
2527         **  Release all msix queue resources:
2528         */
2529         for (int i = 0; i < adapter->num_queues; i++, que++) {
2530                 rid = que->msix + 1;
2531                 if (que->tag != NULL) {
2532                         bus_teardown_intr(dev, que->res, que->tag);
2533                         que->tag = NULL;
2534                 }
2535                 if (que->res != NULL)
2536                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2537         }
2538
2539
2540         /* Clean the Legacy or Link interrupt last */
2541         if (adapter->linkvec) /* we are doing MSIX */
2542                 rid = adapter->linkvec + 1;
2543         else
2544                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2545
2546         if (adapter->tag != NULL) {
2547                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2548                 adapter->tag = NULL;
2549         }
2550         if (adapter->res != NULL)
2551                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2552         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2553                 pci_release_msi(adapter->dev);
2554
2555 mem:
2556         if (adapter->msix)
2557                 pci_release_msi(dev);
2558
2559         if (adapter->msix_mem != NULL)
2560                 bus_release_resource(dev, SYS_RES_MEMORY,
2561                     memrid, adapter->msix_mem);
2562
2563         if (adapter->pci_mem != NULL)
2564                 bus_release_resource(dev, SYS_RES_MEMORY,
2565                     PCIR_BAR(0), adapter->pci_mem);
2566
2567         return;
2568 }
2569
2570 /*********************************************************************
2571  *
2572  *  Setup networking device structure and register an interface.
2573  *
2574  **********************************************************************/
2575 static int
2576 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2577 {
2578         struct ixgbe_hw *hw = &adapter->hw;
2579         struct ifnet   *ifp;
2580
2581         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2582
2583         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2584         if (ifp == NULL) {
2585                 device_printf(dev, "can not allocate ifnet structure\n");
2586                 return (-1);
2587         }
2588         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2589         ifp->if_baudrate = 1000000000;
2590         ifp->if_init = ixgbe_init;
2591         ifp->if_softc = adapter;
2592         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2593         ifp->if_ioctl = ixgbe_ioctl;
2594         ifp->if_start = ixgbe_start;
2595 #if 0 /* __FreeBSD_version >= 800000 */
2596         ifp->if_transmit = ixgbe_mq_start;
2597         ifp->if_qflush = ixgbe_qflush;
2598 #endif
2599         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2600
2601         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2602
2603         adapter->max_frame_size =
2604             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2605
2606         /*
2607          * Tell the upper layer(s) we support long frames.
2608          */
2609         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2610
2611 #if 0 /* NET_TSO */
2612         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2613 #endif
2614         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2615         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2616         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2617 #if 0 /* NET_TSO */
2618                              |  IFCAP_VLAN_HWTSO
2619 #endif
2620                              |  IFCAP_VLAN_MTU;
2621         ifp->if_capenable = ifp->if_capabilities;
2622
2623         /* Don't enable LRO by default */
2624 #if 0 /* NET_LRO */
2625         ifp->if_capabilities |= IFCAP_LRO;
2626 #endif
2627
2628         /*
2629         ** Don't turn this on by default, if vlans are
2630         ** created on another pseudo device (eg. lagg)
2631         ** then vlan events are not passed thru, breaking
2632         ** operation, but with HW FILTER off it works. If
2633         ** using vlans directly on the ixgbe driver you can
2634         ** enable this and get full hardware tag filtering.
2635         */
2636         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2637
2638         /*
2639          * Specify the media types supported by this adapter and register
2640          * callbacks to update media and link information
2641          */
2642         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2643                      ixgbe_media_status);
2644         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2645         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2646         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2647                 ifmedia_add(&adapter->media,
2648                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2649                 ifmedia_add(&adapter->media,
2650                     IFM_ETHER | IFM_1000_T, 0, NULL);
2651         }
2652         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2653         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2654
2655         return (0);
2656 }
2657
2658 static void
2659 ixgbe_config_link(struct adapter *adapter)
2660 {
2661         struct ixgbe_hw *hw = &adapter->hw;
2662         u32     autoneg, err = 0;
2663         bool    sfp, negotiate;
2664
2665         sfp = ixgbe_is_sfp(hw);
2666
2667         if (sfp) { 
2668                 if (hw->phy.multispeed_fiber) {
2669                         hw->mac.ops.setup_sfp(hw);
2670                         ixgbe_enable_tx_laser(hw);
2671                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2672                 } else
2673                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2674         } else {
2675                 if (hw->mac.ops.check_link)
2676                         err = ixgbe_check_link(hw, &autoneg,
2677                             &adapter->link_up, FALSE);
2678                 if (err)
2679                         goto out;
2680                 autoneg = hw->phy.autoneg_advertised;
2681                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2682                         err  = hw->mac.ops.get_link_capabilities(hw,
2683                             &autoneg, &negotiate);
2684                 if (err)
2685                         goto out;
2686                 if (hw->mac.ops.setup_link)
2687                         err = hw->mac.ops.setup_link(hw, autoneg,
2688                             negotiate, adapter->link_up);
2689         }
2690 out:
2691         return;
2692 }
2693
2694 /********************************************************************
2695  * Manage DMA'able memory.
2696  *******************************************************************/
2697 static void
2698 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2699 {
2700         if (error)
2701                 return;
2702         *(bus_addr_t *) arg = segs->ds_addr;
2703         return;
2704 }
2705
2706 static int
2707 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2708                 struct ixgbe_dma_alloc *dma, int mapflags)
2709 {
2710         device_t dev = adapter->dev;
2711         int             r;
2712
2713         r = bus_dma_tag_create(NULL,    /* parent */
2714                                DBA_ALIGN, 0,    /* alignment, bounds */
2715                                BUS_SPACE_MAXADDR,       /* lowaddr */
2716                                BUS_SPACE_MAXADDR,       /* highaddr */
2717                                NULL, NULL,      /* filter, filterarg */
2718                                size,    /* maxsize */
2719                                1,       /* nsegments */
2720                                size,    /* maxsegsize */
2721                                BUS_DMA_ALLOCNOW,        /* flags */
2722                                &dma->dma_tag);
2723         if (r != 0) {
2724                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2725                        "error %u\n", r);
2726                 goto fail_0;
2727         }
2728         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2729                              BUS_DMA_NOWAIT, &dma->dma_map);
2730         if (r != 0) {
2731                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2732                        "error %u\n", r);
2733                 goto fail_1;
2734         }
2735         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2736                             size,
2737                             ixgbe_dmamap_cb,
2738                             &dma->dma_paddr,
2739                             mapflags | BUS_DMA_NOWAIT);
2740         if (r != 0) {
2741                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2742                        "error %u\n", r);
2743                 goto fail_2;
2744         }
2745         dma->dma_size = size;
2746         return (0);
2747 fail_2:
2748         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2749 fail_1:
2750         bus_dma_tag_destroy(dma->dma_tag);
2751 fail_0:
2752         dma->dma_map = NULL;
2753         dma->dma_tag = NULL;
2754         return (r);
2755 }
2756
2757 static void
2758 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2759 {
2760         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2761             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2762         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2763         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2764         bus_dma_tag_destroy(dma->dma_tag);
2765 }
2766
2767
2768 /*********************************************************************
2769  *
2770  *  Allocate memory for the transmit and receive rings, and then
2771  *  the descriptors associated with each, called only once at attach.
2772  *
2773  **********************************************************************/
2774 static int
2775 ixgbe_allocate_queues(struct adapter *adapter)
2776 {
2777         device_t        dev = adapter->dev;
2778         struct ix_queue *que;
2779         struct tx_ring  *txr;
2780         struct rx_ring  *rxr;
2781         int rsize, tsize, error = IXGBE_SUCCESS;
2782         int txconf = 0, rxconf = 0;
2783
2784         /* First allocate the top level queue structs */
2785         if (!(adapter->queues =
2786             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2787             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2788                 device_printf(dev, "Unable to allocate queue memory\n");
2789                 error = ENOMEM;
2790                 goto fail;
2791         }
2792
2793         /* First allocate the TX ring struct memory */
2794         if (!(adapter->tx_rings =
2795             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2796             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2797                 device_printf(dev, "Unable to allocate TX ring memory\n");
2798                 error = ENOMEM;
2799                 goto tx_fail;
2800         }
2801
2802         /* Next allocate the RX */
2803         if (!(adapter->rx_rings =
2804             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2805             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2806                 device_printf(dev, "Unable to allocate RX ring memory\n");
2807                 error = ENOMEM;
2808                 goto rx_fail;
2809         }
2810
2811         /* For the ring itself */
2812         tsize = roundup2(adapter->num_tx_desc *
2813             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2814
2815         /*
2816          * Now set up the TX queues, txconf is needed to handle the
2817          * possibility that things fail midcourse and we need to
2818          * undo memory gracefully
2819          */ 
2820         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2821                 /* Set up some basics */
2822                 txr = &adapter->tx_rings[i];
2823                 txr->adapter = adapter;
2824                 txr->me = i;
2825
2826                 /* Initialize the TX side lock */
2827                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2828                     device_get_nameunit(dev), txr->me);
2829                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2830
2831                 if (ixgbe_dma_malloc(adapter, tsize,
2832                         &txr->txdma, BUS_DMA_NOWAIT)) {
2833                         device_printf(dev,
2834                             "Unable to allocate TX Descriptor memory\n");
2835                         error = ENOMEM;
2836                         goto err_tx_desc;
2837                 }
2838                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2839                 bzero((void *)txr->tx_base, tsize);
2840
2841                 /* Now allocate transmit buffers for the ring */
2842                 if (ixgbe_allocate_transmit_buffers(txr)) {
2843                         device_printf(dev,
2844                             "Critical Failure setting up transmit buffers\n");
2845                         error = ENOMEM;
2846                         goto err_tx_desc;
2847                 }
2848 #if 0 /* __FreeBSD_version >= 800000 */
2849                 /* Allocate a buf ring */
2850                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2851                     M_WAITOK, &txr->tx_mtx);
2852                 if (txr->br == NULL) {
2853                         device_printf(dev,
2854                             "Critical Failure setting up buf ring\n");
2855                         error = ENOMEM;
2856                         goto err_tx_desc;
2857                 }
2858 #endif
2859         }
2860
2861         /*
2862          * Next the RX queues...
2863          */ 
2864         rsize = roundup2(adapter->num_rx_desc *
2865             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2866         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2867                 rxr = &adapter->rx_rings[i];
2868                 /* Set up some basics */
2869                 rxr->adapter = adapter;
2870                 rxr->me = i;
2871
2872                 /* Initialize the RX side lock */
2873                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2874                     device_get_nameunit(dev), rxr->me);
2875                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2876
2877                 if (ixgbe_dma_malloc(adapter, rsize,
2878                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2879                         device_printf(dev,
2880                             "Unable to allocate RxDescriptor memory\n");
2881                         error = ENOMEM;
2882                         goto err_rx_desc;
2883                 }
2884                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2885                 bzero((void *)rxr->rx_base, rsize);
2886
2887                 /* Allocate receive buffers for the ring*/
2888                 if (ixgbe_allocate_receive_buffers(rxr)) {
2889                         device_printf(dev,
2890                             "Critical Failure setting up receive buffers\n");
2891                         error = ENOMEM;
2892                         goto err_rx_desc;
2893                 }
2894         }
2895
2896         /*
2897         ** Finally set up the queue holding structs
2898         */
2899         for (int i = 0; i < adapter->num_queues; i++) {
2900                 que = &adapter->queues[i];
2901                 que->adapter = adapter;
2902                 que->txr = &adapter->tx_rings[i];
2903                 que->rxr = &adapter->rx_rings[i];
2904         }
2905
2906         return (0);
2907
2908 err_rx_desc:
2909         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2910                 ixgbe_dma_free(adapter, &rxr->rxdma);
2911 err_tx_desc:
2912         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2913                 ixgbe_dma_free(adapter, &txr->txdma);
2914         kfree(adapter->rx_rings, M_DEVBUF);
2915 rx_fail:
2916         kfree(adapter->tx_rings, M_DEVBUF);
2917 tx_fail:
2918         kfree(adapter->queues, M_DEVBUF);
2919 fail:
2920         return (error);
2921 }
2922
2923 /*********************************************************************
2924  *
2925  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2926  *  the information needed to transmit a packet on the wire. This is
2927  *  called only once at attach, setup is done every reset.
2928  *
2929  **********************************************************************/
2930 static int
2931 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2932 {
2933         struct adapter *adapter = txr->adapter;
2934         device_t dev = adapter->dev;
2935         struct ixgbe_tx_buf *txbuf;
2936         int error, i;
2937
2938         /*
2939          * Setup DMA descriptor areas.
2940          */
2941         if ((error = bus_dma_tag_create(
2942                                NULL,    /* parent */
2943                                1, 0,            /* alignment, bounds */
2944                                BUS_SPACE_MAXADDR,       /* lowaddr */
2945                                BUS_SPACE_MAXADDR,       /* highaddr */
2946                                NULL, NULL,              /* filter, filterarg */
2947                                IXGBE_TSO_SIZE,          /* maxsize */
2948                                adapter->num_segs,       /* nsegments */
2949                                PAGE_SIZE,               /* maxsegsize */
2950                                0,                       /* flags */
2951                                &txr->txtag))) {
2952                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2953                 goto fail;
2954         }
2955
2956         if (!(txr->tx_buffers =
2957             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2958             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2959                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2960                 error = ENOMEM;
2961                 goto fail;
2962         }
2963
2964         /* Create the descriptor buffer dma maps */
2965         txbuf = txr->tx_buffers;
2966         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2967                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2968                 if (error != 0) {
2969                         device_printf(dev, "Unable to create TX DMA map\n");
2970                         goto fail;
2971                 }
2972         }
2973
2974         return 0;
2975 fail:
2976         /* We free all, it handles case where we are in the middle */
2977         ixgbe_free_transmit_structures(adapter);
2978         return (error);
2979 }
2980
2981 /*********************************************************************
2982  *
2983  *  Initialize a transmit ring.
2984  *
2985  **********************************************************************/
2986 static void
2987 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2988 {
2989         struct adapter *adapter = txr->adapter;
2990         struct ixgbe_tx_buf *txbuf;
2991         int i;
2992 #ifdef DEV_NETMAP
2993         struct netmap_adapter *na = NA(adapter->ifp);
2994         struct netmap_slot *slot;
2995 #endif /* DEV_NETMAP */
2996
2997         /* Clear the old ring contents */
2998         IXGBE_TX_LOCK(txr);
2999 #ifdef DEV_NETMAP
3000         /*
3001          * (under lock): if in netmap mode, do some consistency
3002          * checks and set slot to entry 0 of the netmap ring.
3003          */
3004         slot = netmap_reset(na, NR_TX, txr->me, 0);
3005 #endif /* DEV_NETMAP */
3006         bzero((void *)txr->tx_base,
3007               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3008         /* Reset indices */
3009         txr->next_avail_desc = 0;
3010         txr->next_to_clean = 0;
3011
3012         /* Free any existing tx buffers. */
3013         txbuf = txr->tx_buffers;
3014         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3015                 if (txbuf->m_head != NULL) {
3016                         bus_dmamap_sync(txr->txtag, txbuf->map,
3017                             BUS_DMASYNC_POSTWRITE);
3018                         bus_dmamap_unload(txr->txtag, txbuf->map);
3019                         m_freem(txbuf->m_head);
3020                         txbuf->m_head = NULL;
3021                 }
3022 #ifdef DEV_NETMAP
3023                 /*
3024                  * In netmap mode, set the map for the packet buffer.
3025                  * NOTE: Some drivers (not this one) also need to set
3026                  * the physical buffer address in the NIC ring.
3027                  * Slots in the netmap ring (indexed by "si") are
3028                  * kring->nkr_hwofs positions "ahead" wrt the
3029                  * corresponding slot in the NIC ring. In some drivers
3030                  * (not here) nkr_hwofs can be negative. Function
3031                  * netmap_idx_n2k() handles wraparounds properly.
3032                  */
3033                 if (slot) {
3034                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3035                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3036                 }
3037 #endif /* DEV_NETMAP */
3038                 /* Clear the EOP index */
3039                 txbuf->eop_index = -1;
3040         }
3041
3042 #ifdef IXGBE_FDIR
3043         /* Set the rate at which we sample packets */
3044         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3045                 txr->atr_sample = atr_sample_rate;
3046 #endif
3047
3048         /* Set number of descriptors available */
3049         txr->tx_avail = adapter->num_tx_desc;
3050
3051         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3052             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3053         IXGBE_TX_UNLOCK(txr);
3054 }
3055
3056 /*********************************************************************
3057  *
3058  *  Initialize all transmit rings.
3059  *
3060  **********************************************************************/
3061 static int
3062 ixgbe_setup_transmit_structures(struct adapter *adapter)
3063 {
3064         struct tx_ring *txr = adapter->tx_rings;
3065
3066         for (int i = 0; i < adapter->num_queues; i++, txr++)
3067                 ixgbe_setup_transmit_ring(txr);
3068
3069         return (0);
3070 }
3071
3072 /*********************************************************************
3073  *
3074  *  Enable transmit unit.
3075  *
3076  **********************************************************************/
3077 static void
3078 ixgbe_initialize_transmit_units(struct adapter *adapter)
3079 {
3080         struct tx_ring  *txr = adapter->tx_rings;
3081         struct ixgbe_hw *hw = &adapter->hw;
3082
3083         /* Setup the Base and Length of the Tx Descriptor Ring */
3084
3085         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3086                 u64     tdba = txr->txdma.dma_paddr;
3087                 u32     txctrl;
3088
3089                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3090                        (tdba & 0x00000000ffffffffULL));
3091                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3092                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3093                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3094
3095                 /* Setup the HW Tx Head and Tail descriptor pointers */
3096                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3097                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3098
3099                 /* Setup Transmit Descriptor Cmd Settings */
3100                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3101                 txr->queue_status = IXGBE_QUEUE_IDLE;
3102
3103                 /* Disable Head Writeback */
3104                 switch (hw->mac.type) {
3105                 case ixgbe_mac_82598EB:
3106                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3107                         break;
3108                 case ixgbe_mac_82599EB:
3109                 case ixgbe_mac_X540:
3110                 default:
3111                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3112                         break;
3113                 }
3114                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3115                 switch (hw->mac.type) {
3116                 case ixgbe_mac_82598EB:
3117                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3118                         break;
3119                 case ixgbe_mac_82599EB:
3120                 case ixgbe_mac_X540:
3121                 default:
3122                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3123                         break;
3124                 }
3125
3126         }
3127
3128         if (hw->mac.type != ixgbe_mac_82598EB) {
3129                 u32 dmatxctl, rttdcs;
3130                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3131                 dmatxctl |= IXGBE_DMATXCTL_TE;
3132                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3133                 /* Disable arbiter to set MTQC */
3134                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3135                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3136                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3137                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3138                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3139                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3140         }
3141
3142         return;
3143 }
3144
3145 /*********************************************************************
3146  *
3147  *  Free all transmit rings.
3148  *
3149  **********************************************************************/
3150 static void
3151 ixgbe_free_transmit_structures(struct adapter *adapter)
3152 {
3153         struct tx_ring *txr = adapter->tx_rings;
3154
3155         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3156                 IXGBE_TX_LOCK(txr);
3157                 ixgbe_free_transmit_buffers(txr);
3158                 ixgbe_dma_free(adapter, &txr->txdma);
3159                 IXGBE_TX_UNLOCK(txr);
3160                 IXGBE_TX_LOCK_DESTROY(txr);
3161         }
3162         kfree(adapter->tx_rings, M_DEVBUF);
3163 }
3164
3165 /*********************************************************************
3166  *
3167  *  Free transmit ring related data structures.
3168  *
3169  **********************************************************************/
3170 static void
3171 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3172 {
3173         struct adapter *adapter = txr->adapter;
3174         struct ixgbe_tx_buf *tx_buffer;
3175         int             i;
3176
3177         INIT_DEBUGOUT("free_transmit_ring: begin");
3178
3179         if (txr->tx_buffers == NULL)
3180                 return;
3181
3182         tx_buffer = txr->tx_buffers;
3183         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3184                 if (tx_buffer->m_head != NULL) {
3185                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3186                             BUS_DMASYNC_POSTWRITE);
3187                         bus_dmamap_unload(txr->txtag,
3188                             tx_buffer->map);
3189                         m_freem(tx_buffer->m_head);
3190                         tx_buffer->m_head = NULL;
3191                         if (tx_buffer->map != NULL) {
3192                                 bus_dmamap_destroy(txr->txtag,
3193                                     tx_buffer->map);
3194                                 tx_buffer->map = NULL;
3195                         }
3196                 } else if (tx_buffer->map != NULL) {
3197                         bus_dmamap_unload(txr->txtag,
3198                             tx_buffer->map);
3199                         bus_dmamap_destroy(txr->txtag,
3200                             tx_buffer->map);
3201                         tx_buffer->map = NULL;
3202                 }
3203         }
3204 #if 0 /* __FreeBSD_version >= 800000 */
3205         if (txr->br != NULL)
3206                 buf_ring_free(txr->br, M_DEVBUF);
3207 #endif
3208         if (txr->tx_buffers != NULL) {
3209                 kfree(txr->tx_buffers, M_DEVBUF);
3210                 txr->tx_buffers = NULL;
3211         }
3212         if (txr->txtag != NULL) {
3213                 bus_dma_tag_destroy(txr->txtag);
3214                 txr->txtag = NULL;
3215         }
3216         return;
3217 }
3218
3219 /*********************************************************************
3220  *
3221  *  Advanced Context Descriptor setup for VLAN or CSUM
3222  *
3223  **********************************************************************/
3224
3225 static bool
3226 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3227 {
3228         struct adapter *adapter = txr->adapter;
3229         struct ixgbe_adv_tx_context_desc *TXD;
3230         struct ixgbe_tx_buf        *tx_buffer;
3231         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3232         struct ether_vlan_header *eh;
3233         struct ip *ip;
3234         struct ip6_hdr *ip6;
3235         int  ehdrlen, ip_hlen = 0;
3236         u16     etype;
3237         u8      ipproto = 0;
3238         bool    offload = TRUE;
3239         int ctxd = txr->next_avail_desc;
3240 #ifdef NET_VLAN
3241         u16 vtag = 0;
3242 #endif
3243
3244
3245         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3246                 offload = FALSE;
3247
3248         tx_buffer = &txr->tx_buffers[ctxd];
3249         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3250
3251         /*
3252         ** In advanced descriptors the vlan tag must 
3253         ** be placed into the descriptor itself.
3254         */
3255 #ifdef NET_VLAN
3256         if (mp->m_flags & M_VLANTAG) {
3257                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3258                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3259         } else if (offload == FALSE)
3260                 return FALSE;
3261 #endif
3262
3263         /*
3264          * Determine where frame payload starts.
3265          * Jump over vlan headers if already present,
3266          * helpful for QinQ too.
3267          */
3268         eh = mtod(mp, struct ether_vlan_header *);
3269         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3270                 etype = ntohs(eh->evl_proto);
3271                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3272         } else {
3273                 etype = ntohs(eh->evl_encap_proto);
3274                 ehdrlen = ETHER_HDR_LEN;
3275         }
3276
3277         /* Set the ether header length */
3278         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3279
3280         switch (etype) {
3281                 case ETHERTYPE_IP:
3282                         ip = (struct ip *)(mp->m_data + ehdrlen);
3283                         ip_hlen = ip->ip_hl << 2;
3284                         ipproto = ip->ip_p;
3285                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3286                         break;
3287                 case ETHERTYPE_IPV6:
3288                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3289                         ip_hlen = sizeof(struct ip6_hdr);
3290                         /* XXX-BZ this will go badly in case of ext hdrs. */
3291                         ipproto = ip6->ip6_nxt;
3292                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3293                         break;
3294                 default:
3295                         offload = FALSE;
3296                         break;
3297         }
3298
3299         vlan_macip_lens |= ip_hlen;
3300         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3301
3302         switch (ipproto) {
3303                 case IPPROTO_TCP:
3304                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3305                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3306                         break;
3307
3308                 case IPPROTO_UDP:
3309                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3310                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3311                         break;
3312
3313 #if 0
3314                 case IPPROTO_SCTP:
3315                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3316                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3317                         break;
3318 #endif
3319                 default:
3320                         offload = FALSE;
3321                         break;
3322         }
3323
3324         /* Now copy bits into descriptor */
3325         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3326         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3327         TXD->seqnum_seed = htole32(0);
3328         TXD->mss_l4len_idx = htole32(0);
3329
3330         tx_buffer->m_head = NULL;
3331         tx_buffer->eop_index = -1;
3332
3333         /* We've consumed the first desc, adjust counters */
3334         if (++ctxd == adapter->num_tx_desc)
3335                 ctxd = 0;
3336         txr->next_avail_desc = ctxd;
3337         --txr->tx_avail;
3338
3339         return (offload);
3340 }
3341
3342 /**********************************************************************
3343  *
3344  *  Setup work for hardware segmentation offload (TSO) on
3345  *  adapters using advanced tx descriptors
3346  *
3347  **********************************************************************/
3348 #if 0   /* NET_TSO */
3349 static bool
3350 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3351     u32 *olinfo_status)
3352 {
3353         struct adapter *adapter = txr->adapter;
3354         struct ixgbe_adv_tx_context_desc *TXD;
3355         struct ixgbe_tx_buf        *tx_buffer;
3356 #ifdef NET_VLAN
3357         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3358         u16 vtag = 0, eh_type;
3359 #else
3360         u16 eh_type;
3361         u32 type_tucmd_mlhl = 0;
3362 #endif
3363         u32 mss_l4len_idx = 0, len;
3364         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3365         struct ether_vlan_header *eh;
3366 #ifdef INET6
3367         struct ip6_hdr *ip6;
3368 #endif
3369 #ifdef INET
3370         struct ip *ip;
3371 #endif
3372         struct tcphdr *th;
3373
3374
3375         /*
3376          * Determine where frame payload starts.
3377          * Jump over vlan headers if already present
3378          */
3379         eh = mtod(mp, struct ether_vlan_header *);
3380         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3381                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3382                 eh_type = eh->evl_proto;
3383         } else {
3384                 ehdrlen = ETHER_HDR_LEN;
3385                 eh_type = eh->evl_encap_proto;
3386         }
3387
3388         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3389         len = ehdrlen + sizeof(struct tcphdr);
3390         switch (ntohs(eh_type)) {
3391 #ifdef INET6
3392         case ETHERTYPE_IPV6:
3393                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3394                         return FALSE;
3395                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3396                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3397                 if (ip6->ip6_nxt != IPPROTO_TCP)
3398                         return FALSE;
3399                 ip_hlen = sizeof(struct ip6_hdr);
3400                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3401                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3402                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3403                 break;
3404 #endif
3405 #ifdef INET
3406         case ETHERTYPE_IP:
3407                 if (mp->m_len < len + sizeof(struct ip))
3408                         return FALSE;
3409                 ip = (struct ip *)(mp->m_data + ehdrlen);
3410                 if (ip->ip_p != IPPROTO_TCP)
3411                         return FALSE;
3412                 ip->ip_sum = 0;
3413                 ip_hlen = ip->ip_hl << 2;
3414                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3415                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3416                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3417                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3418                 /* Tell transmit desc to also do IPv4 checksum. */
3419                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3420                 break;
3421 #endif
3422         default:
3423                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3424                     __func__, ntohs(eh_type));
3425                 break;
3426         }
3427
3428         ctxd = txr->next_avail_desc;
3429         tx_buffer = &txr->tx_buffers[ctxd];
3430         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3431
3432         tcp_hlen = th->th_off << 2;
3433
3434         /* This is used in the transmit desc in encap */
3435         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3436
3437         /* VLAN MACLEN IPLEN */
3438 #ifdef NET_VLAN
3439         if (mp->m_flags & M_VLANTAG) {
3440                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3441                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3442         }
3443
3444         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3445         vlan_macip_lens |= ip_hlen;
3446         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3447 #endif
3448
3449         /* ADV DTYPE TUCMD */
3450         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3451         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3452         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3453
3454         /* MSS L4LEN IDX */
3455         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3456         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3457         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3458
3459         TXD->seqnum_seed = htole32(0);
3460         tx_buffer->m_head = NULL;
3461         tx_buffer->eop_index = -1;
3462
3463         if (++ctxd == adapter->num_tx_desc)
3464                 ctxd = 0;
3465
3466         txr->tx_avail--;
3467         txr->next_avail_desc = ctxd;
3468         return TRUE;
3469 }
3470 #endif
3471
3472 #ifdef IXGBE_FDIR
3473 /*
3474 ** This routine parses packet headers so that Flow
3475 ** Director can make a hashed filter table entry 
3476 ** allowing traffic flows to be identified and kept
3477 ** on the same cpu.  This would be a performance
3478 ** hit, but we only do it at IXGBE_FDIR_RATE of
3479 ** packets.
3480 */
3481 static void
3482 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3483 {
3484         struct adapter                  *adapter = txr->adapter;
3485         struct ix_queue                 *que;
3486         struct ip                       *ip;
3487         struct tcphdr                   *th;
3488         struct udphdr                   *uh;
3489         struct ether_vlan_header        *eh;
3490         union ixgbe_atr_hash_dword      input = {.dword = 0}; 
3491         union ixgbe_atr_hash_dword      common = {.dword = 0}; 
3492         int                             ehdrlen, ip_hlen;
3493         u16                             etype;
3494
3495         eh = mtod(mp, struct ether_vlan_header *);
3496         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3497                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3498                 etype = eh->evl_proto;
3499         } else {
3500                 ehdrlen = ETHER_HDR_LEN;
3501                 etype = eh->evl_encap_proto;
3502         }
3503
3504         /* Only handling IPv4 */
3505         if (etype != htons(ETHERTYPE_IP))
3506                 return;
3507
3508         ip = (struct ip *)(mp->m_data + ehdrlen);
3509         ip_hlen = ip->ip_hl << 2;
3510
3511         /* check if we're UDP or TCP */
3512         switch (ip->ip_p) {
3513         case IPPROTO_TCP:
3514                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3515                 /* src and dst are inverted */
3516                 common.port.dst ^= th->th_sport;
3517                 common.port.src ^= th->th_dport;
3518                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3519                 break;
3520         case IPPROTO_UDP:
3521                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3522                 /* src and dst are inverted */
3523                 common.port.dst ^= uh->uh_sport;
3524                 common.port.src ^= uh->uh_dport;
3525                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3526                 break;
3527         default:
3528                 return;
3529         }
3530
3531         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3532         if (mp->m_pkthdr.ether_vtag)
3533                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3534         else
3535                 common.flex_bytes ^= etype;
3536         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3537
3538         que = &adapter->queues[txr->me];
3539         /*
3540         ** This assumes the Rx queue and Tx
3541         ** queue are bound to the same CPU
3542         */
3543         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3544             input, common, que->msix);
3545 }
3546 #endif /* IXGBE_FDIR */
3547
3548 /**********************************************************************
3549  *
3550  *  Examine each tx_buffer in the used queue. If the hardware is done
3551  *  processing the packet then free associated resources. The
3552  *  tx_buffer is put back on the free queue.
3553  *
3554  **********************************************************************/
3555 static bool
3556 ixgbe_txeof(struct tx_ring *txr)
3557 {
3558         struct adapter  *adapter = txr->adapter;
3559         struct ifnet    *ifp = adapter->ifp;
3560         u32     first, last, done, processed;
3561         struct ixgbe_tx_buf *tx_buffer;
3562         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3563
3564         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3565
3566 #ifdef DEV_NETMAP
3567         if (ifp->if_capenable & IFCAP_NETMAP) {
3568                 struct netmap_adapter *na = NA(ifp);
3569                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3570
3571                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3572
3573                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3574                     BUS_DMASYNC_POSTREAD);
3575                 /*
3576                  * In netmap mode, all the work is done in the context
3577                  * of the client thread. Interrupt handlers only wake up
3578                  * clients, which may be sleeping on individual rings
3579                  * or on a global resource for all rings.
3580                  * To implement tx interrupt mitigation, we wake up the client
3581                  * thread roughly every half ring, even if the NIC interrupts
3582                  * more frequently. This is implemented as follows:
3583                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3584                  *   the slot that should wake up the thread (nkr_num_slots
3585                  *   means the user thread should not be woken up);
3586                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3587                  *   or the slot has the DD bit set.
3588                  *
3589                  * When the driver has separate locks, we need to
3590                  * release and re-acquire txlock to avoid deadlocks.
3591                  * XXX see if we can find a better way.
3592                  */
3593                 if (!netmap_mitigate ||
3594                     (kring->nr_kflags < kring->nkr_num_slots &&
3595                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3596                         kring->nr_kflags = kring->nkr_num_slots;
3597                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3598                         IXGBE_TX_UNLOCK(txr);
3599                         IXGBE_CORE_LOCK(adapter);
3600                         selwakeuppri(&na->tx_si, PI_NET);
3601                         IXGBE_CORE_UNLOCK(adapter);
3602                         IXGBE_TX_LOCK(txr);
3603                 }
3604                 return FALSE;
3605         }
3606 #endif /* DEV_NETMAP */
3607
3608         if (txr->tx_avail == adapter->num_tx_desc) {
3609                 txr->queue_status = IXGBE_QUEUE_IDLE;
3610                 return FALSE;
3611         }
3612
3613         processed = 0;
3614         first = txr->next_to_clean;
3615         tx_buffer = &txr->tx_buffers[first];
3616         /* For cleanup we just use legacy struct */
3617         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3618         last = tx_buffer->eop_index;
3619         if (last == -1)
3620                 return FALSE;
3621         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3622
3623         /*
3624         ** Get the index of the first descriptor
3625         ** BEYOND the EOP and call that 'done'.
3626         ** I do this so the comparison in the
3627         ** inner while loop below can be simple
3628         */
3629         if (++last == adapter->num_tx_desc) last = 0;
3630         done = last;
3631
3632         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3633             BUS_DMASYNC_POSTREAD);
3634         /*
3635         ** Only the EOP descriptor of a packet now has the DD
3636         ** bit set, this is what we look for...
3637         */
3638         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3639                 /* We clean the range of the packet */
3640                 while (first != done) {
3641                         tx_desc->upper.data = 0;
3642                         tx_desc->lower.data = 0;
3643                         tx_desc->buffer_addr = 0;
3644                         ++txr->tx_avail;
3645                         ++processed;
3646
3647                         if (tx_buffer->m_head) {
3648                                 txr->bytes +=
3649                                     tx_buffer->m_head->m_pkthdr.len;
3650                                 bus_dmamap_sync(txr->txtag,
3651                                     tx_buffer->map,
3652                                     BUS_DMASYNC_POSTWRITE);
3653                                 bus_dmamap_unload(txr->txtag,
3654                                     tx_buffer->map);
3655                                 m_freem(tx_buffer->m_head);
3656                                 tx_buffer->m_head = NULL;
3657                                 tx_buffer->map = NULL;
3658                         }
3659                         tx_buffer->eop_index = -1;
3660                         txr->watchdog_time = ticks;
3661
3662                         if (++first == adapter->num_tx_desc)
3663                                 first = 0;
3664
3665                         tx_buffer = &txr->tx_buffers[first];
3666                         tx_desc =
3667                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3668                 }
3669                 ++txr->packets;
3670                 ++ifp->if_opackets;
3671                 /* See if there is more work now */
3672                 last = tx_buffer->eop_index;
3673                 if (last != -1) {
3674                         eop_desc =
3675                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3676                         /* Get next done point */
3677                         if (++last == adapter->num_tx_desc) last = 0;
3678                         done = last;
3679                 } else
3680                         break;
3681         }
3682         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3683             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3684
3685         txr->next_to_clean = first;
3686
3687         /*
3688         ** Watchdog calculation, we know there's
3689         ** work outstanding or the first return
3690         ** would have been taken, so none processed
3691         ** for too long indicates a hang.
3692         */
3693         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3694                 txr->queue_status = IXGBE_QUEUE_HUNG;
3695
3696         /* With a minimum free clear the depleted state bit.  */
3697         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3698                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3699
3700         if (txr->tx_avail == adapter->num_tx_desc) {
3701                 txr->queue_status = IXGBE_QUEUE_IDLE;
3702                 return (FALSE);
3703         }
3704
3705         return TRUE;
3706 }
3707
3708 /*********************************************************************
3709  *
3710  *  Refresh mbuf buffers for RX descriptor rings
3711  *   - now keeps its own state so discards due to resource
3712  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3713  *     it just returns, keeping its placeholder, thus it can simply
3714  *     be recalled to try again.
3715  *
3716  **********************************************************************/
3717 static void
3718 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3719 {
3720         struct adapter          *adapter = rxr->adapter;
3721         bus_dma_segment_t       hseg[1];
3722         bus_dma_segment_t       pseg[1];
3723         struct ixgbe_rx_buf     *rxbuf;
3724         struct mbuf             *mh, *mp;
3725         int                     i, j, nsegs, error;
3726         bool                    refreshed = FALSE;
3727
3728         i = j = rxr->next_to_refresh;
3729         /* Control the loop with one beyond */
3730         if (++j == adapter->num_rx_desc)
3731                 j = 0;
3732
3733         while (j != limit) {
3734                 rxbuf = &rxr->rx_buffers[i];
3735                 if (rxr->hdr_split == FALSE)
3736                         goto no_split;
3737
3738                 if (rxbuf->m_head == NULL) {
3739                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3740                         if (mh == NULL)
3741                                 goto update;
3742                 } else
3743                         mh = rxbuf->m_head;
3744
3745                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3746                 mh->m_len = MHLEN;
3747                 mh->m_flags |= M_PKTHDR;
3748                 /* Get the memory mapping */
3749                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3750                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3751                 if (error != 0) {
3752                         kprintf("Refresh mbufs: hdr dmamap load"
3753                             " failure - %d\n", error);
3754                         m_free(mh);
3755                         rxbuf->m_head = NULL;
3756                         goto update;
3757                 }
3758                 rxbuf->m_head = mh;
3759                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3760                     BUS_DMASYNC_PREREAD);
3761                 rxr->rx_base[i].read.hdr_addr =
3762                     htole64(hseg[0].ds_addr);
3763
3764 no_split:
3765                 if (rxbuf->m_pack == NULL) {
3766                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3767                             M_PKTHDR, adapter->rx_mbuf_sz);
3768                         if (mp == NULL)
3769                                 goto update;
3770                 } else
3771                         mp = rxbuf->m_pack;
3772
3773                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3774                 /* Get the memory mapping */
3775                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3776                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3777                 if (error != 0) {
3778                         kprintf("Refresh mbufs: payload dmamap load"
3779                             " failure - %d\n", error);
3780                         m_free(mp);
3781                         rxbuf->m_pack = NULL;
3782                         goto update;
3783                 }
3784                 rxbuf->m_pack = mp;
3785                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3786                     BUS_DMASYNC_PREREAD);
3787                 rxr->rx_base[i].read.pkt_addr =
3788                     htole64(pseg[0].ds_addr);
3789
3790                 refreshed = TRUE;
3791                 /* Next is precalculated */
3792                 i = j;
3793                 rxr->next_to_refresh = i;
3794                 if (++j == adapter->num_rx_desc)
3795                         j = 0;
3796         }
3797 update:
3798         if (refreshed) /* Update hardware tail index */
3799                 IXGBE_WRITE_REG(&adapter->hw,
3800                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3801         return;
3802 }
3803
3804 /*********************************************************************
3805  *
3806  *  Allocate memory for rx_buffer structures. Since we use one
3807  *  rx_buffer per received packet, the maximum number of rx_buffer's
3808  *  that we'll need is equal to the number of receive descriptors
3809  *  that we've allocated.
3810  *
3811  **********************************************************************/
3812 static int
3813 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3814 {
3815         struct  adapter         *adapter = rxr->adapter;
3816         device_t                dev = adapter->dev;
3817         struct ixgbe_rx_buf     *rxbuf;
3818         int                     i, bsize, error;
3819
3820         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3821         if (!(rxr->rx_buffers =
3822             (struct ixgbe_rx_buf *) kmalloc(bsize,
3823             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3824                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3825                 error = ENOMEM;
3826                 goto fail;
3827         }
3828
3829         if ((error = bus_dma_tag_create(NULL,   /* parent */
3830                                    1, 0,        /* alignment, bounds */
3831                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3832                                    BUS_SPACE_MAXADDR,   /* highaddr */
3833                                    NULL, NULL,          /* filter, filterarg */
3834                                    MSIZE,               /* maxsize */
3835                                    1,                   /* nsegments */
3836                                    MSIZE,               /* maxsegsize */
3837                                    0,                   /* flags */
3838                                    &rxr->htag))) {
3839                 device_printf(dev, "Unable to create RX DMA tag\n");
3840                 goto fail;
3841         }
3842
3843         if ((error = bus_dma_tag_create(NULL,   /* parent */
3844                                    1, 0,        /* alignment, bounds */
3845                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3846                                    BUS_SPACE_MAXADDR,   /* highaddr */
3847                                    NULL, NULL,          /* filter, filterarg */
3848                                    MJUM16BYTES,         /* maxsize */
3849                                    1,                   /* nsegments */
3850                                    MJUM16BYTES,         /* maxsegsize */
3851                                    0,                   /* flags */
3852                                    &rxr->ptag))) {
3853                 device_printf(dev, "Unable to create RX DMA tag\n");
3854                 goto fail;
3855         }
3856
3857         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3858                 rxbuf = &rxr->rx_buffers[i];
3859                 error = bus_dmamap_create(rxr->htag,
3860                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3861                 if (error) {
3862                         device_printf(dev, "Unable to create RX head map\n");
3863                         goto fail;
3864                 }
3865                 error = bus_dmamap_create(rxr->ptag,
3866                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3867                 if (error) {
3868                         device_printf(dev, "Unable to create RX pkt map\n");
3869                         goto fail;
3870                 }
3871         }
3872
3873         return (0);
3874
3875 fail:
3876         /* Frees all, but can handle partial completion */
3877         ixgbe_free_receive_structures(adapter);
3878         return (error);
3879 }
3880
3881 /*
3882 ** Used to detect a descriptor that has
3883 ** been merged by Hardware RSC.
3884 */
3885 static inline u32
3886 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3887 {
3888         return (le32toh(rx->wb.lower.lo_dword.data) &
3889             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3890 }
3891
3892 /*********************************************************************
3893  *
3894  *  Initialize Hardware RSC (LRO) feature on 82599
3895  *  for an RX ring, this is toggled by the LRO capability
3896  *  even though it is transparent to the stack.
3897  *
3898  **********************************************************************/
3899 #if 0   /* NET_LRO */
3900 static void
3901 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3902 {
3903         struct  adapter         *adapter = rxr->adapter;
3904         struct  ixgbe_hw        *hw = &adapter->hw;
3905         u32                     rscctrl, rdrxctl;
3906
3907         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3908         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3909 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3910         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3911 #endif /* DEV_NETMAP */
3912         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3913         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3914         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3915
3916         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3917         rscctrl |= IXGBE_RSCCTL_RSCEN;
3918         /*
3919         ** Limit the total number of descriptors that
3920         ** can be combined, so it does not exceed 64K
3921         */
3922         if (adapter->rx_mbuf_sz == MCLBYTES)
3923                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3924         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3925                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3926         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3927                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3928         else  /* Using 16K cluster */
3929                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3930
3931         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3932
3933         /* Enable TCP header recognition */
3934         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3935             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3936             IXGBE_PSRTYPE_TCPHDR));
3937
3938         /* Disable RSC for ACK packets */
3939         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3940             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3941
3942         rxr->hw_rsc = TRUE;
3943 }
3944 #endif
3945
3946 static void     
3947 ixgbe_free_receive_ring(struct rx_ring *rxr)
3948
3949         struct  adapter         *adapter;
3950         struct ixgbe_rx_buf       *rxbuf;
3951         int i;
3952
3953         adapter = rxr->adapter;
3954         for (i = 0; i < adapter->num_rx_desc; i++) {
3955                 rxbuf = &rxr->rx_buffers[i];
3956                 if (rxbuf->m_head != NULL) {
3957                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3958                             BUS_DMASYNC_POSTREAD);
3959                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3960                         rxbuf->m_head->m_flags |= M_PKTHDR;
3961                         m_freem(rxbuf->m_head);
3962                 }
3963                 if (rxbuf->m_pack != NULL) {
3964                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3965                             BUS_DMASYNC_POSTREAD);
3966                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3967                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3968                         m_freem(rxbuf->m_pack);
3969                 }
3970                 rxbuf->m_head = NULL;
3971                 rxbuf->m_pack = NULL;
3972         }
3973 }
3974
3975
3976 /*********************************************************************
3977  *
3978  *  Initialize a receive ring and its buffers.
3979  *
3980  **********************************************************************/
3981 static int
3982 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3983 {
3984         struct  adapter         *adapter;
3985         struct ifnet            *ifp;
3986         device_t                dev;
3987         struct ixgbe_rx_buf     *rxbuf;
3988         bus_dma_segment_t       pseg[1], hseg[1];
3989 #if 0   /* NET_LRO */
3990         struct lro_ctrl         *lro = &rxr->lro;
3991 #endif
3992         int                     rsize, nsegs, error = 0;
3993 #ifdef DEV_NETMAP
3994         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3995         struct netmap_slot *slot;
3996 #endif /* DEV_NETMAP */
3997
3998         adapter = rxr->adapter;
3999         ifp = adapter->ifp;
4000         dev = adapter->dev;
4001
4002         /* Clear the ring contents */
4003         IXGBE_RX_LOCK(rxr);
4004 #ifdef DEV_NETMAP
4005         /* same as in ixgbe_setup_transmit_ring() */
4006         slot = netmap_reset(na, NR_RX, rxr->me, 0);
4007 #endif /* DEV_NETMAP */
4008         rsize = roundup2(adapter->num_rx_desc *
4009             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4010         bzero((void *)rxr->rx_base, rsize);
4011
4012         /* Free current RX buffer structs and their mbufs */
4013         ixgbe_free_receive_ring(rxr);
4014