kernel - callout_init() -> callout_init_mp() in selected cases
[dragonfly.git] / sys / dev / netif / e1000 / if_em.c
1 /******************************************************************************
2
3   Copyright (c) 2001-2010, Intel Corporation 
4   All rights reserved.
5   
6   Redistribution and use in source and binary forms, with or without 
7   modification, are permitted provided that the following conditions are met:
8   
9    1. Redistributions of source code must retain the above copyright notice, 
10       this list of conditions and the following disclaimer.
11   
12    2. Redistributions in binary form must reproduce the above copyright 
13       notice, this list of conditions and the following disclaimer in the 
14       documentation and/or other materials provided with the distribution.
15   
16    3. Neither the name of the Intel Corporation nor the names of its 
17       contributors may be used to endorse or promote products derived from 
18       this software without specific prior written permission.
19   
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31
32 ******************************************************************************/
33
34 #ifdef HAVE_KERNEL_OPTION_HEADERS
35 #include "opt_device_polling.h"
36 #include "opt_inet.h"
37 #include "opt_altq.h"
38 #endif
39
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #if __FreeBSD_version >= 800000
43 #include <sys/buf_ring.h>
44 #endif
45 #include <sys/bus.h>
46 #include <sys/endian.h>
47 #include <sys/kernel.h>
48 #include <sys/kthread.h>
49 #include <sys/malloc.h>
50 #include <sys/mbuf.h>
51 #include <sys/module.h>
52 #include <sys/rman.h>
53 #include <sys/socket.h>
54 #include <sys/sockio.h>
55 #include <sys/sysctl.h>
56 #include <sys/taskqueue.h>
57 #include <sys/eventhandler.h>
58
59 #include <net/bpf.h>
60 #include <net/ethernet.h>
61 #include <net/if.h>
62 #include <net/if_arp.h>
63 #include <net/if_dl.h>
64 #include <net/if_media.h>
65 #include <net/ifq_var.h>
66
67 #include <net/if_types.h>
68 #include <net/vlan/if_vlan_var.h>
69 #include <net/vlan/if_vlan_ether.h>
70
71 #include <netinet/in_systm.h>
72 #include <netinet/in.h>
73 #include <netinet/if_ether.h>
74 #include <netinet/ip.h>
75 #include <netinet/ip6.h>
76 #include <netinet/tcp.h>
77 #include <netinet/udp.h>
78
79 #include <sys/in_cksum.h>
80 #include <bus/pci/pcivar.h>
81 #include <bus/pci/pcireg.h>
82
83 #include "e1000_api.h"
84 #include "e1000_82571.h"
85 #include "if_em.h"
86 #include "ifcap_defines.h" // XXX
87
88 /*********************************************************************
89  *  Set this to one to display debug statistics
90  *********************************************************************/
91 int     em_display_debug_stats = 0;
92
93 /*********************************************************************
94  *  Driver version:
95  *********************************************************************/
96 char em_driver_version[] = "6.9.25";
97
98
99 /*********************************************************************
100  *  PCI Device ID Table
101  *
102  *  Used by probe to select devices to load on
103  *  Last field stores an index into e1000_strings
104  *  Last entry must be all 0s
105  *
106  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
107  *********************************************************************/
108
109 static em_vendor_info_t em_vendor_info_array[] =
110 {
111         /* Intel(R) PRO/1000 Network Connection */
112         { 0x8086, E1000_DEV_ID_82540EM,         PCI_ANY_ID, PCI_ANY_ID, 0},
113         { 0x8086, E1000_DEV_ID_82540EM_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
114         { 0x8086, E1000_DEV_ID_82540EP,         PCI_ANY_ID, PCI_ANY_ID, 0},
115         { 0x8086, E1000_DEV_ID_82540EP_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
116         { 0x8086, E1000_DEV_ID_82540EP_LP,      PCI_ANY_ID, PCI_ANY_ID, 0},
117
118         { 0x8086, E1000_DEV_ID_82541EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
119         { 0x8086, E1000_DEV_ID_82541ER,         PCI_ANY_ID, PCI_ANY_ID, 0},
120         { 0x8086, E1000_DEV_ID_82541ER_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
121         { 0x8086, E1000_DEV_ID_82541EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
122         { 0x8086, E1000_DEV_ID_82541GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
123         { 0x8086, E1000_DEV_ID_82541GI_LF,      PCI_ANY_ID, PCI_ANY_ID, 0},
124         { 0x8086, E1000_DEV_ID_82541GI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
125
126         { 0x8086, E1000_DEV_ID_82542,           PCI_ANY_ID, PCI_ANY_ID, 0},
127
128         { 0x8086, E1000_DEV_ID_82543GC_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
129         { 0x8086, E1000_DEV_ID_82543GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
130
131         { 0x8086, E1000_DEV_ID_82544EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
132         { 0x8086, E1000_DEV_ID_82544EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
133         { 0x8086, E1000_DEV_ID_82544GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
134         { 0x8086, E1000_DEV_ID_82544GC_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
135
136         { 0x8086, E1000_DEV_ID_82545EM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
137         { 0x8086, E1000_DEV_ID_82545EM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
138         { 0x8086, E1000_DEV_ID_82545GM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
139         { 0x8086, E1000_DEV_ID_82545GM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
140         { 0x8086, E1000_DEV_ID_82545GM_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
141
142         { 0x8086, E1000_DEV_ID_82546EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
143         { 0x8086, E1000_DEV_ID_82546EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
144         { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
145         { 0x8086, E1000_DEV_ID_82546GB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
146         { 0x8086, E1000_DEV_ID_82546GB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
147         { 0x8086, E1000_DEV_ID_82546GB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
148         { 0x8086, E1000_DEV_ID_82546GB_PCIE,    PCI_ANY_ID, PCI_ANY_ID, 0},
149         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
150         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
151                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
152
153         { 0x8086, E1000_DEV_ID_82547EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
154         { 0x8086, E1000_DEV_ID_82547EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
155         { 0x8086, E1000_DEV_ID_82547GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
156
157         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
158         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
159         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
160         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
161                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
162         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
163                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
164         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
165                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
166         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
167                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
168         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
169                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
170         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
171                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
172         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
173         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
174         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
175         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
176
177         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
178         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
179         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
180         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
181         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
182                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
183         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
184                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
185         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
186                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
187         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
188                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
189         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
190         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
191         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
192         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
193         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
194         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
195         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
196         { 0x8086, E1000_DEV_ID_ICH8_82567V_3,   PCI_ANY_ID, PCI_ANY_ID, 0},
197         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
198         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
199         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
200         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
201         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
202         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
203         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
204         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
205         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
206         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
207         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
208         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
209         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
210         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
211         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
212         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
213         { 0x8086, E1000_DEV_ID_PCH_M_HV_LM,     PCI_ANY_ID, PCI_ANY_ID, 0},
214         { 0x8086, E1000_DEV_ID_PCH_M_HV_LC,     PCI_ANY_ID, PCI_ANY_ID, 0},
215         { 0x8086, E1000_DEV_ID_PCH_D_HV_DM,     PCI_ANY_ID, PCI_ANY_ID, 0},
216         { 0x8086, E1000_DEV_ID_PCH_D_HV_DC,     PCI_ANY_ID, PCI_ANY_ID, 0},
217         /* required last entry */
218         { 0, 0, 0, 0, 0}
219 };
220
221 /*********************************************************************
222  *  Table of branding strings for all supported NICs.
223  *********************************************************************/
224
225 static char *em_strings[] = {
226         "Intel(R) PRO/1000 Network Connection"
227 };
228
229 /*********************************************************************
230  *  Function prototypes
231  *********************************************************************/
232 static int      em_probe(device_t);
233 static int      em_attach(device_t);
234 static int      em_detach(device_t);
235 static int      em_shutdown(device_t);
236 static int      em_suspend(device_t);
237 static int      em_resume(device_t);
238 static void     em_start(struct ifnet *);
239 static void     em_start_locked(struct ifnet *ifp);
240 #if __FreeBSD_version >= 800000
241 static int      em_mq_start(struct ifnet *, struct mbuf *);
242 static int      em_mq_start_locked(struct ifnet *, struct mbuf *);
243 static void     em_qflush(struct ifnet *);
244 #endif
245 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
246 static void     em_init(void *);
247 static void     em_init_locked(struct adapter *);
248 static void     em_stop(void *);
249 static void     em_media_status(struct ifnet *, struct ifmediareq *);
250 static int      em_media_change(struct ifnet *);
251 static void     em_identify_hardware(struct adapter *);
252 static int      em_allocate_pci_resources(struct adapter *);
253 static int      em_allocate_legacy(struct adapter *adapter);
254 static int      em_allocate_msix(struct adapter *adapter);
255 static int      em_setup_msix(struct adapter *);
256 static void     em_free_pci_resources(struct adapter *);
257 static void     em_local_timer(void *);
258 static int      em_hardware_init(struct adapter *);
259 static void     em_setup_interface(device_t, struct adapter *);
260 static void     em_setup_transmit_structures(struct adapter *);
261 static void     em_initialize_transmit_unit(struct adapter *);
262 static int      em_setup_receive_structures(struct adapter *);
263 static void     em_initialize_receive_unit(struct adapter *);
264 static void     em_enable_intr(struct adapter *);
265 static void     em_disable_intr(struct adapter *);
266 static void     em_free_transmit_structures(struct adapter *);
267 static void     em_free_receive_structures(struct adapter *);
268 static void     em_update_stats_counters(struct adapter *);
269 static void     em_txeof(struct adapter *);
270 static void     em_tx_purge(struct adapter *);
271 static int      em_allocate_receive_structures(struct adapter *);
272 static int      em_allocate_transmit_structures(struct adapter *);
273 static int      em_rxeof(struct adapter *, int);
274 #ifndef __NO_STRICT_ALIGNMENT
275 static int      em_fixup_rx(struct adapter *);
276 #endif
277 static void     em_receive_checksum(struct adapter *, struct e1000_rx_desc *,
278                     struct mbuf *);
279 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
280                     u32 *, u32 *);
281 #ifdef NET_TSO
282 static bool     em_tso_setup(struct adapter *, struct mbuf *,
283                     u32 *, u32 *);
284 #endif
285 static void     em_set_promisc(struct adapter *);
286 static void     em_disable_promisc(struct adapter *);
287 static void     em_set_multi(struct adapter *);
288 static void     em_print_hw_stats(struct adapter *);
289 static void     em_update_link_status(struct adapter *);
290 static int      em_get_buf(struct adapter *, int);
291
292 static void     em_register_vlan(void *, struct ifnet *, u16);
293 static void     em_unregister_vlan(void *, struct ifnet *, u16);
294 static void     em_setup_vlan_hw_support(struct adapter *);
295
296 static int      em_xmit(struct adapter *, struct mbuf **);
297 static void     em_smartspeed(struct adapter *);
298 static int      em_82547_fifo_workaround(struct adapter *, int);
299 static void     em_82547_update_fifo_head(struct adapter *, int);
300 static int      em_82547_tx_fifo_reset(struct adapter *);
301 static void     em_82547_move_tail(void *);
302 static int      em_dma_malloc(struct adapter *, bus_size_t,
303                     struct em_dma_alloc *, int);
304 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
305 static void     em_print_debug_info(struct adapter *);
306 static void     em_print_nvm_info(struct adapter *);
307 static int      em_is_valid_ether_addr(u8 *);
308 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
309 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
310 static u32      em_fill_descriptors (bus_addr_t address, u32 length,
311                     PDESC_ARRAY desc_array);
312 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
313 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
314                     const char *, struct em_int_delay_info *, int, int);
315 /* Management and WOL Support */
316 static void     em_init_manageability(struct adapter *);
317 static void     em_release_manageability(struct adapter *);
318 static void     em_get_hw_control(struct adapter *);
319 static void     em_release_hw_control(struct adapter *);
320 static void     em_get_wakeup(device_t);
321 static void     em_enable_wakeup(device_t);
322 static int      em_enable_phy_wakeup(struct adapter *);
323
324 #ifdef EM_LEGACY_IRQ
325 static void     em_intr(void *);
326 #else /* FAST IRQ */
327 static void     em_irq_fast(void *);
328
329 /* MSIX handlers */
330 static void     em_msix_tx(void *);
331 static void     em_msix_rx(void *);
332 static void     em_msix_link(void *);
333 static void     em_handle_rx(void *context, int pending);
334 static void     em_handle_tx(void *context, int pending);
335
336 static void     em_handle_rxtx(void *context, int pending);
337 static void     em_handle_link(void *context, int pending);
338 static void     em_add_rx_process_limit(struct adapter *, const char *,
339                     const char *, int *, int);
340 #endif /* ~EM_LEGACY_IRQ */
341
342 #ifdef DEVICE_POLLING
343 static poll_handler_t em_poll;
344 #endif /* POLLING */
345
346 /*********************************************************************
347  *  FreeBSD Device Interface Entry Points
348  *********************************************************************/
349
350 static device_method_t em_methods[] = {
351         /* Device interface */
352         DEVMETHOD(device_probe, em_probe),
353         DEVMETHOD(device_attach, em_attach),
354         DEVMETHOD(device_detach, em_detach),
355         DEVMETHOD(device_shutdown, em_shutdown),
356         DEVMETHOD(device_suspend, em_suspend),
357         DEVMETHOD(device_resume, em_resume),
358         {0, 0}
359 };
360
361 static driver_t em_driver = {
362         "em", em_methods, sizeof(struct adapter),
363 };
364
365 static devclass_t em_devclass;
366 DRIVER_MODULE(em, pci, em_driver, em_devclass, NULL, NULL);
367 MODULE_DEPEND(em, pci, 1, 1, 1);
368 MODULE_DEPEND(em, ether, 1, 1, 1);
369
370 /*********************************************************************
371  *  Tunable default values.
372  *********************************************************************/
373
374 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
375 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
376 #define M_TSO_LEN                       66
377
378 /* Allow common code without TSO */
379 #ifndef CSUM_TSO
380 #define CSUM_TSO        0
381 #endif
382
383 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
384 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
385 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
386 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
387 static int em_rxd = EM_DEFAULT_RXD;
388 static int em_txd = EM_DEFAULT_TXD;
389 static int em_smart_pwr_down = FALSE;
390 /* Controls whether promiscuous also shows bad packets */
391 static int em_debug_sbp = FALSE;
392 /* Local switch for MSI/MSIX */
393 static int em_enable_msi = TRUE;
394
395 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
396 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
397 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
398 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
399 TUNABLE_INT("hw.em.rxd", &em_rxd);
400 TUNABLE_INT("hw.em.txd", &em_txd);
401 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
402 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
403 TUNABLE_INT("hw.em.enable_msi", &em_enable_msi);
404
405 #ifndef EM_LEGACY_IRQ
406 /* How many packets rxeof tries to clean at a time */
407 static int em_rx_process_limit = 100;
408 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
409 #endif
410
411 /* Flow control setting - default to FULL */
412 static int em_fc_setting = e1000_fc_full;
413 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting);
414
415 /*
416 ** Shadow VFTA table, this is needed because
417 ** the real vlan filter table gets cleared during
418 ** a soft reset and the driver needs to be able
419 ** to repopulate it.
420 */
421 static u32 em_shadow_vfta[EM_VFTA_SIZE];
422
423 /* Global used in WOL setup with multiport cards */
424 static int global_quad_port_a = 0;
425
426 /*********************************************************************
427  *  Device identification routine
428  *
429  *  em_probe determines if the driver should be loaded on
430  *  adapter based on PCI vendor/device id of the adapter.
431  *
432  *  return BUS_PROBE_DEFAULT on success, positive on failure
433  *********************************************************************/
434
435 static int
436 em_probe(device_t dev)
437 {
438         char            adapter_name[60];
439         u16             pci_vendor_id = 0;
440         u16             pci_device_id = 0;
441         u16             pci_subvendor_id = 0;
442         u16             pci_subdevice_id = 0;
443         em_vendor_info_t *ent;
444
445         INIT_DEBUGOUT("em_probe: begin");
446
447         pci_vendor_id = pci_get_vendor(dev);
448         if (pci_vendor_id != EM_VENDOR_ID)
449                 return (ENXIO);
450
451         pci_device_id = pci_get_device(dev);
452         pci_subvendor_id = pci_get_subvendor(dev);
453         pci_subdevice_id = pci_get_subdevice(dev);
454
455         ent = em_vendor_info_array;
456         while (ent->vendor_id != 0) {
457                 if ((pci_vendor_id == ent->vendor_id) &&
458                     (pci_device_id == ent->device_id) &&
459
460                     ((pci_subvendor_id == ent->subvendor_id) ||
461                     (ent->subvendor_id == PCI_ANY_ID)) &&
462
463                     ((pci_subdevice_id == ent->subdevice_id) ||
464                     (ent->subdevice_id == PCI_ANY_ID))) {
465                         ksprintf(adapter_name, "%s %s",
466                                 em_strings[ent->index],
467                                 em_driver_version);
468                         device_set_desc_copy(dev, adapter_name);
469                         return (BUS_PROBE_DEFAULT);
470                 }
471                 ent++;
472         }
473
474         return (ENXIO);
475 }
476
477 /*********************************************************************
478  *  Device initialization routine
479  *
480  *  The attach entry point is called when the driver is being loaded.
481  *  This routine identifies the type of hardware, allocates all resources
482  *  and initializes the hardware.
483  *
484  *  return 0 on success, positive on failure
485  *********************************************************************/
486
487 static int
488 em_attach(device_t dev)
489 {
490         struct adapter  *adapter;
491         int             tsize, rsize;
492         int             error = 0;
493
494         INIT_DEBUGOUT("em_attach: begin");
495
496         adapter = device_get_softc(dev);
497         adapter->dev = adapter->osdep.dev = dev;
498
499         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
500         EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
501         EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
502
503         /* SYSCTL stuff */
504         sysctl_ctx_init(&adapter->sysctl_ctx);
505         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
506                                         SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
507                                         device_get_nameunit(adapter->dev),
508                                         CTLFLAG_RD, 0, "");
509         if (adapter->sysctl_tree == NULL) {
510                 device_printf(adapter->dev, "can't add sysctl node\n");
511                 error = ENOMEM;
512                 goto err_sysctl;
513         }
514
515         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
516             SYSCTL_CHILDREN(adapter->sysctl_tree),
517             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
518             em_sysctl_debug_info, "I", "Debug Information");
519
520         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
521             SYSCTL_CHILDREN(adapter->sysctl_tree),
522             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
523             em_sysctl_stats, "I", "Statistics");
524
525         callout_init_mp(&adapter->timer);
526         callout_init_mp(&adapter->tx_fifo_timer);
527
528         /* Determine hardware and mac info */
529         em_identify_hardware(adapter);
530
531         /* Setup PCI resources */
532         if (em_allocate_pci_resources(adapter)) {
533                 device_printf(dev, "Allocation of PCI resources failed\n");
534                 error = ENXIO;
535                 goto err_pci;
536         }
537
538         /*
539         ** For ICH8 and family we need to
540         ** map the flash memory, and this
541         ** must happen after the MAC is 
542         ** identified
543         */
544         if ((adapter->hw.mac.type == e1000_ich8lan) ||
545             (adapter->hw.mac.type == e1000_pchlan) ||
546             (adapter->hw.mac.type == e1000_ich9lan) ||
547             (adapter->hw.mac.type == e1000_ich10lan)) {
548                 int rid = EM_BAR_TYPE_FLASH;
549                 adapter->flash = bus_alloc_resource_any(dev,
550                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
551                 if (adapter->flash == NULL) {
552                         device_printf(dev, "Mapping of Flash failed\n");
553                         error = ENXIO;
554                         goto err_pci;
555                 }
556                 /* This is used in the shared code */
557                 adapter->hw.flash_address = (u8 *)adapter->flash;
558                 adapter->osdep.flash_bus_space_tag =
559                     rman_get_bustag(adapter->flash);
560                 adapter->osdep.flash_bus_space_handle =
561                     rman_get_bushandle(adapter->flash);
562         }
563
564         /* Do Shared Code initialization */
565         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
566                 device_printf(dev, "Setup of Shared code failed\n");
567                 error = ENXIO;
568                 goto err_pci;
569         }
570
571         e1000_get_bus_info(&adapter->hw);
572
573         /* Set up some sysctls for the tunable interrupt delays */
574         em_add_int_delay_sysctl(adapter, "rx_int_delay",
575             "receive interrupt delay in usecs", &adapter->rx_int_delay,
576             E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
577         em_add_int_delay_sysctl(adapter, "tx_int_delay",
578             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
579             E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
580         if (adapter->hw.mac.type >= e1000_82540) {
581                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
582                     "receive interrupt delay limit in usecs",
583                     &adapter->rx_abs_int_delay,
584                     E1000_REGISTER(&adapter->hw, E1000_RADV),
585                     em_rx_abs_int_delay_dflt);
586                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
587                     "transmit interrupt delay limit in usecs",
588                     &adapter->tx_abs_int_delay,
589                     E1000_REGISTER(&adapter->hw, E1000_TADV),
590                     em_tx_abs_int_delay_dflt);
591         }
592
593 #ifndef EM_LEGACY_IRQ
594         /* Sysctls for limiting the amount of work done in the taskqueue */
595         em_add_rx_process_limit(adapter, "rx_processing_limit",
596             "max number of rx packets to process", &adapter->rx_process_limit,
597             em_rx_process_limit);
598 #endif
599
600         /*
601          * Validate number of transmit and receive descriptors. It
602          * must not exceed hardware maximum, and must be multiple
603          * of E1000_DBA_ALIGN.
604          */
605         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
606             (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
607             (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
608             (em_txd < EM_MIN_TXD)) {
609                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
610                     EM_DEFAULT_TXD, em_txd);
611                 adapter->num_tx_desc = EM_DEFAULT_TXD;
612         } else
613                 adapter->num_tx_desc = em_txd;
614         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
615             (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
616             (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
617             (em_rxd < EM_MIN_RXD)) {
618                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
619                     EM_DEFAULT_RXD, em_rxd);
620                 adapter->num_rx_desc = EM_DEFAULT_RXD;
621         } else
622                 adapter->num_rx_desc = em_rxd;
623
624         adapter->hw.mac.autoneg = DO_AUTO_NEG;
625         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
626         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
627         adapter->rx_buffer_len = 2048;
628
629         e1000_init_script_state_82541(&adapter->hw, TRUE);
630         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
631
632         /* Copper options */
633         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
634                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
635                 adapter->hw.phy.disable_polarity_correction = FALSE;
636                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
637         }
638
639         /*
640          * Set the frame limits assuming
641          * standard ethernet sized frames.
642          */
643         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
644         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
645
646         /*
647          * This controls when hardware reports transmit completion
648          * status.
649          */
650         adapter->hw.mac.report_tx_early = 1;
651
652         tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
653             EM_DBA_ALIGN);
654
655         /* Allocate Transmit Descriptor ring */
656         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
657                 device_printf(dev, "Unable to allocate tx_desc memory\n");
658                 error = ENOMEM;
659                 goto err_tx_desc;
660         }
661         adapter->tx_desc_base = 
662             (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
663
664         rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
665             EM_DBA_ALIGN);
666
667         /* Allocate Receive Descriptor ring */
668         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
669                 device_printf(dev, "Unable to allocate rx_desc memory\n");
670                 error = ENOMEM;
671                 goto err_rx_desc;
672         }
673         adapter->rx_desc_base =
674             (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
675
676         /*
677         ** Start from a known state, this is
678         ** important in reading the nvm and
679         ** mac from that.
680         */
681         e1000_reset_hw(&adapter->hw);
682
683         /* Make sure we have a good EEPROM before we read from it */
684         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
685                 /*
686                 ** Some PCI-E parts fail the first check due to
687                 ** the link being in sleep state, call it again,
688                 ** if it fails a second time its a real issue.
689                 */
690                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
691                         device_printf(dev,
692                             "The EEPROM Checksum Is Not Valid\n");
693                         error = EIO;
694                         goto err_hw_init;
695                 }
696         }
697
698         /* Copy the permanent MAC address out of the EEPROM */
699         if (e1000_read_mac_addr(&adapter->hw) < 0) {
700                 device_printf(dev, "EEPROM read error while reading MAC"
701                     " address\n");
702                 error = EIO;
703                 goto err_hw_init;
704         }
705
706         if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
707                 device_printf(dev, "Invalid MAC address\n");
708                 error = EIO;
709                 goto err_hw_init;
710         }
711
712         /* Initialize the hardware */
713         if (em_hardware_init(adapter)) {
714                 device_printf(dev, "Unable to initialize the hardware\n");
715                 error = EIO;
716                 goto err_hw_init;
717         }
718
719         /* Allocate transmit descriptors and buffers */
720         if (em_allocate_transmit_structures(adapter)) {
721                 device_printf(dev, "Could not setup transmit structures\n");
722                 error = ENOMEM;
723                 goto err_tx_struct;
724         }
725
726         /* Allocate receive descriptors and buffers */
727         if (em_allocate_receive_structures(adapter)) {
728                 device_printf(dev, "Could not setup receive structures\n");
729                 error = ENOMEM;
730                 goto err_rx_struct;
731         }
732
733         /*
734         **  Do interrupt configuration
735         */
736         if (adapter->msi > 1) /* Do MSI/X */
737                 error = em_allocate_msix(adapter);
738         else  /* MSI or Legacy */
739                 error = em_allocate_legacy(adapter);
740         if (error)
741                 goto err_rx_struct;
742
743         /*
744          * Get Wake-on-Lan and Management info for later use
745          */
746         em_get_wakeup(dev);
747
748         /* Setup OS specific network interface */
749         em_setup_interface(dev, adapter);
750
751         /* Initialize statistics */
752         em_update_stats_counters(adapter);
753
754         adapter->hw.mac.get_link_status = 1;
755         em_update_link_status(adapter);
756
757         /* Indicate SOL/IDER usage */
758         if (e1000_check_reset_block(&adapter->hw))
759                 device_printf(dev,
760                     "PHY reset is blocked due to SOL/IDER session.\n");
761
762         /* Do we need workaround for 82544 PCI-X adapter? */
763         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
764             adapter->hw.mac.type == e1000_82544)
765                 adapter->pcix_82544 = TRUE;
766         else
767                 adapter->pcix_82544 = FALSE;
768
769         /* Register for VLAN events */
770         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
771             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
772         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
773             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 
774
775         /* Non-AMT based hardware can now take control from firmware */
776         if (adapter->has_manage && !adapter->has_amt)
777                 em_get_hw_control(adapter);
778
779         /* Tell the stack that the interface is not active */
780         adapter->ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
781
782         INIT_DEBUGOUT("em_attach: end");
783
784         return (0);
785
786 err_rx_struct:
787         em_free_transmit_structures(adapter);
788 err_tx_struct:
789 err_hw_init:
790         em_release_hw_control(adapter);
791         em_dma_free(adapter, &adapter->rxdma);
792 err_rx_desc:
793         em_dma_free(adapter, &adapter->txdma);
794 err_tx_desc:
795 err_pci:
796         em_free_pci_resources(adapter);
797 err_sysctl:
798         sysctl_ctx_free(&adapter->sysctl_ctx);
799         EM_TX_LOCK_DESTROY(adapter);
800         EM_RX_LOCK_DESTROY(adapter);
801         EM_CORE_LOCK_DESTROY(adapter);
802
803         return (error);
804 }
805
806 /*********************************************************************
807  *  Device removal routine
808  *
809  *  The detach entry point is called when the driver is being removed.
810  *  This routine stops the adapter and deallocates all the resources
811  *  that were allocated for driver operation.
812  *
813  *  return 0 on success, positive on failure
814  *********************************************************************/
815
816 static int
817 em_detach(device_t dev)
818 {
819         struct adapter  *adapter = device_get_softc(dev);
820
821         INIT_DEBUGOUT("em_detach: begin");
822
823         /* Make sure VLANS are not using driver */
824         if (adapter->ifp->if_vlantrunks != NULL) {
825                 device_printf(dev,"Vlan in use, detach first\n");
826                 return (EBUSY);
827         }
828
829 #ifdef DEVICE_POLLING
830         if (ifp->if_capenable & IFCAP_POLLING)
831                 ether_poll_deregister(ifp);
832 #endif
833
834         EM_CORE_LOCK(adapter);
835         EM_TX_LOCK(adapter);
836         adapter->in_detach = 1;
837         em_stop(adapter);
838         e1000_phy_hw_reset(&adapter->hw);
839
840         em_release_manageability(adapter);
841
842         EM_TX_UNLOCK(adapter);
843         EM_CORE_UNLOCK(adapter);
844
845         /* Unregister VLAN events */
846         if (adapter->vlan_attach != NULL)
847                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
848         if (adapter->vlan_detach != NULL)
849                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 
850
851         ether_ifdetach(adapter->ifp);
852         callout_stop(&adapter->timer);
853         callout_stop(&adapter->tx_fifo_timer);
854
855         em_free_pci_resources(adapter);
856         bus_generic_detach(dev);
857
858         em_free_transmit_structures(adapter);
859         em_free_receive_structures(adapter);
860
861         /* Free Transmit Descriptor ring */
862         if (adapter->tx_desc_base) {
863                 em_dma_free(adapter, &adapter->txdma);
864                 adapter->tx_desc_base = NULL;
865         }
866
867         /* Free Receive Descriptor ring */
868         if (adapter->rx_desc_base) {
869                 em_dma_free(adapter, &adapter->rxdma);
870                 adapter->rx_desc_base = NULL;
871         }
872
873         em_release_hw_control(adapter);
874         sysctl_ctx_free(&adapter->sysctl_ctx);
875         EM_TX_LOCK_DESTROY(adapter);
876         EM_RX_LOCK_DESTROY(adapter);
877         EM_CORE_LOCK_DESTROY(adapter);
878
879         return (0);
880 }
881
882 /*********************************************************************
883  *
884  *  Shutdown entry point
885  *
886  **********************************************************************/
887
888 static int
889 em_shutdown(device_t dev)
890 {
891         return em_suspend(dev);
892 }
893
894 /*
895  * Suspend/resume device methods.
896  */
897 static int
898 em_suspend(device_t dev)
899 {
900         struct adapter *adapter = device_get_softc(dev);
901
902         EM_CORE_LOCK(adapter);
903
904         em_release_manageability(adapter);
905         em_release_hw_control(adapter);
906         em_enable_wakeup(dev);
907
908         EM_CORE_UNLOCK(adapter);
909
910         return bus_generic_suspend(dev);
911 }
912
913 static int
914 em_resume(device_t dev)
915 {
916         struct adapter *adapter = device_get_softc(dev);
917         struct ifnet *ifp = adapter->ifp;
918
919         EM_CORE_LOCK(adapter);
920         em_init_locked(adapter);
921         em_init_manageability(adapter);
922         EM_CORE_UNLOCK(adapter);
923         em_start(ifp);
924
925         return bus_generic_resume(dev);
926 }
927
928
929 /*********************************************************************
930  *  Transmit entry point
931  *
932  *  em_start is called by the stack to initiate a transmit.
933  *  The driver will remain in this routine as long as there are
934  *  packets to transmit and transmit resources are available.
935  *  In case resources are not available stack is notified and
936  *  the packet is requeued.
937  **********************************************************************/
938
939 #if 0
940 static int
941 em_mq_start_locked(struct ifnet *ifp, struct mbuf *m)
942 {
943         struct adapter  *adapter = ifp->if_softc;
944         struct mbuf     *next;
945         int error = E1000_SUCCESS;
946
947         EM_TX_LOCK_ASSERT(adapter);
948         /* To allow being called from a tasklet */
949         if (m == NULL)
950                 goto process;
951
952         if (((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) !=
953             IFF_RUNNING)
954             || (!adapter->link_active)) {
955                 error = drbr_enqueue(ifp, adapter->br, m);
956                 return (error);
957         } else if (!drbr_needs_enqueue(ifp, adapter->br) &&
958             (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) {
959                 if ((error = em_xmit(adapter, &m)) != 0) {
960                         if (m)
961                                 error = drbr_enqueue(ifp, adapter->br, m);
962                         return (error);
963                 } else {
964                         /*
965                          * We've bypassed the buf ring so we need to update
966                          * ifp directly
967                          */
968                         drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
969                         /*
970                         ** Send a copy of the frame to the BPF
971                         ** listener and set the watchdog on.
972                         */
973                         ETHER_BPF_MTAP(ifp, m);
974                         adapter->watchdog_check = TRUE;
975                 }
976         } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0)
977                 return (error);
978         
979 process:
980         if (drbr_empty(ifp, adapter->br))
981                 return(error);
982         /* Process the queue */
983         while (TRUE) {
984                 if ((ifp->if_flags & IFF_RUNNING) == 0)
985                         break;
986                 next = drbr_dequeue(ifp, adapter->br);
987                 if (next == NULL)
988                         break;
989                 if ((error = em_xmit(adapter, &next)) != 0) {
990                         if (next != NULL)
991                                 error = drbr_enqueue(ifp, adapter->br, next);
992                         break;
993                 }
994                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
995                 ETHER_BPF_MTAP(ifp, next);
996                 /* Set the watchdog */
997                 adapter->watchdog_check = TRUE;
998         }
999
1000         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1001                 ifp->if_flags |= IFF_OACTIVE;
1002
1003         return (error);
1004 }
1005
1006 /*
1007 ** Multiqueue capable stack interface, this is not
1008 ** yet truely multiqueue, but that is coming...
1009 */
1010 static int
1011 em_mq_start(struct ifnet *ifp, struct mbuf *m)
1012 {
1013         
1014         struct adapter *adapter = ifp->if_softc;
1015         int error = 0;
1016
1017         if (EM_TX_TRYLOCK(adapter)) {
1018                 if (ifp->if_flags & IFF_RUNNING)
1019                         error = em_mq_start_locked(ifp, m);
1020                 EM_TX_UNLOCK(adapter);
1021         } else 
1022                 error = drbr_enqueue(ifp, adapter->br, m);
1023
1024         return (error);
1025 }
1026
1027 static void
1028 em_qflush(struct ifnet *ifp)
1029 {
1030         struct mbuf *m;
1031         struct adapter *adapter = (struct adapter *)ifp->if_softc;
1032
1033         EM_TX_LOCK(adapter);
1034         while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL)
1035                 m_freem(m);
1036         if_qflush(ifp);
1037         EM_TX_UNLOCK(adapter);
1038 }
1039 #endif
1040
1041 static void
1042 em_start_locked(struct ifnet *ifp)
1043 {
1044         struct adapter  *adapter = ifp->if_softc;
1045         struct mbuf     *m_head;
1046
1047         EM_TX_LOCK_ASSERT(adapter);
1048
1049         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) !=
1050             IFF_RUNNING)
1051                 return;
1052         if (!adapter->link_active)
1053                 return;
1054
1055         while (!ifq_is_empty(&ifp->if_snd)) {
1056
1057                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
1058                 if (m_head == NULL)
1059                         break;
1060                 /*
1061                  *  Encapsulation can modify our pointer, and or make it
1062                  *  NULL on failure.  In that event, we can't requeue.
1063                  */
1064                 if (em_xmit(adapter, &m_head)) {
1065                         if (m_head == NULL)
1066                                 break;
1067                         ifp->if_flags |= IFF_OACTIVE;
1068                         ifq_prepend(&ifp->if_snd, m_head);
1069                         break;
1070                 }
1071
1072                 /* Send a copy of the frame to the BPF listener */
1073                 ETHER_BPF_MTAP(ifp, m_head);
1074
1075                 /* Set timeout in case hardware has problems transmitting. */
1076                 adapter->watchdog_check = TRUE;
1077         }
1078         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1079                 ifp->if_flags |= IFF_OACTIVE;
1080
1081         return;
1082 }
1083
1084 static void
1085 em_start(struct ifnet *ifp)
1086 {
1087         struct adapter *adapter = ifp->if_softc;
1088
1089         EM_TX_LOCK(adapter);
1090         if (ifp->if_flags & IFF_RUNNING)
1091                 em_start_locked(ifp);
1092         EM_TX_UNLOCK(adapter);
1093 }
1094
1095 /*********************************************************************
1096  *  Ioctl entry point
1097  *
1098  *  em_ioctl is called when the user wants to configure the
1099  *  interface.
1100  *
1101  *  return 0 on success, positive on failure
1102  **********************************************************************/
1103
1104 static int
1105 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred * uc)
1106 {
1107         struct adapter  *adapter = ifp->if_softc;
1108         struct ifreq *ifr = (struct ifreq *)data;
1109 #ifdef INET
1110         struct ifaddr *ifa = (struct ifaddr *)data;
1111 #endif
1112         int error = 0;
1113
1114         if (adapter->in_detach)
1115                 return (error);
1116
1117         switch (command) {
1118         case SIOCSIFADDR:
1119 #ifdef INET
1120                 if (ifa->ifa_addr->sa_family == AF_INET) {
1121                         /*
1122                          * XXX
1123                          * Since resetting hardware takes a very long time
1124                          * and results in link renegotiation we only
1125                          * initialize the hardware only when it is absolutely
1126                          * required.
1127                          */
1128                         ifp->if_flags |= IFF_UP;
1129                         if (!(ifp->if_flags & IFF_RUNNING)) {
1130                                 EM_CORE_LOCK(adapter);
1131                                 em_init_locked(adapter);
1132                                 EM_CORE_UNLOCK(adapter);
1133                         }
1134                         arp_ifinit(ifp, ifa);
1135                 } else
1136 #endif
1137                         error = ether_ioctl(ifp, command, data);
1138                 break;
1139         case SIOCSIFMTU:
1140             {
1141                 int max_frame_size;
1142                 u16 eeprom_data = 0;
1143
1144                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1145
1146                 EM_CORE_LOCK(adapter);
1147                 switch (adapter->hw.mac.type) {
1148                 case e1000_82573:
1149                         /*
1150                          * 82573 only supports jumbo frames
1151                          * if ASPM is disabled.
1152                          */
1153                         e1000_read_nvm(&adapter->hw,
1154                             NVM_INIT_3GIO_3, 1, &eeprom_data);
1155                         if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1156                                 max_frame_size = ETHER_MAX_LEN;
1157                                 break;
1158                         }
1159                         /* Allow Jumbo frames - fall thru */
1160                 case e1000_82571:
1161                 case e1000_82572:
1162                 case e1000_ich9lan:
1163                 case e1000_ich10lan:
1164                 case e1000_82574:
1165                 case e1000_80003es2lan: /* Limit Jumbo Frame size */
1166                         max_frame_size = 9234;
1167                         break;
1168                 case e1000_pchlan:
1169                         max_frame_size = 4096;
1170                         break;
1171                         /* Adapters that do not support jumbo frames */
1172                 case e1000_82542:
1173                 case e1000_82583:
1174                 case e1000_ich8lan:
1175                         max_frame_size = ETHER_MAX_LEN;
1176                         break;
1177                 default:
1178                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1179                 }
1180                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1181                     ETHER_CRC_LEN) {
1182                         EM_CORE_UNLOCK(adapter);
1183                         error = EINVAL;
1184                         break;
1185                 }
1186
1187                 ifp->if_mtu = ifr->ifr_mtu;
1188                 adapter->max_frame_size =
1189                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1190                 em_init_locked(adapter);
1191                 EM_CORE_UNLOCK(adapter);
1192                 break;
1193             }
1194         case SIOCSIFFLAGS:
1195                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1196                     SIOCSIFFLAGS (Set Interface Flags)");
1197                 EM_CORE_LOCK(adapter);
1198                 if (ifp->if_flags & IFF_UP) {
1199                         if ((ifp->if_flags & IFF_RUNNING)) {
1200                                 if ((ifp->if_flags ^ adapter->if_flags) &
1201                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1202                                         em_disable_promisc(adapter);
1203                                         em_set_promisc(adapter);
1204                                 }
1205                         } else
1206                                 em_init_locked(adapter);
1207                 } else
1208                         if (ifp->if_flags & IFF_RUNNING) {
1209                                 EM_TX_LOCK(adapter);
1210                                 em_stop(adapter);
1211                                 EM_TX_UNLOCK(adapter);
1212                         }
1213                 adapter->if_flags = ifp->if_flags;
1214                 EM_CORE_UNLOCK(adapter);
1215                 break;
1216         case SIOCADDMULTI:
1217         case SIOCDELMULTI:
1218                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1219                 if (ifp->if_flags & IFF_RUNNING) {
1220                         EM_CORE_LOCK(adapter);
1221                         em_disable_intr(adapter);
1222                         em_set_multi(adapter);
1223                         if (adapter->hw.mac.type == e1000_82542 && 
1224                             adapter->hw.revision_id == E1000_REVISION_2) {
1225                                 em_initialize_receive_unit(adapter);
1226                         }
1227 #ifdef DEVICE_POLLING
1228                         if (!(ifp->if_capenable & IFCAP_POLLING))
1229 #endif
1230                                 em_enable_intr(adapter);
1231                         EM_CORE_UNLOCK(adapter);
1232                 }
1233                 break;
1234         case SIOCSIFMEDIA:
1235                 /* Check SOL/IDER usage */
1236                 EM_CORE_LOCK(adapter);
1237                 if (e1000_check_reset_block(&adapter->hw)) {
1238                         EM_CORE_UNLOCK(adapter);
1239                         device_printf(adapter->dev, "Media change is"
1240                             " blocked due to SOL/IDER session.\n");
1241                         break;
1242                 }
1243                 EM_CORE_UNLOCK(adapter);
1244         case SIOCGIFMEDIA:
1245                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1246                     SIOCxIFMEDIA (Get/Set Interface Media)");
1247                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1248                 break;
1249         case SIOCSIFCAP:
1250             {
1251                 int mask, reinit;
1252
1253                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1254                 reinit = 0;
1255                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1256 #ifdef DEVICE_POLLING
1257                 if (mask & IFCAP_POLLING) {
1258                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1259                                 error = ether_poll_register(em_poll, ifp);
1260                                 if (error)
1261                                         return (error);
1262                                 EM_CORE_LOCK(adapter);
1263                                 em_disable_intr(adapter);
1264                                 ifp->if_capenable |= IFCAP_POLLING;
1265                                 EM_CORE_UNLOCK(adapter);
1266                         } else {
1267                                 error = ether_poll_deregister(ifp);
1268                                 /* Enable interrupt even in error case */
1269                                 EM_CORE_LOCK(adapter);
1270                                 em_enable_intr(adapter);
1271                                 ifp->if_capenable &= ~IFCAP_POLLING;
1272                                 EM_CORE_UNLOCK(adapter);
1273                         }
1274                 }
1275 #endif
1276                 if (mask & IFCAP_HWCSUM) {
1277                         ifp->if_capenable ^= IFCAP_HWCSUM;
1278                         reinit = 1;
1279                 }
1280 #ifdef NET_TSO
1281                 if (mask & IFCAP_TSO4) {
1282                         ifp->if_capenable ^= IFCAP_TSO4;
1283                         reinit = 1;
1284                 }
1285 #endif
1286                 if (mask & IFCAP_VLAN_HWTAGGING) {
1287                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1288                         reinit = 1;
1289                 }
1290
1291                 if (mask & IFCAP_VLAN_HWFILTER) {
1292                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1293                         reinit = 1;
1294                 }
1295
1296                 if ((mask & IFCAP_WOL) &&
1297                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
1298                         if (mask & IFCAP_WOL_MCAST)
1299                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1300                         if (mask & IFCAP_WOL_MAGIC)
1301                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1302                 }
1303
1304                 if (reinit && (ifp->if_flags & IFF_RUNNING))
1305                         em_init(adapter);
1306 #if 0
1307                 VLAN_CAPABILITIES(ifp);
1308 #endif
1309                 break;
1310             }
1311
1312         default:
1313                 error = ether_ioctl(ifp, command, data);
1314                 break;
1315         }
1316
1317         return (error);
1318 }
1319
1320
1321 /*********************************************************************
1322  *  Init entry point
1323  *
1324  *  This routine is used in two ways. It is used by the stack as
1325  *  init entry point in network interface structure. It is also used
1326  *  by the driver as a hw/sw initialization routine to get to a
1327  *  consistent state.
1328  *
1329  *  return 0 on success, positive on failure
1330  **********************************************************************/
1331
1332 static void
1333 em_init_locked(struct adapter *adapter)
1334 {
1335         struct ifnet    *ifp = adapter->ifp;
1336         device_t        dev = adapter->dev;
1337         u32             pba;
1338
1339         INIT_DEBUGOUT("em_init: begin");
1340
1341         EM_CORE_LOCK_ASSERT(adapter);
1342
1343         EM_TX_LOCK(adapter);
1344         em_stop(adapter);
1345         EM_TX_UNLOCK(adapter);
1346
1347         /*
1348          * Packet Buffer Allocation (PBA)
1349          * Writing PBA sets the receive portion of the buffer
1350          * the remainder is used for the transmit buffer.
1351          *
1352          * Devices before the 82547 had a Packet Buffer of 64K.
1353          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1354          * After the 82547 the buffer was reduced to 40K.
1355          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1356          *   Note: default does not leave enough room for Jumbo Frame >10k.
1357          */
1358         switch (adapter->hw.mac.type) {
1359         case e1000_82547:
1360         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1361                 if (adapter->max_frame_size > 8192)
1362                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1363                 else
1364                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1365                 adapter->tx_fifo_head = 0;
1366                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1367                 adapter->tx_fifo_size =
1368                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1369                 break;
1370         /* Total Packet Buffer on these is 48K */
1371         case e1000_82571:
1372         case e1000_82572:
1373         case e1000_80003es2lan:
1374                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1375                 break;
1376         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1377                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1378                 break;
1379         case e1000_82574:
1380         case e1000_82583:
1381                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1382                 break;
1383         case e1000_ich9lan:
1384         case e1000_ich10lan:
1385         case e1000_pchlan:
1386                 pba = E1000_PBA_10K;
1387                 break;
1388         case e1000_ich8lan:
1389                 pba = E1000_PBA_8K;
1390                 break;
1391         default:
1392                 /* Devices before 82547 had a Packet Buffer of 64K.   */
1393                 if (adapter->max_frame_size > 8192)
1394                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1395                 else
1396                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1397         }
1398
1399         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1400         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1401         
1402         /* Get the latest mac address, User can use a LAA */
1403         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1404               ETHER_ADDR_LEN);
1405
1406         /* Put the address into the Receive Address Array */
1407         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1408
1409         /*
1410          * With the 82571 adapter, RAR[0] may be overwritten
1411          * when the other port is reset, we make a duplicate
1412          * in RAR[14] for that eventuality, this assures
1413          * the interface continues to function.
1414          */
1415         if (adapter->hw.mac.type == e1000_82571) {
1416                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1417                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1418                     E1000_RAR_ENTRIES - 1);
1419         }
1420
1421         /* Initialize the hardware */
1422         if (em_hardware_init(adapter)) {
1423                 device_printf(dev, "Unable to initialize the hardware\n");
1424                 return;
1425         }
1426         em_update_link_status(adapter);
1427
1428         /* Setup VLAN support, basic and offload if available */
1429         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1430         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1431                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1432                         /* Use real VLAN Filter support */
1433                         em_setup_vlan_hw_support(adapter);
1434                 else {
1435                         u32 ctrl;
1436                         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1437                         ctrl |= E1000_CTRL_VME;
1438                         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1439                 }
1440         }
1441
1442         /* Set hardware offload abilities */
1443         ifp->if_hwassist = 0;
1444         if (adapter->hw.mac.type >= e1000_82543) {
1445                 if (ifp->if_capenable & IFCAP_TXCSUM)
1446                         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1447 #ifdef NET_TSO
1448                 if (ifp->if_capenable & IFCAP_TSO4)
1449                         ifp->if_hwassist |= CSUM_TSO;
1450 #endif
1451         }
1452
1453         /* Configure for OS presence */
1454         em_init_manageability(adapter);
1455
1456         /* Prepare transmit descriptors and buffers */
1457         em_setup_transmit_structures(adapter);
1458         em_initialize_transmit_unit(adapter);
1459
1460         /* Setup Multicast table */
1461         em_set_multi(adapter);
1462
1463         /* Prepare receive descriptors and buffers */
1464         if (em_setup_receive_structures(adapter)) {
1465                 device_printf(dev, "Could not setup receive structures\n");
1466                 EM_TX_LOCK(adapter);
1467                 em_stop(adapter);
1468                 EM_TX_UNLOCK(adapter);
1469                 return;
1470         }
1471         em_initialize_receive_unit(adapter);
1472
1473         /* Don't lose promiscuous settings */
1474         em_set_promisc(adapter);
1475
1476         ifp->if_flags |= IFF_RUNNING;
1477         ifp->if_flags &= ~IFF_OACTIVE;
1478
1479         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1480         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1481
1482         /* MSI/X configuration for 82574 */
1483         if (adapter->hw.mac.type == e1000_82574) {
1484                 int tmp;
1485                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1486                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1487                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1488                 /*
1489                 ** Set the IVAR - interrupt vector routing.
1490                 ** Each nibble represents a vector, high bit
1491                 ** is enable, other 3 bits are the MSIX table
1492                 ** entry, we map RXQ0 to 0, TXQ0 to 1, and
1493                 ** Link (other) to 2, hence the magic number.
1494                 */
1495                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1496         }
1497
1498 #ifdef DEVICE_POLLING
1499         /*
1500          * Only enable interrupts if we are not polling, make sure
1501          * they are off otherwise.
1502          */
1503         if (ifp->if_capenable & IFCAP_POLLING)
1504                 em_disable_intr(adapter);
1505         else
1506 #endif /* DEVICE_POLLING */
1507                 em_enable_intr(adapter);
1508
1509         /* AMT based hardware can now take control from firmware */
1510         if (adapter->has_manage && adapter->has_amt)
1511                 em_get_hw_control(adapter);
1512
1513         /* Don't reset the phy next time init gets called */
1514         adapter->hw.phy.reset_disable = TRUE;
1515 }
1516
1517 static void
1518 em_init(void *arg)
1519 {
1520         struct adapter *adapter = arg;
1521
1522         EM_CORE_LOCK(adapter);
1523         em_init_locked(adapter);
1524         EM_CORE_UNLOCK(adapter);
1525 }
1526
1527
1528 #ifdef DEVICE_POLLING
1529 /*********************************************************************
1530  *
1531  *  Legacy polling routine  
1532  *
1533  *********************************************************************/
1534 static int
1535 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1536 {
1537         struct adapter *adapter = ifp->if_softc;
1538         u32             reg_icr, rx_done = 0;
1539
1540         EM_CORE_LOCK(adapter);
1541         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1542                 EM_CORE_UNLOCK(adapter);
1543                 return (rx_done);
1544         }
1545
1546         if (cmd == POLL_AND_CHECK_STATUS) {
1547                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1548                 /* Link status change */
1549                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1550                         adapter->hw.mac.get_link_status = 1;
1551                         em_update_link_status(adapter);
1552                 }
1553                 if (reg_icr & E1000_ICR_RXO)
1554                         adapter->rx_overruns++;
1555         }
1556         EM_CORE_UNLOCK(adapter);
1557
1558         rx_done = em_rxeof(adapter, count);
1559
1560         EM_TX_LOCK(adapter);
1561         em_txeof(adapter);
1562 #if 0
1563         if (!drbr_empty(ifp, adapter->br))
1564                 em_mq_start_locked(ifp, NULL);
1565 #else
1566         if (!ifq_is_empty(&ifp->if_snd))
1567                 em_start_locked(ifp);
1568 #endif
1569         EM_TX_UNLOCK(adapter);
1570         return (rx_done);
1571 }
1572 #endif /* DEVICE_POLLING */
1573
1574 #ifdef EM_LEGACY_IRQ 
1575 /*********************************************************************
1576  *
1577  *  Legacy Interrupt Service routine  
1578  *
1579  *********************************************************************/
1580
1581 static void
1582 em_intr(void *arg)
1583 {
1584         struct adapter  *adapter = arg;
1585         struct ifnet    *ifp = adapter->ifp;
1586         u32             reg_icr;
1587
1588
1589         if (ifp->if_capenable & IFCAP_POLLING)
1590                 return;
1591
1592         EM_CORE_LOCK(adapter);
1593         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1594         if (reg_icr & E1000_ICR_RXO)
1595                 adapter->rx_overruns++;
1596         if ((reg_icr == 0xffffffff) || (reg_icr == 0)||
1597             (adapter->hw.mac.type >= e1000_82571 &&
1598             (reg_icr & E1000_ICR_INT_ASSERTED) == 0))
1599                         goto out;
1600
1601         if ((ifp->if_flags & IFF_RUNNING) == 0)
1602                         goto out;
1603
1604         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1605                 callout_stop(&adapter->timer);
1606                 adapter->hw.mac.get_link_status = 1;
1607                 em_update_link_status(adapter);
1608                 /* Deal with TX cruft when link lost */
1609                 em_tx_purge(adapter);
1610                 callout_reset(&adapter->timer, hz,
1611                     em_local_timer, adapter);
1612                 goto out;
1613         }
1614
1615         EM_TX_LOCK(adapter);
1616         em_txeof(adapter);
1617         em_rxeof(adapter, -1);
1618         em_txeof(adapter);
1619         if (ifp->if_flags & IFF_RUNNING &&
1620             !ifq_is_empty(&ifp->if_snd))
1621                 em_start_locked(ifp);
1622         EM_TX_UNLOCK(adapter);
1623
1624 out:
1625         EM_CORE_UNLOCK(adapter);
1626         return;
1627 }
1628
1629 #else /* EM_FAST_IRQ, then fast interrupt routines only */
1630
1631 static void
1632 em_handle_link(void *context, int pending)
1633 {
1634         struct adapter  *adapter = context;
1635         struct ifnet *ifp = adapter->ifp;
1636
1637         if (!(ifp->if_flags & IFF_RUNNING))
1638                 return;
1639
1640         EM_CORE_LOCK(adapter);
1641         callout_stop(&adapter->timer);
1642         em_update_link_status(adapter);
1643         /* Deal with TX cruft when link lost */
1644         em_tx_purge(adapter);
1645         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1646         EM_CORE_UNLOCK(adapter);
1647 }
1648
1649
1650 /* Combined RX/TX handler, used by Legacy and MSI */
1651 static void
1652 em_handle_rxtx(void *context, int pending)
1653 {
1654         struct adapter  *adapter = context;
1655         struct ifnet    *ifp = adapter->ifp;
1656
1657
1658         if (ifp->if_flags & IFF_RUNNING) {
1659                 if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1660                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1661                 EM_TX_LOCK(adapter);
1662                 em_txeof(adapter);
1663 #if 0
1664                 if (!drbr_empty(ifp, adapter->br))
1665                         em_mq_start_locked(ifp, NULL);
1666 #else
1667                 if (!ifq_is_empty(&ifp->if_snd))
1668                         em_start_locked(ifp);
1669 #endif
1670                 EM_TX_UNLOCK(adapter);
1671         }
1672
1673         em_enable_intr(adapter);
1674 }
1675
1676 /*********************************************************************
1677  *
1678  *  Fast Legacy/MSI Combined Interrupt Service routine  
1679  *
1680  *********************************************************************/
1681 #define FILTER_STRAY
1682 #define FILTER_HANDLED
1683 static void
1684 em_irq_fast(void *arg)
1685 {
1686         struct adapter  *adapter = arg;
1687         struct ifnet    *ifp;
1688         u32             reg_icr;
1689
1690         ifp = adapter->ifp;
1691
1692         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1693
1694         /* Hot eject?  */
1695         if (reg_icr == 0xffffffff)
1696                 return FILTER_STRAY;
1697
1698         /* Definitely not our interrupt.  */
1699         if (reg_icr == 0x0)
1700                 return FILTER_STRAY;
1701
1702         /*
1703          * Starting with the 82571 chip, bit 31 should be used to
1704          * determine whether the interrupt belongs to us.
1705          */
1706         if (adapter->hw.mac.type >= e1000_82571 &&
1707             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1708                 return FILTER_STRAY;
1709
1710         /*
1711          * Mask interrupts until the taskqueue is finished running.  This is
1712          * cheap, just assume that it is needed.  This also works around the
1713          * MSI message reordering errata on certain systems.
1714          */
1715         em_disable_intr(adapter);
1716         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1717
1718         /* Link status change */
1719         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1720                 adapter->hw.mac.get_link_status = 1;
1721                 taskqueue_enqueue(taskqueue_swi, &adapter->link_task);
1722         }
1723
1724         if (reg_icr & E1000_ICR_RXO)
1725                 adapter->rx_overruns++;
1726         return FILTER_HANDLED;
1727 }
1728
1729 /*********************************************************************
1730  *
1731  *  MSIX Interrupt Service Routines
1732  *
1733  **********************************************************************/
1734 #define EM_MSIX_TX      0x00040000
1735 #define EM_MSIX_RX      0x00010000
1736 #define EM_MSIX_LINK    0x00100000
1737
1738 static void
1739 em_msix_tx(void *arg)
1740 {
1741         struct adapter *adapter = arg;
1742         struct ifnet    *ifp = adapter->ifp;
1743
1744         ++adapter->tx_irq;
1745         if (ifp->if_flags & IFF_RUNNING) {
1746                 EM_TX_LOCK(adapter);
1747                 em_txeof(adapter);
1748                 EM_TX_UNLOCK(adapter);
1749                 taskqueue_enqueue(adapter->tq, &adapter->tx_task);
1750         }
1751         /* Reenable this interrupt */
1752         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX);
1753         return;
1754 }
1755
1756 /*********************************************************************
1757  *
1758  *  MSIX RX Interrupt Service routine
1759  *
1760  **********************************************************************/
1761
1762 static void
1763 em_msix_rx(void *arg)
1764 {
1765         struct adapter *adapter = arg;
1766         struct ifnet    *ifp = adapter->ifp;
1767
1768         ++adapter->rx_irq;
1769         if ((ifp->if_flags & IFF_RUNNING) &&
1770             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1771                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1772         /* Reenable this interrupt */
1773         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX);
1774         return;
1775 }
1776
1777 /*********************************************************************
1778  *
1779  *  MSIX Link Fast Interrupt Service routine
1780  *
1781  **********************************************************************/
1782
1783 static void
1784 em_msix_link(void *arg)
1785 {
1786         struct adapter  *adapter = arg;
1787         u32             reg_icr;
1788
1789         ++adapter->link_irq;
1790         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1791
1792         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1793                 adapter->hw.mac.get_link_status = 1;
1794                 taskqueue_enqueue(taskqueue_swi, &adapter->link_task);
1795         }
1796         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1797             EM_MSIX_LINK | E1000_IMS_LSC);
1798         return;
1799 }
1800
1801 static void
1802 em_handle_rx(void *context, int pending)
1803 {
1804         struct adapter  *adapter = context;
1805         struct ifnet    *ifp = adapter->ifp;
1806
1807         if ((ifp->if_flags & IFF_RUNNING) &&
1808             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1809                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1810
1811 }
1812
1813 static void
1814 em_handle_tx(void *context, int pending)
1815 {
1816         struct adapter  *adapter = context;
1817         struct ifnet    *ifp = adapter->ifp;
1818
1819         if (ifp->if_flags & IFF_RUNNING) {
1820                 if (!EM_TX_TRYLOCK(adapter))
1821                         return;
1822                 em_txeof(adapter);
1823 #if 0
1824                 if (!drbr_empty(ifp, adapter->br))
1825                         em_mq_start_locked(ifp, NULL);
1826 #else
1827                 if (!ifq_is_empty(&ifp->if_snd))
1828                         em_start_locked(ifp);
1829 #endif
1830                 EM_TX_UNLOCK(adapter);
1831         }
1832 }
1833 #endif /* EM_FAST_IRQ */
1834
1835 /*********************************************************************
1836  *
1837  *  Media Ioctl callback
1838  *
1839  *  This routine is called whenever the user queries the status of
1840  *  the interface using ifconfig.
1841  *
1842  **********************************************************************/
1843 static void
1844 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1845 {
1846         struct adapter *adapter = ifp->if_softc;
1847         u_char fiber_type = IFM_1000_SX;
1848
1849         INIT_DEBUGOUT("em_media_status: begin");
1850
1851         EM_CORE_LOCK(adapter);
1852         em_update_link_status(adapter);
1853
1854         ifmr->ifm_status = IFM_AVALID;
1855         ifmr->ifm_active = IFM_ETHER;
1856
1857         if (!adapter->link_active) {
1858                 EM_CORE_UNLOCK(adapter);
1859                 return;
1860         }
1861
1862         ifmr->ifm_status |= IFM_ACTIVE;
1863
1864         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1865             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1866                 if (adapter->hw.mac.type == e1000_82545)
1867                         fiber_type = IFM_1000_LX;
1868                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1869         } else {
1870                 switch (adapter->link_speed) {
1871                 case 10:
1872                         ifmr->ifm_active |= IFM_10_T;
1873                         break;
1874                 case 100:
1875                         ifmr->ifm_active |= IFM_100_TX;
1876                         break;
1877                 case 1000:
1878                         ifmr->ifm_active |= IFM_1000_T;
1879                         break;
1880                 }
1881                 if (adapter->link_duplex == FULL_DUPLEX)
1882                         ifmr->ifm_active |= IFM_FDX;
1883                 else
1884                         ifmr->ifm_active |= IFM_HDX;
1885         }
1886         EM_CORE_UNLOCK(adapter);
1887 }
1888
1889 /*********************************************************************
1890  *
1891  *  Media Ioctl callback
1892  *
1893  *  This routine is called when the user changes speed/duplex using
1894  *  media/mediopt option with ifconfig.
1895  *
1896  **********************************************************************/
1897 static int
1898 em_media_change(struct ifnet *ifp)
1899 {
1900         struct adapter *adapter = ifp->if_softc;
1901         struct ifmedia  *ifm = &adapter->media;
1902
1903         INIT_DEBUGOUT("em_media_change: begin");
1904
1905         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1906                 return (EINVAL);
1907
1908         EM_CORE_LOCK(adapter);
1909         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1910         case IFM_AUTO:
1911                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1912                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1913                 break;
1914         case IFM_1000_LX:
1915         case IFM_1000_SX:
1916         case IFM_1000_T:
1917                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1918                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1919                 break;
1920         case IFM_100_TX:
1921                 adapter->hw.mac.autoneg = FALSE;
1922                 adapter->hw.phy.autoneg_advertised = 0;
1923                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1924                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1925                 else
1926                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1927                 break;
1928         case IFM_10_T:
1929                 adapter->hw.mac.autoneg = FALSE;
1930                 adapter->hw.phy.autoneg_advertised = 0;
1931                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1932                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1933                 else
1934                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1935                 break;
1936         default:
1937                 device_printf(adapter->dev, "Unsupported media type\n");
1938         }
1939
1940         /* As the speed/duplex settings my have changed we need to
1941          * reset the PHY.
1942          */
1943         adapter->hw.phy.reset_disable = FALSE;
1944
1945         em_init_locked(adapter);
1946         EM_CORE_UNLOCK(adapter);
1947
1948         return (0);
1949 }
1950
1951 /*********************************************************************
1952  *
1953  *  This routine maps the mbufs to tx descriptors.
1954  *
1955  *  return 0 on success, positive on failure
1956  **********************************************************************/
1957
1958 static int
1959 em_xmit(struct adapter *adapter, struct mbuf **m_headp)
1960 {
1961         bus_dma_segment_t       segs[EM_MAX_SCATTER];
1962         bus_dmamap_t            map;
1963         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
1964         struct e1000_tx_desc    *ctxd = NULL;
1965         struct mbuf             *m_head;
1966         u32                     txd_upper, txd_lower, txd_used, txd_saved;
1967         int                     nsegs, i, j, first, last = 0;
1968         int                     error, do_tso, tso_desc = 0;
1969
1970         m_head = *m_headp;
1971         txd_upper = txd_lower = txd_used = txd_saved = 0;
1972
1973 #ifdef NET_TSO
1974         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1975 #else
1976         do_tso = 0;
1977 #endif
1978
1979         /*
1980          * Force a cleanup if number of TX descriptors
1981          * available hits the threshold
1982          */
1983         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1984                 em_txeof(adapter);
1985                 /* Now do we at least have a minimal? */
1986                 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
1987                         adapter->no_tx_desc_avail1++;
1988                         return (ENOBUFS);
1989                 }
1990         }
1991
1992
1993         /*
1994          * TSO workaround: 
1995          *  If an mbuf is only header we need  
1996          *     to pull 4 bytes of data into it. 
1997          */
1998         if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
1999                 m_head = m_pullup(m_head, M_TSO_LEN + 4);
2000                 *m_headp = m_head;
2001                 if (m_head == NULL)
2002                         return (ENOBUFS);
2003         }
2004
2005         /*
2006          * Map the packet for DMA
2007          *
2008          * Capture the first descriptor index,
2009          * this descriptor will have the index
2010          * of the EOP which is the only one that
2011          * now gets a DONE bit writeback.
2012          */
2013         first = adapter->next_avail_tx_desc;
2014         tx_buffer = &adapter->tx_buffer_area[first];
2015         tx_buffer_mapped = tx_buffer;
2016         map = tx_buffer->map;
2017
2018         error = bus_dmamap_load_mbuf_segment(adapter->txtag, map,
2019             *m_headp, segs, EM_MAX_SCATTER, &nsegs, BUS_DMA_NOWAIT);
2020
2021         /*
2022          * There are two types of errors we can (try) to handle:
2023          * - EFBIG means the mbuf chain was too long and bus_dma ran
2024          *   out of segments.  Defragment the mbuf chain and try again.
2025          * - ENOMEM means bus_dma could not obtain enough bounce buffers
2026          *   at this point in time.  Defer sending and try again later.
2027          * All other errors, in particular EINVAL, are fatal and prevent the
2028          * mbuf chain from ever going through.  Drop it and report error.
2029          */
2030         if (error == EFBIG) {
2031                 struct mbuf *m;
2032
2033                 m = m_defrag(*m_headp, MB_DONTWAIT);
2034                 if (m == NULL) {
2035                         adapter->mbuf_alloc_failed++;
2036                         m_freem(*m_headp);
2037                         *m_headp = NULL;
2038                         return (ENOBUFS);
2039                 }
2040                 *m_headp = m;
2041
2042                 /* Try it again */
2043                 error = bus_dmamap_load_mbuf_segment(adapter->txtag, map,
2044                     *m_headp, segs, EM_MAX_SCATTER, &nsegs, BUS_DMA_NOWAIT);
2045
2046                 if (error) {
2047                         adapter->no_tx_dma_setup++;
2048                         m_freem(*m_headp);
2049                         *m_headp = NULL;
2050                         return (error);
2051                 }
2052         } else if (error != 0) {
2053                 adapter->no_tx_dma_setup++;
2054                 return (error);
2055         }
2056
2057         /*
2058          * TSO Hardware workaround, if this packet is not
2059          * TSO, and is only a single descriptor long, and
2060          * it follows a TSO burst, then we need to add a
2061          * sentinel descriptor to prevent premature writeback.
2062          */
2063         if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
2064                 if (nsegs == 1)
2065                         tso_desc = TRUE;
2066                 adapter->tx_tso = FALSE;
2067         }
2068
2069         if (nsegs > (adapter->num_tx_desc_avail - 2)) {
2070                 adapter->no_tx_desc_avail2++;
2071                 bus_dmamap_unload(adapter->txtag, map);
2072                 return (ENOBUFS);
2073         }
2074         m_head = *m_headp;
2075
2076         /* Do hardware assists */
2077 #ifdef NET_TSO
2078         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2079                 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
2080                 if (error != TRUE)
2081                         return (ENXIO); /* something foobar */
2082                 /* we need to make a final sentinel transmit desc */
2083                 tso_desc = TRUE;
2084         } else
2085 #endif
2086         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2087                 em_transmit_checksum_setup(adapter,  m_head,
2088                     &txd_upper, &txd_lower);
2089
2090         i = adapter->next_avail_tx_desc;
2091         if (adapter->pcix_82544) 
2092                 txd_saved = i;
2093
2094         /* Set up our transmit descriptors */
2095         for (j = 0; j < nsegs; j++) {
2096                 bus_size_t seg_len;
2097                 bus_addr_t seg_addr;
2098                 /* If adapter is 82544 and on PCIX bus */
2099                 if(adapter->pcix_82544) {
2100                         DESC_ARRAY      desc_array;
2101                         u32             array_elements, counter;
2102                         /*
2103                          * Check the Address and Length combination and
2104                          * split the data accordingly
2105                          */
2106                         array_elements = em_fill_descriptors(segs[j].ds_addr,
2107                             segs[j].ds_len, &desc_array);
2108                         for (counter = 0; counter < array_elements; counter++) {
2109                                 if (txd_used == adapter->num_tx_desc_avail) {
2110                                         adapter->next_avail_tx_desc = txd_saved;
2111                                         adapter->no_tx_desc_avail2++;
2112                                         bus_dmamap_unload(adapter->txtag, map);
2113                                         return (ENOBUFS);
2114                                 }
2115                                 tx_buffer = &adapter->tx_buffer_area[i];
2116                                 ctxd = &adapter->tx_desc_base[i];
2117                                 ctxd->buffer_addr = htole64(
2118                                     desc_array.descriptor[counter].address);
2119                                 ctxd->lower.data = htole32(
2120                                     (adapter->txd_cmd | txd_lower | (u16)
2121                                     desc_array.descriptor[counter].length));
2122                                 ctxd->upper.data =
2123                                     htole32((txd_upper));
2124                                 last = i;
2125                                 if (++i == adapter->num_tx_desc)
2126                                          i = 0;
2127                                 tx_buffer->m_head = NULL;
2128                                 tx_buffer->next_eop = -1;
2129                                 txd_used++;
2130                         }
2131                 } else {
2132                         tx_buffer = &adapter->tx_buffer_area[i];
2133                         ctxd = &adapter->tx_desc_base[i];
2134                         seg_addr = segs[j].ds_addr;
2135                         seg_len  = segs[j].ds_len;
2136                         /*
2137                         ** TSO Workaround:
2138                         ** If this is the last descriptor, we want to
2139                         ** split it so we have a small final sentinel
2140                         */
2141                         if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2142                                 seg_len -= 4;
2143                                 ctxd->buffer_addr = htole64(seg_addr);
2144                                 ctxd->lower.data = htole32(
2145                                 adapter->txd_cmd | txd_lower | seg_len);
2146                                 ctxd->upper.data =
2147                                     htole32(txd_upper);
2148                                 if (++i == adapter->num_tx_desc)
2149                                         i = 0;
2150                                 /* Now make the sentinel */     
2151                                 ++txd_used; /* using an extra txd */
2152                                 ctxd = &adapter->tx_desc_base[i];
2153                                 tx_buffer = &adapter->tx_buffer_area[i];
2154                                 ctxd->buffer_addr =
2155                                     htole64(seg_addr + seg_len);
2156                                 ctxd->lower.data = htole32(
2157                                 adapter->txd_cmd | txd_lower | 4);
2158                                 ctxd->upper.data =
2159                                     htole32(txd_upper);
2160                                 last = i;
2161                                 if (++i == adapter->num_tx_desc)
2162                                         i = 0;
2163                         } else {
2164                                 ctxd->buffer_addr = htole64(seg_addr);
2165                                 ctxd->lower.data = htole32(
2166                                 adapter->txd_cmd | txd_lower | seg_len);
2167                                 ctxd->upper.data =
2168                                     htole32(txd_upper);
2169                                 last = i;
2170                                 if (++i == adapter->num_tx_desc)
2171                                         i = 0;
2172                         }
2173                         tx_buffer->m_head = NULL;
2174                         tx_buffer->next_eop = -1;
2175                 }
2176         }
2177
2178         adapter->next_avail_tx_desc = i;
2179         if (adapter->pcix_82544)
2180                 adapter->num_tx_desc_avail -= txd_used;
2181         else {
2182                 adapter->num_tx_desc_avail -= nsegs;
2183                 if (tso_desc) /* TSO used an extra for sentinel */
2184                         adapter->num_tx_desc_avail -= txd_used;
2185         }
2186
2187         /*
2188         ** Handle VLAN tag
2189         */
2190         if (m_head->m_flags & M_VLANTAG) {
2191                 /* Set the vlan id. */
2192                 ctxd->upper.fields.special =
2193                     htole16(m_head->m_pkthdr.ether_vlantag);
2194                 /* Tell hardware to add tag */
2195                 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
2196         }
2197
2198         tx_buffer->m_head = m_head;
2199         tx_buffer_mapped->map = tx_buffer->map;
2200         tx_buffer->map = map;
2201         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
2202
2203         /*
2204          * Last Descriptor of Packet
2205          * needs End Of Packet (EOP)
2206          * and Report Status (RS)
2207          */
2208         ctxd->lower.data |=
2209             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2210         /*
2211          * Keep track in the first buffer which
2212          * descriptor will be written back
2213          */
2214         tx_buffer = &adapter->tx_buffer_area[first];
2215         tx_buffer->next_eop = last;
2216         adapter->watchdog_time = ticks;
2217
2218         /*
2219          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2220          * that this frame is available to transmit.
2221          */
2222         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2223             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2224         if (adapter->hw.mac.type == e1000_82547 &&
2225             adapter->link_duplex == HALF_DUPLEX)
2226                 em_82547_move_tail(adapter);
2227         else {
2228                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
2229                 if (adapter->hw.mac.type == e1000_82547)
2230                         em_82547_update_fifo_head(adapter,
2231                             m_head->m_pkthdr.len);
2232         }
2233
2234         return (0);
2235 }
2236
2237 /*********************************************************************
2238  *
2239  * 82547 workaround to avoid controller hang in half-duplex environment.
2240  * The workaround is to avoid queuing a large packet that would span
2241  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
2242  * in this case. We do that only when FIFO is quiescent.
2243  *
2244  **********************************************************************/
2245 static void
2246 em_82547_move_tail_locked(void *arg)
2247 {
2248         struct adapter *adapter = arg;
2249
2250         struct e1000_tx_desc *tx_desc;
2251         u16     hw_tdt, sw_tdt, length = 0;
2252         bool    eop = 0;
2253
2254         EM_TX_LOCK_ASSERT(adapter);
2255
2256         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2257         sw_tdt = adapter->next_avail_tx_desc;
2258         
2259         while (hw_tdt != sw_tdt) {
2260                 tx_desc = &adapter->tx_desc_base[hw_tdt];
2261                 length += tx_desc->lower.flags.length;
2262                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2263                 if (++hw_tdt == adapter->num_tx_desc)
2264                         hw_tdt = 0;
2265
2266                 if (eop) {
2267                         if (em_82547_fifo_workaround(adapter, length)) {
2268                                 adapter->tx_fifo_wrk_cnt++;
2269                                 callout_reset(&adapter->tx_fifo_timer, 1,
2270                                         em_82547_move_tail, adapter);
2271                                 break;
2272                         }
2273                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2274                         em_82547_update_fifo_head(adapter, length);
2275                         length = 0;
2276                 }
2277         }       
2278 }
2279
2280 static void
2281 em_82547_move_tail(void *arg)
2282 {
2283         struct adapter *adapter = arg;
2284         EM_TX_LOCK(adapter);
2285         em_82547_move_tail_locked(arg);
2286         EM_TX_UNLOCK(adapter);
2287 }
2288
2289 static int
2290 em_82547_fifo_workaround(struct adapter *adapter, int len)
2291 {       
2292         int fifo_space, fifo_pkt_len;
2293
2294         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2295
2296         if (adapter->link_duplex == HALF_DUPLEX) {
2297                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2298
2299                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2300                         if (em_82547_tx_fifo_reset(adapter))
2301                                 return (0);
2302                         else
2303                                 return (1);
2304                 }
2305         }
2306
2307         return (0);
2308 }
2309
2310 static void
2311 em_82547_update_fifo_head(struct adapter *adapter, int len)
2312 {
2313         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2314         
2315         /* tx_fifo_head is always 16 byte aligned */
2316         adapter->tx_fifo_head += fifo_pkt_len;
2317         if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
2318                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2319         }
2320 }
2321
2322
2323 static int
2324 em_82547_tx_fifo_reset(struct adapter *adapter)
2325 {
2326         u32 tctl;
2327
2328         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2329             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2330             (E1000_READ_REG(&adapter->hw, E1000_TDFT) == 
2331             E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2332             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2333             E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2334             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2335                 /* Disable TX unit */
2336                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2337                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2338                     tctl & ~E1000_TCTL_EN);
2339
2340                 /* Reset FIFO pointers */
2341                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2342                     adapter->tx_head_addr);
2343                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2344                     adapter->tx_head_addr);
2345                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2346                     adapter->tx_head_addr);
2347                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2348                     adapter->tx_head_addr);
2349
2350                 /* Re-enable TX unit */
2351                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2352                 E1000_WRITE_FLUSH(&adapter->hw);
2353
2354                 adapter->tx_fifo_head = 0;
2355                 adapter->tx_fifo_reset_cnt++;
2356
2357                 return (TRUE);
2358         }
2359         else {
2360                 return (FALSE);
2361         }
2362 }
2363
2364 static void
2365 em_set_promisc(struct adapter *adapter)
2366 {
2367         struct ifnet    *ifp = adapter->ifp;
2368         u32             reg_rctl;
2369
2370         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2371
2372         if (ifp->if_flags & IFF_PROMISC) {
2373                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2374                 /* Turn this on if you want to see bad packets */
2375                 if (em_debug_sbp)
2376                         reg_rctl |= E1000_RCTL_SBP;
2377                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2378         } else if (ifp->if_flags & IFF_ALLMULTI) {
2379                 reg_rctl |= E1000_RCTL_MPE;
2380                 reg_rctl &= ~E1000_RCTL_UPE;
2381                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2382         }
2383 }
2384
2385 static void
2386 em_disable_promisc(struct adapter *adapter)
2387 {
2388         u32     reg_rctl;
2389
2390         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2391
2392         reg_rctl &=  (~E1000_RCTL_UPE);
2393         reg_rctl &=  (~E1000_RCTL_MPE);
2394         reg_rctl &=  (~E1000_RCTL_SBP);
2395         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2396 }
2397
2398
2399 /*********************************************************************
2400  *  Multicast Update
2401  *
2402  *  This routine is called whenever multicast address list is updated.
2403  *
2404  **********************************************************************/
2405
2406 static void
2407 em_set_multi(struct adapter *adapter)
2408 {
2409         struct ifnet    *ifp = adapter->ifp;
2410         struct ifmultiaddr *ifma;
2411         u32 reg_rctl = 0;
2412         u8  *mta; /* Multicast array memory */
2413         int mcnt = 0;
2414
2415         IOCTL_DEBUGOUT("em_set_multi: begin");
2416
2417         if (adapter->hw.mac.type == e1000_82542 && 
2418             adapter->hw.revision_id == E1000_REVISION_2) {
2419                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2420                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2421                         e1000_pci_clear_mwi(&adapter->hw);
2422                 reg_rctl |= E1000_RCTL_RST;
2423                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2424                 msec_delay(5);
2425         }
2426
2427         /* Allocate temporary memory to setup array */
2428         mta = kmalloc(sizeof(u8) *
2429             (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
2430             M_DEVBUF, M_INTWAIT | M_ZERO);
2431         if (mta == NULL)
2432                 panic("em_set_multi memory failure\n");
2433
2434 #if 0
2435 #if __FreeBSD_version < 800000
2436         IF_ADDR_LOCK(ifp);
2437 #else
2438         if_maddr_rlock(ifp);
2439 #endif
2440 #endif
2441         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2442                 if (ifma->ifma_addr->sa_family != AF_LINK)
2443                         continue;
2444
2445                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2446                         break;
2447
2448                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2449                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2450                 mcnt++;
2451         }
2452 #if 0
2453 #if __FreeBSD_version < 800000
2454         IF_ADDR_UNLOCK(ifp);
2455 #else
2456         if_maddr_runlock(ifp);
2457 #endif
2458 #endif
2459         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2460                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2461                 reg_rctl |= E1000_RCTL_MPE;
2462                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2463         } else
2464                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2465
2466         if (adapter->hw.mac.type == e1000_82542 && 
2467             adapter->hw.revision_id == E1000_REVISION_2) {
2468                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2469                 reg_rctl &= ~E1000_RCTL_RST;
2470                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2471                 msec_delay(5);
2472                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2473                         e1000_pci_set_mwi(&adapter->hw);
2474         }
2475         kfree(mta, M_DEVBUF);
2476 }
2477
2478
2479 /*********************************************************************
2480  *  Timer routine
2481  *
2482  *  This routine checks for link status and updates statistics.
2483  *
2484  **********************************************************************/
2485
2486 static void
2487 em_local_timer_locked(void *arg)
2488 {
2489         struct adapter  *adapter = arg;
2490         struct ifnet    *ifp = adapter->ifp;
2491
2492         EM_CORE_LOCK_ASSERT(adapter);
2493
2494 #ifndef DEVICE_POLLING
2495         taskqueue_enqueue(adapter->tq,
2496             &adapter->rxtx_task);
2497 #endif
2498         em_update_link_status(adapter);
2499         em_update_stats_counters(adapter);
2500
2501         /* Reset LAA into RAR[0] on 82571 */
2502         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2503                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2504
2505         if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
2506                 em_print_hw_stats(adapter);
2507
2508         em_smartspeed(adapter);
2509
2510         /*
2511          * We check the watchdog: the time since
2512          * the last TX descriptor was cleaned.
2513          * This implies a functional TX engine.
2514          */
2515         if ((adapter->watchdog_check == TRUE) &&
2516             (ticks - adapter->watchdog_time > EM_WATCHDOG))
2517                 goto hung;
2518
2519         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2520         return;
2521 hung:
2522         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2523         adapter->ifp->if_flags &= ~IFF_RUNNING;
2524         adapter->watchdog_events++;
2525         em_init_locked(adapter);
2526 }
2527
2528 static void
2529 em_local_timer(void *arg)
2530 {
2531         struct adapter  *adapter = arg;
2532
2533         EM_CORE_LOCK(adapter);
2534         em_local_timer_locked(arg);
2535         EM_CORE_UNLOCK(adapter);
2536 }
2537
2538
2539 static void
2540 em_update_link_status(struct adapter *adapter)
2541 {
2542         struct e1000_hw *hw = &adapter->hw;
2543         struct ifnet *ifp = adapter->ifp;
2544         device_t dev = adapter->dev;
2545         u32 link_check = 0;
2546
2547         /* Get the cached link value or read phy for real */
2548         switch (hw->phy.media_type) {
2549         case e1000_media_type_copper:
2550                 if (hw->mac.get_link_status) {
2551                         /* Do the work to read phy */
2552                         e1000_check_for_link(hw);
2553                         link_check = !hw->mac.get_link_status;
2554                         if (link_check) /* ESB2 fix */
2555                                 e1000_cfg_on_link_up(hw);
2556                 } else
2557                         link_check = TRUE;
2558                 break;
2559         case e1000_media_type_fiber:
2560                 e1000_check_for_link(hw);
2561                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2562                                  E1000_STATUS_LU);
2563                 break;
2564         case e1000_media_type_internal_serdes:
2565                 e1000_check_for_link(hw);
2566                 link_check = adapter->hw.mac.serdes_has_link;
2567                 break;
2568         default:
2569         case e1000_media_type_unknown:
2570                 break;
2571         }
2572
2573         /* Now check for a transition */
2574         if (link_check && (adapter->link_active == 0)) {
2575                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2576                     &adapter->link_duplex);
2577                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2578                 if ((adapter->link_speed != SPEED_1000) &&
2579                     ((hw->mac.type == e1000_82571) ||
2580                     (hw->mac.type == e1000_82572))) {
2581                         int tarc0;
2582                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2583                         tarc0 &= ~SPEED_MODE_BIT;
2584                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2585                 }
2586                 if (bootverbose)
2587                         device_printf(dev, "Link is up %d Mbps %s\n",
2588                             adapter->link_speed,
2589                             ((adapter->link_duplex == FULL_DUPLEX) ?
2590                             "Full Duplex" : "Half Duplex"));
2591                 adapter->link_active = 1;
2592                 adapter->smartspeed = 0;
2593                 ifp->if_baudrate = adapter->link_speed * 1000000;
2594                 ifp->if_link_state = LINK_STATE_UP;
2595                 if_link_state_change(ifp);
2596         } else if (!link_check && (adapter->link_active == 1)) {
2597                 ifp->if_baudrate = adapter->link_speed = 0;
2598                 adapter->link_duplex = 0;
2599                 if (bootverbose)
2600                         device_printf(dev, "Link is Down\n");
2601                 adapter->link_active = 0;
2602                 /* Link down, disable watchdog */
2603                 adapter->watchdog_check = FALSE;
2604                 ifp->if_link_state = LINK_STATE_DOWN;
2605                 if_link_state_change(ifp);
2606         }
2607 }
2608
2609 /*********************************************************************
2610  *
2611  *  This routine disables all traffic on the adapter by issuing a
2612  *  global reset on the MAC and deallocates TX/RX buffers.
2613  *
2614  *  This routine should always be called with BOTH the CORE
2615  *  and TX locks.
2616  **********************************************************************/
2617
2618 static void
2619 em_stop(void *arg)
2620 {
2621         struct adapter  *adapter = arg;
2622         struct ifnet    *ifp = adapter->ifp;
2623
2624         EM_CORE_LOCK_ASSERT(adapter);
2625         EM_TX_LOCK_ASSERT(adapter);
2626
2627         INIT_DEBUGOUT("em_stop: begin");
2628
2629         em_disable_intr(adapter);
2630         callout_stop(&adapter->timer);
2631         callout_stop(&adapter->tx_fifo_timer);
2632
2633         /* Tell the stack that the interface is no longer active */
2634         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
2635
2636         e1000_reset_hw(&adapter->hw);
2637         if (adapter->hw.mac.type >= e1000_82544)
2638                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2639 }
2640
2641
2642 /*********************************************************************
2643  *
2644  *  Determine hardware revision.
2645  *
2646  **********************************************************************/
2647 static void
2648 em_identify_hardware(struct adapter *adapter)
2649 {
2650         device_t dev = adapter->dev;
2651
2652         /* Make sure our PCI config space has the necessary stuff set */
2653         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2654         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2655             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2656                 device_printf(dev, "Memory Access and/or Bus Master bits "
2657                     "were not set!\n");
2658                 adapter->hw.bus.pci_cmd_word |=
2659                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2660                 pci_write_config(dev, PCIR_COMMAND,
2661                     adapter->hw.bus.pci_cmd_word, 2);
2662         }
2663
2664         /* Save off the information about this board */
2665         adapter->hw.vendor_id = pci_get_vendor(dev);
2666         adapter->hw.device_id = pci_get_device(dev);
2667         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2668         adapter->hw.subsystem_vendor_id =
2669             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2670         adapter->hw.subsystem_device_id =
2671             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2672
2673         /* Do Shared Code Init and Setup */
2674         if (e1000_set_mac_type(&adapter->hw)) {
2675                 device_printf(dev, "Setup init failure\n");
2676                 return;
2677         }
2678 }
2679
2680 static int
2681 em_allocate_pci_resources(struct adapter *adapter)
2682 {
2683         device_t        dev = adapter->dev;
2684         int             val, rid, error = E1000_SUCCESS;
2685
2686         rid = PCIR_BAR(0);
2687         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2688             &rid, RF_ACTIVE);
2689         if (adapter->memory == NULL) {
2690                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2691                 return (ENXIO);
2692         }
2693         adapter->osdep.mem_bus_space_tag =
2694             rman_get_bustag(adapter->memory);
2695         adapter->osdep.mem_bus_space_handle =
2696             rman_get_bushandle(adapter->memory);
2697         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2698
2699         /* Only older adapters use IO mapping */
2700         if ((adapter->hw.mac.type > e1000_82543) &&
2701             (adapter->hw.mac.type < e1000_82571)) {
2702                 /* Figure our where our IO BAR is ? */
2703                 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2704                         val = pci_read_config(dev, rid, 4);
2705                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2706                                 adapter->io_rid = rid;
2707                                 break;
2708                         }
2709                         rid += 4;
2710                         /* check for 64bit BAR */
2711                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2712                                 rid += 4;
2713                 }
2714                 if (rid >= PCIR_CIS) {
2715                         device_printf(dev, "Unable to locate IO BAR\n");
2716                         return (ENXIO);
2717                 }
2718                 adapter->ioport = bus_alloc_resource_any(dev,
2719                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2720                 if (adapter->ioport == NULL) {
2721                         device_printf(dev, "Unable to allocate bus resource: "
2722                             "ioport\n");
2723                         return (ENXIO);
2724                 }
2725                 adapter->hw.io_base = 0;
2726                 adapter->osdep.io_bus_space_tag =
2727                     rman_get_bustag(adapter->ioport);
2728                 adapter->osdep.io_bus_space_handle =
2729                     rman_get_bushandle(adapter->ioport);
2730         }
2731
2732         /*
2733         ** Init the resource arrays
2734         **  used by MSIX setup 
2735         */
2736         for (int i = 0; i < 3; i++) {
2737                 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2738                 adapter->tag[i] = NULL;
2739                 adapter->res[i] = NULL;
2740         }
2741
2742         /*
2743          * Setup MSI/X or MSI if PCI Express
2744          */
2745         if (em_enable_msi)
2746                 adapter->msi = em_setup_msix(adapter);
2747
2748         adapter->hw.back = &adapter->osdep;
2749
2750         return (error);
2751 }
2752
2753 /*********************************************************************
2754  *
2755  *  Setup the Legacy or MSI Interrupt handler
2756  *
2757  **********************************************************************/
2758 int
2759 em_allocate_legacy(struct adapter *adapter)
2760 {
2761         device_t dev = adapter->dev;
2762         int error;
2763
2764         /* Manually turn off all interrupts */
2765         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2766
2767         /* Legacy RID is 0 */
2768         if (adapter->msi == 0)
2769                 adapter->rid[0] = 0;
2770
2771         /* We allocate a single interrupt resource */
2772         adapter->res[0] = bus_alloc_resource_any(dev,
2773             SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2774         if (adapter->res[0] == NULL) {
2775                 device_printf(dev, "Unable to allocate bus resource: "
2776                     "interrupt\n");
2777                 return (ENXIO);
2778         }
2779
2780 #ifdef EM_LEGACY_IRQ
2781         /* We do Legacy setup */
2782         if ((error = bus_setup_intr(dev, adapter->res[0],
2783             /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_intr, adapter,
2784             &adapter->tag[0], NULL)) != 0) {
2785                 device_printf(dev, "Failed to register interrupt handler");
2786                 return (error);
2787         }
2788
2789 #else /* FAST_IRQ */
2790         /*
2791          * Try allocating a fast interrupt and the associated deferred
2792          * processing contexts.
2793          */
2794         TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2795         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2796         adapter->tq = taskqueue_create("em_taskq", M_INTWAIT,
2797             taskqueue_thread_enqueue, &adapter->tq);
2798         taskqueue_start_threads(&adapter->tq, 1, TDPRI_KERN_DAEMON /*PI_NET*/, -1, "%s taskq",
2799             device_get_nameunit(adapter->dev));
2800         if ((error = bus_setup_intr(dev, adapter->res[0],
2801             /*INTR_TYPE_NET |*/ 0, em_irq_fast, adapter,
2802             &adapter->tag[0], NULL)) != 0) {
2803                 device_printf(dev, "Failed to register fast interrupt "
2804                             "handler: %d\n", error);
2805                 taskqueue_free(adapter->tq);
2806                 adapter->tq = NULL;
2807                 return (error);
2808         }
2809 #endif  /* EM_LEGACY_IRQ */
2810         
2811         return (0);
2812 }
2813
2814 /*********************************************************************
2815  *
2816  *  Setup the MSIX Interrupt handlers
2817  *   This is not really Multiqueue, rather
2818  *   its just multiple interrupt vectors.
2819  *
2820  **********************************************************************/
2821 int
2822 em_allocate_msix(struct adapter *adapter)
2823 {
2824         device_t dev = adapter->dev;
2825         int error;
2826
2827         /* Make sure all interrupts are disabled */
2828         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2829
2830         /* First get the resources */
2831         for (int i = 0; i < adapter->msi; i++) {
2832                 adapter->res[i] = bus_alloc_resource_any(dev,
2833                     SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE);
2834                 if (adapter->res[i] == NULL) {
2835                         device_printf(dev,
2836                             "Unable to allocate bus resource: "
2837                             "MSIX Interrupt\n");
2838                         return (ENXIO);
2839                 }
2840         }
2841
2842         /*
2843          * Now allocate deferred processing contexts.
2844          */
2845         TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter);
2846         TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter);
2847         /*
2848          * Handle compatibility for msi case for deferral due to
2849          * trylock failure
2850          */
2851         TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter);
2852         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2853         adapter->tq = taskqueue_create("em_taskq", M_INTWAIT,
2854             taskqueue_thread_enqueue, &adapter->tq);
2855         taskqueue_start_threads(&adapter->tq, 1, TDPRI_KERN_DAEMON /*PI_NET*/, -1, "%s taskq",
2856             device_get_nameunit(adapter->dev));
2857
2858         /*
2859          * And setup the interrupt handlers
2860          */
2861
2862         /* First slot to RX */
2863         if ((error = bus_setup_intr(dev, adapter->res[0],
2864             /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_msix_rx, adapter,
2865             &adapter->tag[0], NULL)) != 0) {
2866                 device_printf(dev, "Failed to register RX handler");
2867                 return (error);
2868         }
2869
2870         /* Next TX */
2871         if ((error = bus_setup_intr(dev, adapter->res[1],
2872             /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_msix_tx, adapter,
2873             &adapter->tag[1], NULL)) != 0) {
2874                 device_printf(dev, "Failed to register TX handler");
2875                 return (error);
2876         }
2877
2878         /* And Link */
2879         if ((error = bus_setup_intr(dev, adapter->res[2],
2880             /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_msix_link, adapter,
2881             &adapter->tag[2], NULL)) != 0) {
2882                 device_printf(dev, "Failed to register TX handler");
2883                 return (error);
2884         }
2885
2886         return (0);
2887 }
2888
2889
2890 static void
2891 em_free_pci_resources(struct adapter *adapter)
2892 {
2893         device_t dev = adapter->dev;
2894
2895         /* Make sure the for loop below runs once */
2896         if (adapter->msi == 0)
2897                 adapter->msi = 1;
2898
2899         /*
2900          * First release all the interrupt resources:
2901          *      notice that since these are just kept
2902          *      in an array we can do the same logic
2903          *      whether its MSIX or just legacy.
2904          */
2905         for (int i = 0; i < adapter->msi; i++) {
2906                 if (adapter->tag[i] != NULL) {
2907                         bus_teardown_intr(dev, adapter->res[i],
2908                             adapter->tag[i]);
2909                         adapter->tag[i] = NULL;
2910                 }
2911                 if (adapter->res[i] != NULL) {
2912                         bus_release_resource(dev, SYS_RES_IRQ,
2913                             adapter->rid[i], adapter->res[i]);
2914                 }
2915         }
2916
2917         if (adapter->msi)
2918                 pci_release_msi(dev);
2919
2920         if (adapter->msix != NULL)
2921                 bus_release_resource(dev, SYS_RES_MEMORY,
2922                     PCIR_BAR(EM_MSIX_BAR), adapter->msix);
2923
2924         if (adapter->memory != NULL)
2925                 bus_release_resource(dev, SYS_RES_MEMORY,
2926                     PCIR_BAR(0), adapter->memory);
2927
2928         if (adapter->flash != NULL)
2929                 bus_release_resource(dev, SYS_RES_MEMORY,
2930                     EM_FLASH, adapter->flash);
2931
2932         if (adapter->ioport != NULL)
2933                 bus_release_resource(dev, SYS_RES_IOPORT,
2934                     adapter->io_rid, adapter->ioport);
2935 }
2936
2937 /*
2938  * Setup MSI or MSI/X
2939  */
2940 static int
2941 em_setup_msix(struct adapter *adapter)
2942 {
2943         device_t dev = adapter->dev;
2944         int val = 0;
2945
2946         if (adapter->hw.mac.type < e1000_82571)
2947                 return (0);
2948
2949         /* Setup MSI/X for Hartwell */
2950         if (adapter->hw.mac.type == e1000_82574) {
2951                 /* Map the MSIX BAR */
2952                 int rid = PCIR_BAR(EM_MSIX_BAR);
2953                 adapter->msix = bus_alloc_resource_any(dev,
2954                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2955                 if (!adapter->msix) {
2956                         /* May not be enabled */
2957                         device_printf(adapter->dev,
2958                             "Unable to map MSIX table \n");
2959                         goto msi;
2960                 }
2961                 val = pci_msix_count(dev); 
2962                 /*
2963                 ** 82574 can be configured for 5 but
2964                 ** we limit use to 3.
2965                 */
2966                 if (val > 3) val = 3;
2967                 if ((val) && pci_alloc_msix(dev, &val) == 0) {
2968                         device_printf(adapter->dev,"Using MSIX interrupts\n");
2969                         return (val);
2970                 }
2971         }
2972 msi:
2973         val = pci_msi_count(dev);
2974         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2975                 adapter->msi = 1;
2976                 device_printf(adapter->dev,"Using MSI interrupt\n");
2977                 return (val);
2978         } 
2979         return (0);
2980 }
2981
2982 /*********************************************************************
2983  *
2984  *  Initialize the hardware to a configuration
2985  *  as specified by the adapter structure.
2986  *
2987  **********************************************************************/
2988 static int
2989 em_hardware_init(struct adapter *adapter)
2990 {
2991         device_t dev = adapter->dev;
2992         u16     rx_buffer_size;
2993
2994         INIT_DEBUGOUT("em_hardware_init: begin");
2995
2996         /* Issue a global reset */
2997         e1000_reset_hw(&adapter->hw);
2998
2999         /* When hardware is reset, fifo_head is also reset */
3000         adapter->tx_fifo_head = 0;
3001
3002         /* Set up smart power down as default off on newer adapters. */
3003         if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 ||
3004             adapter->hw.mac.type == e1000_82572)) {
3005                 u16 phy_tmp = 0;
3006
3007                 /* Speed up time to link by disabling smart power down. */
3008                 e1000_read_phy_reg(&adapter->hw,
3009                     IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3010                 phy_tmp &= ~IGP02E1000_PM_SPD;
3011                 e1000_write_phy_reg(&adapter->hw,
3012                     IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3013         }
3014
3015         /*
3016          * These parameters control the automatic generation (Tx) and
3017          * response (Rx) to Ethernet PAUSE frames.
3018          * - High water mark should allow for at least two frames to be
3019          *   received after sending an XOFF.
3020          * - Low water mark works best when it is very near the high water mark.
3021          *   This allows the receiver to restart by sending XON when it has
3022          *   drained a bit. Here we use an arbitary value of 1500 which will
3023          *   restart after one full frame is pulled from the buffer. There
3024          *   could be several smaller frames in the buffer and if so they will
3025          *   not trigger the XON until their total number reduces the buffer
3026          *   by 1500.
3027          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3028          */
3029         rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
3030             0xffff) << 10 );
3031
3032         adapter->hw.fc.high_water = rx_buffer_size -
3033             roundup2(adapter->max_frame_size, 1024);
3034         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
3035
3036         if (adapter->hw.mac.type == e1000_80003es2lan)
3037                 adapter->hw.fc.pause_time = 0xFFFF;
3038         else
3039                 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
3040         adapter->hw.fc.send_xon = TRUE;
3041
3042         /* Set Flow control, use the tunable location if sane */
3043         if ((em_fc_setting >= 0) || (em_fc_setting < 4))
3044                 adapter->hw.fc.requested_mode = em_fc_setting;
3045         else
3046                 adapter->hw.fc.requested_mode = e1000_fc_none;
3047
3048         /* Override - workaround for PCHLAN issue */
3049         if (adapter->hw.mac.type == e1000_pchlan)
3050                 adapter->hw.fc.requested_mode = e1000_fc_rx_pause;
3051
3052         if (e1000_init_hw(&adapter->hw) < 0) {
3053                 device_printf(dev, "Hardware Initialization Failed\n");
3054                 return (EIO);
3055         }
3056
3057         e1000_check_for_link(&adapter->hw);
3058
3059         return (0);
3060 }
3061
3062 /*********************************************************************
3063  *
3064  *  Setup networking device structure and register an interface.
3065  *
3066  **********************************************************************/
3067 static void
3068 em_setup_interface(device_t dev, struct adapter *adapter)
3069 {
3070         struct ifnet   *ifp;
3071
3072         INIT_DEBUGOUT("em_setup_interface: begin");
3073
3074         ifp = adapter->ifp = &adapter->arpcom.ac_if;
3075         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3076         ifp->if_mtu = ETHERMTU;
3077         ifp->if_init =  em_init;
3078         ifp->if_softc = adapter;
3079         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3080         ifp->if_ioctl = em_ioctl;
3081         ifp->if_start = em_start;
3082         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
3083         ifq_set_ready(&ifp->if_snd);
3084
3085         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
3086
3087         ifp->if_capabilities = ifp->if_capenable = 0;
3088
3089 #if __FreeBSD_version >= 800000
3090         /* Multiqueue tx functions */
3091         ifp->if_transmit = em_mq_start;
3092         ifp->if_qflush = em_qflush;
3093         adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx);
3094 #endif  
3095         if (adapter->hw.mac.type >= e1000_82543) {
3096                 int version_cap;
3097                 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3098                 ifp->if_capabilities |= version_cap;
3099                 ifp->if_capenable |= version_cap;
3100         }
3101
3102 #ifdef NET_TSO
3103         /* Identify TSO capable adapters */
3104         if ((adapter->hw.mac.type > e1000_82544) &&
3105             (adapter->hw.mac.type != e1000_82547))
3106                 ifp->if_capabilities |= IFCAP_TSO4;
3107         /*
3108          * By default only enable on PCI-E, this
3109          * can be overriden by ifconfig.
3110          */
3111         if (adapter->hw.mac.type >= e1000_82571)
3112                 ifp->if_capenable |= IFCAP_TSO4;
3113 #endif
3114         /*
3115          * Tell the upper layer(s) we
3116          * support full VLAN capability
3117          */
3118         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3119         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3120         ifp->if_capenable |= (IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING);
3121
3122         /*
3123         ** Dont turn this on by default, if vlans are
3124         ** created on another pseudo device (eg. lagg)
3125         ** then vlan events are not passed thru, breaking
3126         ** operation, but with HW FILTER off it works. If
3127         ** using vlans directly on the em driver you can
3128         ** enable this and get full hardware tag filtering. 
3129         */
3130         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3131
3132 #ifdef DEVICE_POLLING
3133         ifp->if_capabilities |= IFCAP_POLLING;
3134 #endif
3135
3136         /* Limit WOL to MAGIC, not clear others are used */
3137         if (adapter->wol) {
3138                 ifp->if_capabilities |= IFCAP_WOL_MAGIC;
3139                 ifp->if_capenable |= IFCAP_WOL_MAGIC;
3140         }
3141                 
3142         /*
3143          * Specify the media types supported by this adapter and register
3144          * callbacks to update media and link information
3145          */
3146         ifmedia_init(&adapter->media, IFM_IMASK,
3147             em_media_change, em_media_status);
3148         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3149             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3150                 u_char fiber_type = IFM_1000_SX;        /* default type */
3151
3152                 if (adapter->hw.mac.type == e1000_82545)
3153                         fiber_type = IFM_1000_LX;
3154                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 
3155                             0, NULL);
3156                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3157         } else {
3158                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3159                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3160                             0, NULL);
3161                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3162                             0, NULL);
3163                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3164                             0, NULL);
3165                 if (adapter->hw.phy.type != e1000_phy_ife) {
3166                         ifmedia_add(&adapter->media,
3167                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3168                         ifmedia_add(&adapter->media,
3169                                 IFM_ETHER | IFM_1000_T, 0, NULL);
3170                 }
3171         }
3172         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3173         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3174 }
3175
3176
3177 /*********************************************************************
3178  *
3179  *  Workaround for SmartSpeed on 82541 and 82547 controllers
3180  *
3181  **********************************************************************/
3182 static void
3183 em_smartspeed(struct adapter *adapter)
3184 {
3185         u16 phy_tmp;
3186
3187         if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
3188             adapter->hw.mac.autoneg == 0 ||
3189             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
3190                 return;
3191
3192         if (adapter->smartspeed == 0) {
3193                 /* If Master/Slave config fault is asserted twice,
3194                  * we assume back-to-back */
3195                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3196                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
3197                         return;
3198                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3199                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
3200                         e1000_read_phy_reg(&adapter->hw,
3201                             PHY_1000T_CTRL, &phy_tmp);
3202                         if(phy_tmp & CR_1000T_MS_ENABLE) {
3203                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
3204                                 e1000_write_phy_reg(&adapter->hw,
3205                                     PHY_1000T_CTRL, phy_tmp);
3206                                 adapter->smartspeed++;
3207                                 if(adapter->hw.mac.autoneg &&
3208                                    !e1000_copper_link_autoneg(&adapter->hw) &&
3209                                    !e1000_read_phy_reg(&adapter->hw,
3210                                     PHY_CONTROL, &phy_tmp)) {
3211                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3212                                                     MII_CR_RESTART_AUTO_NEG);
3213                                         e1000_write_phy_reg(&adapter->hw,
3214                                             PHY_CONTROL, phy_tmp);
3215                                 }
3216                         }
3217                 }
3218                 return;
3219         } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
3220                 /* If still no link, perhaps using 2/3 pair cable */
3221                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
3222                 phy_tmp |= CR_1000T_MS_ENABLE;
3223                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
3224                 if(adapter->hw.mac.autoneg &&
3225                    !e1000_copper_link_autoneg(&adapter->hw) &&
3226                    !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
3227                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3228                                     MII_CR_RESTART_AUTO_NEG);
3229                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
3230                 }
3231         }
3232         /* Restart process after EM_SMARTSPEED_MAX iterations */
3233         if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
3234                 adapter->smartspeed = 0;
3235 }
3236
3237
3238 /*
3239  * Manage DMA'able memory.
3240  */
3241 static void
3242 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3243 {
3244         if (error)
3245                 return;
3246         *(bus_addr_t *) arg = segs[0].ds_addr;
3247 }
3248
3249 static int
3250 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3251         struct em_dma_alloc *dma, int mapflags)
3252 {
3253         int error;
3254
3255         error = bus_dma_tag_create(NULL,                /* parent */
3256                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
3257                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3258                                 BUS_SPACE_MAXADDR,      /* highaddr */
3259                                 NULL, NULL,             /* filter, filterarg */
3260                                 size,                   /* maxsize */
3261                                 1,                      /* nsegments */
3262                                 size,                   /* maxsegsize */
3263                                 0,                      /* flags */
3264                                 &dma->dma_tag);
3265         if (error) {
3266                 device_printf(adapter->dev,
3267                     "%s: bus_dma_tag_create failed: %d\n",
3268                     __func__, error);
3269                 goto fail_0;
3270         }
3271
3272         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3273             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3274         if (error) {
3275                 device_printf(adapter->dev,
3276                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3277                     __func__, (uintmax_t)size, error);
3278                 goto fail_2;
3279         }
3280
3281         dma->dma_paddr = 0;
3282         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3283             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3284         if (error || dma->dma_paddr == 0) {
3285                 device_printf(adapter->dev,
3286                     "%s: bus_dmamap_load failed: %d\n",
3287                     __func__, error);
3288                 goto fail_3;
3289         }
3290
3291         return (0);
3292
3293 fail_3:
3294         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3295 fail_2:
3296         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3297         bus_dma_tag_destroy(dma->dma_tag);
3298 fail_0:
3299         dma->dma_map = NULL;
3300         dma->dma_tag = NULL;
3301
3302         return (error);
3303 }
3304
3305 static void
3306 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3307 {
3308         if (dma->dma_tag == NULL)
3309                 return;
3310         if (dma->dma_map != NULL) {
3311                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3312                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3313                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3314                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3315                 dma->dma_map = NULL;
3316         }
3317         bus_dma_tag_destroy(dma->dma_tag);
3318         dma->dma_tag = NULL;
3319 }
3320
3321
3322 /*********************************************************************
3323  *
3324  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3325  *  the information needed to transmit a packet on the wire.
3326  *
3327  **********************************************************************/
3328 static int
3329 em_allocate_transmit_structures(struct adapter *adapter)
3330 {
3331         device_t dev = adapter->dev;
3332         struct em_buffer *tx_buffer;
3333         int error;
3334
3335         /*
3336          * Create DMA tags for tx descriptors
3337          */
3338         if ((error = bus_dma_tag_create(NULL,           /* parent */
3339                                 1, 0,                   /* alignment, bounds */
3340                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3341                                 BUS_SPACE_MAXADDR,      /* highaddr */
3342                                 NULL, NULL,             /* filter, filterarg */
3343                                 EM_TSO_SIZE,            /* maxsize */
3344                                 EM_MAX_SCATTER,         /* nsegments */
3345                                 EM_TSO_SEG_SIZE,        /* maxsegsize */
3346                                 0,                      /* flags */
3347                                 &adapter->txtag)) != 0) {
3348                 device_printf(dev, "Unable to allocate TX DMA tag\n");
3349                 goto fail;
3350         }
3351
3352         adapter->tx_buffer_area = kmalloc(sizeof(struct em_buffer) *
3353             adapter->num_tx_desc, M_DEVBUF, M_INTWAIT | M_ZERO);
3354         if (adapter->tx_buffer_area == NULL) {
3355                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3356                 error = ENOMEM;
3357                 goto fail;
3358         }
3359
3360         /* Create the descriptor buffer dma maps */
3361         for (int i = 0; i < adapter->num_tx_desc; i++) {
3362                 tx_buffer = &adapter->tx_buffer_area[i];
3363                 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
3364                 if (error != 0) {
3365                         device_printf(dev, "Unable to create TX DMA map\n");
3366                         goto fail;
3367                 }
3368                 tx_buffer->next_eop = -1;
3369         }
3370
3371         return (0);
3372 fail:
3373         em_free_transmit_structures(adapter);
3374         return (error);
3375 }
3376
3377 /*********************************************************************
3378  *
3379  *  (Re)Initialize transmit structures.
3380  *
3381  **********************************************************************/
3382 static void
3383 em_setup_transmit_structures(struct adapter *adapter)
3384 {
3385         struct em_buffer *tx_buffer;
3386
3387         /* Clear the old ring contents */
3388         bzero(adapter->tx_desc_base,
3389             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3390
3391         /* Free any existing TX buffers */
3392         for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3393                 tx_buffer = &adapter->tx_buffer_area[i];
3394                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3395                     BUS_DMASYNC_POSTWRITE);
3396                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
3397                 m_freem(tx_buffer->m_head);
3398                 tx_buffer->m_head = NULL;
3399                 tx_buffer->next_eop = -1;
3400         }
3401
3402         /* Reset state */
3403         adapter->next_avail_tx_desc = 0;
3404         adapter->next_tx_to_clean = 0;
3405         adapter->num_tx_desc_avail = adapter->num_tx_desc;
3406
3407         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3408             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3409
3410         return;
3411 }
3412
3413 /*********************************************************************
3414  *
3415  *  Enable transmit unit.
3416  *
3417  **********************************************************************/
3418 static void
3419 em_initialize_transmit_unit(struct adapter *adapter)
3420 {
3421         u32     tctl, tarc, tipg = 0;
3422         u64     bus_addr;
3423
3424          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3425         /* Setup the Base and Length of the Tx Descriptor Ring */
3426         bus_addr = adapter->txdma.dma_paddr;
3427         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
3428             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3429         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
3430             (u32)(bus_addr >> 32));
3431         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
3432             (u32)bus_addr);
3433         /* Setup the HW Tx Head and Tail descriptor pointers */
3434         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
3435         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
3436
3437         HW_DEBUGOUT2("Base = %x, Length = %x\n",
3438             E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
3439             E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
3440
3441         /* Set the default values for the Tx Inter Packet Gap timer */
3442         switch (adapter->hw.mac.type) {
3443         case e1000_82542:
3444                 tipg = DEFAULT_82542_TIPG_IPGT;
3445                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3446                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3447                 break;
3448         case e1000_80003es2lan:
3449                 tipg = DEFAULT_82543_TIPG_IPGR1;
3450                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3451                     E1000_TIPG_IPGR2_SHIFT;
3452                 break;
3453         default:
3454                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3455                     (adapter->hw.phy.media_type ==
3456                     e1000_media_type_internal_serdes))
3457                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3458                 else
3459                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3460                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3461                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3462         }
3463
3464         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3465         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3466         if(adapter->hw.mac.type >= e1000_82540)
3467                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3468                     adapter->tx_abs_int_delay.value);
3469
3470         if ((adapter->hw.mac.type == e1000_82571) ||
3471             (adapter->hw.mac.type == e1000_82572)) {
3472                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3473                 tarc |= SPEED_MODE_BIT;
3474                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3475         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3476                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3477                 tarc |= 1;
3478                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3479                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3480                 tarc |= 1;
3481                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3482         }
3483
3484         /* Program the Transmit Control Register */
3485         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3486         tctl &= ~E1000_TCTL_CT;
3487         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3488                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3489
3490         if (adapter->hw.mac.type >= e1000_82571)
3491                 tctl |= E1000_TCTL_MULR;
3492
3493         /* This write will effectively turn on the transmit unit. */
3494         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3495
3496         /* Setup Transmit Descriptor Base Settings */   
3497         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3498
3499         if (adapter->tx_int_delay.value > 0)
3500                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3501 }
3502
3503 /*********************************************************************
3504  *
3505  *  Free all transmit related data structures.
3506  *
3507  **********************************************************************/
3508 static void
3509 em_free_transmit_structures(struct adapter *adapter)
3510 {
3511         struct em_buffer *tx_buffer;
3512
3513         INIT_DEBUGOUT("free_transmit_structures: begin");
3514
3515         if (adapter->tx_buffer_area != NULL) {
3516                 for (int i = 0; i < adapter->num_tx_desc; i++) {
3517                         tx_buffer = &adapter->tx_buffer_area[i];
3518                         if (tx_buffer->m_head != NULL) {
3519                                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3520                                     BUS_DMASYNC_POSTWRITE);
3521                                 bus_dmamap_unload(adapter->txtag,
3522                                     tx_buffer->map);
3523                                 m_freem(tx_buffer->m_head);
3524                                 tx_buffer->m_head = NULL;
3525                         } else if (tx_buffer->map != NULL)
3526                                 bus_dmamap_unload(adapter->txtag,
3527                                     tx_buffer->map);
3528                         if (tx_buffer->map != NULL) {
3529                                 bus_dmamap_destroy(adapter->txtag,
3530                                     tx_buffer->map);
3531                                 tx_buffer->map = NULL;
3532                         }
3533                 }
3534         }
3535         if (adapter->tx_buffer_area != NULL) {
3536                 kfree(adapter->tx_buffer_area, M_DEVBUF);
3537                 adapter->tx_buffer_area = NULL;
3538         }
3539         if (adapter->txtag != NULL) {
3540                 bus_dma_tag_destroy(adapter->txtag);
3541                 adapter->txtag = NULL;
3542         }
3543 #if __FreeBSD_version >= 800000
3544         if (adapter->br != NULL)
3545                 buf_ring_free(adapter->br, M_DEVBUF);
3546 #endif
3547 }
3548
3549 /*********************************************************************
3550  *
3551  *  The offload context needs to be set when we transfer the first
3552  *  packet of a particular protocol (TCP/UDP). This routine has been
3553  *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3554  *
3555  *  Added back the old method of keeping the current context type
3556  *  and not setting if unnecessary, as this is reported to be a
3557  *  big performance win.  -jfv
3558  **********************************************************************/
3559 static void
3560 em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
3561     u32 *txd_upper, u32 *txd_lower)
3562 {
3563         struct e1000_context_desc *TXD = NULL;
3564         struct em_buffer *tx_buffer;
3565         struct ether_vlan_header *eh;
3566         struct ip *ip = NULL;
3567         struct ip6_hdr *ip6;
3568         int curr_txd, ehdrlen;
3569         u32 cmd, hdr_len, ip_hlen;
3570         u16 etype;
3571         u8 ipproto;
3572
3573
3574         cmd = hdr_len = ipproto = 0;
3575         curr_txd = adapter->next_avail_tx_desc;
3576
3577         /*
3578          * Determine where frame payload starts.
3579          * Jump over vlan headers if already present,
3580          * helpful for QinQ too.
3581          */
3582         eh = mtod(mp, struct ether_vlan_header *);
3583         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3584                 etype = ntohs(eh->evl_proto);
3585                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3586         } else {
3587                 etype = ntohs(eh->evl_encap_proto);
3588                 ehdrlen = ETHER_HDR_LEN;
3589         }
3590
3591         /*
3592          * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3593          * TODO: Support SCTP too when it hits the tree.
3594          */
3595         switch (etype) {
3596         case ETHERTYPE_IP:
3597                 ip = (struct ip *)(mp->m_data + ehdrlen);
3598                 ip_hlen = ip->ip_hl << 2;
3599
3600                 /* Setup of IP header checksum. */
3601                 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3602                         /*
3603                          * Start offset for header checksum calculation.
3604                          * End offset for header checksum calculation.
3605                          * Offset of place to put the checksum.
3606                          */
3607                         TXD = (struct e1000_context_desc *)
3608                             &adapter->tx_desc_base[curr_txd];
3609                         TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3610                         TXD->lower_setup.ip_fields.ipcse =
3611                             htole16(ehdrlen + ip_hlen);
3612                         TXD->lower_setup.ip_fields.ipcso =
3613                             ehdrlen + offsetof(struct ip, ip_sum);
3614                         cmd |= E1000_TXD_CMD_IP;
3615                         *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3616                 }
3617
3618                 if (mp->m_len < ehdrlen + ip_hlen)
3619                         return; /* failure */
3620
3621                 hdr_len = ehdrlen + ip_hlen;
3622                 ipproto = ip->ip_p;
3623
3624                 break;
3625         case ETHERTYPE_IPV6:
3626                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3627                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3628
3629                 if (mp->m_len < ehdrlen + ip_hlen)
3630                         return; /* failure */
3631
3632                 /* IPv6 doesn't have a header checksum. */
3633
3634                 hdr_len = ehdrlen + ip_hlen;
3635                 ipproto = ip6->ip6_nxt;
3636
3637                 break;
3638         default:
3639                 *txd_upper = 0;
3640                 *txd_lower = 0;
3641                 return;
3642         }
3643
3644         switch (ipproto) {
3645         case IPPROTO_TCP:
3646                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3647                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3648                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3649                         /* no need for context if already set */
3650                         if (adapter->last_hw_offload == CSUM_TCP)
3651                                 return;
3652                         adapter->last_hw_offload = CSUM_TCP;
3653                         /*
3654                          * Start offset for payload checksum calculation.
3655                          * End offset for payload checksum calculation.
3656                          * Offset of place to put the checksum.
3657                          */
3658                         TXD = (struct e1000_context_desc *)
3659                             &adapter->tx_desc_base[curr_txd];
3660                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3661                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3662                         TXD->upper_setup.tcp_fields.tucso =
3663                             hdr_len + offsetof(struct tcphdr, th_sum);
3664                         cmd |= E1000_TXD_CMD_TCP;
3665                 }
3666                 break;
3667         case IPPROTO_UDP:
3668         {
3669                 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3670                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3671                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3672                         /* no need for context if already set */
3673                         if (adapter->last_hw_offload == CSUM_UDP)
3674                                 return;
3675                         adapter->last_hw_offload = CSUM_UDP;
3676                         /*
3677                          * Start offset for header checksum calculation.
3678                          * End offset for header checksum calculation.
3679                          * Offset of place to put the checksum.
3680                          */
3681                         TXD = (struct e1000_context_desc *)
3682                             &adapter->tx_desc_base[curr_txd];
3683                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3684                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3685                         TXD->upper_setup.tcp_fields.tucso =
3686                             hdr_len + offsetof(struct udphdr, uh_sum);
3687                 }
3688                 /* Fall Thru */
3689         }
3690         default:
3691                 break;
3692         }
3693
3694         TXD->tcp_seg_setup.data = htole32(0);
3695         TXD->cmd_and_length =
3696             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3697         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3698         tx_buffer->m_head = NULL;
3699         tx_buffer->next_eop = -1;
3700
3701         if (++curr_txd == adapter->num_tx_desc)
3702                 curr_txd = 0;
3703
3704         adapter->num_tx_desc_avail--;
3705         adapter->next_avail_tx_desc = curr_txd;
3706 }
3707
3708
3709 #ifdef NET_TSO
3710 /**********************************************************************
3711  *
3712  *  Setup work for hardware segmentation offload (TSO)
3713  *
3714  **********************************************************************/
3715 static bool
3716 em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper,
3717    u32 *txd_lower)
3718 {
3719         struct e1000_context_desc *TXD;
3720         struct em_buffer *tx_buffer;
3721         struct ether_vlan_header *eh;
3722         struct ip *ip;
3723         struct ip6_hdr *ip6;
3724         struct tcphdr *th;
3725         int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6;
3726         u16 etype;
3727
3728         /*
3729          * This function could/should be extended to support IP/IPv6
3730          * fragmentation as well.  But as they say, one step at a time.
3731          */
3732
3733         /*
3734          * Determine where frame payload starts.
3735          * Jump over vlan headers if already present,
3736          * helpful for QinQ too.
3737          */
3738         eh = mtod(mp, struct ether_vlan_header *);
3739         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3740                 etype = ntohs(eh->evl_proto);
3741                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3742         } else {
3743                 etype = ntohs(eh->evl_encap_proto);
3744                 ehdrlen = ETHER_HDR_LEN;
3745         }
3746
3747         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3748         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3749                 return FALSE;   /* -1 */
3750
3751         /*
3752          * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3753          * TODO: Support SCTP too when it hits the tree.
3754          */
3755         switch (etype) {
3756         case ETHERTYPE_IP:
3757                 isip6 = 0;
3758                 ip = (struct ip *)(mp->m_data + ehdrlen);
3759                 if (ip->ip_p != IPPROTO_TCP)
3760                         return FALSE;   /* 0 */
3761                 ip->ip_len = 0;
3762                 ip->ip_sum = 0;
3763                 ip_hlen = ip->ip_hl << 2;
3764                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3765                         return FALSE;   /* -1 */
3766                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3767 #if 1
3768                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3769                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3770 #else
3771                 th->th_sum = mp->m_pkthdr.csum_data;
3772 #endif
3773                 break;
3774         case ETHERTYPE_IPV6:
3775                 isip6 = 1;
3776                 return FALSE;                   /* Not supported yet. */
3777                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3778                 if (ip6->ip6_nxt != IPPROTO_TCP)
3779                         return FALSE;   /* 0 */
3780                 ip6->ip6_plen = 0;
3781                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3782                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3783                         return FALSE;   /* -1 */
3784                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3785 #if 0
3786                 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3787                     htons(IPPROTO_TCP));        /* XXX: function notyet. */
3788 #else
3789                 th->th_sum = mp->m_pkthdr.csum_data;
3790 #endif
3791                 break;
3792         default:
3793                 return FALSE;
3794         }
3795         hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3796
3797         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3798                       E1000_TXD_DTYP_D |        /* Data descr type */
3799                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3800
3801         /* IP and/or TCP header checksum calculation and insertion. */
3802         *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3803                       E1000_TXD_POPTS_TXSM) << 8;
3804
3805         curr_txd = adapter->next_avail_tx_desc;
3806         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3807         TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3808
3809         /* IPv6 doesn't have a header checksum. */
3810         if (!isip6) {
3811                 /*
3812                  * Start offset for header checksum calculation.
3813                  * End offset for header checksum calculation.
3814                  * Offset of place put the checksum.
3815                  */
3816                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3817                 TXD->lower_setup.ip_fields.ipcse =
3818                     htole16(ehdrlen + ip_hlen - 1);
3819                 TXD->lower_setup.ip_fields.ipcso =
3820                     ehdrlen + offsetof(struct ip, ip_sum);
3821         }
3822         /*
3823          * Start offset for payload checksum calculation.
3824          * End offset for payload checksum calculation.
3825          * Offset of place to put the checksum.
3826          */
3827         TXD->upper_setup.tcp_fields.tucss =
3828             ehdrlen + ip_hlen;
3829         TXD->upper_setup.tcp_fields.tucse = 0;
3830         TXD->upper_setup.tcp_fields.tucso =
3831             ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3832         /*
3833          * Payload size per packet w/o any headers.
3834          * Length of all headers up to payload.
3835          */
3836         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3837         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3838
3839         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3840                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3841                                 E1000_TXD_CMD_TSE |     /* TSE context */
3842                                 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */
3843                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3844                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3845
3846         tx_buffer->m_head = NULL;
3847         tx_buffer->next_eop = -1;
3848
3849         if (++curr_txd == adapter->num_tx_desc)
3850                 curr_txd = 0;
3851
3852         adapter->num_tx_desc_avail--;
3853         adapter->next_avail_tx_desc = curr_txd;
3854         adapter->tx_tso = TRUE;
3855
3856         return TRUE;
3857 }
3858
3859 #endif
3860
3861 /**********************************************************************
3862  *
3863  *  Examine each tx_buffer in the used queue. If the hardware is done
3864  *  processing the packet then free associated resources. The
3865  *  tx_buffer is put back on the free queue.
3866  *
3867  **********************************************************************/
3868 static void
3869 em_txeof(struct adapter *adapter)
3870 {