network code: Convert if_multiaddrs from LIST to TAILQ.
[dragonfly.git] / sys / dev / netif / e1000 / if_em.c
CommitLineData
d9d871af
MN
1/******************************************************************************
2
3 Copyright (c) 2001-2010, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
d9d871af
MN
33
34#ifdef HAVE_KERNEL_OPTION_HEADERS
35#include "opt_device_polling.h"
36#include "opt_inet.h"
37#include "opt_altq.h"
38#endif
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#if __FreeBSD_version >= 800000
43#include <sys/buf_ring.h>
44#endif
45#include <sys/bus.h>
46#include <sys/endian.h>
47#include <sys/kernel.h>
48#include <sys/kthread.h>
49#include <sys/malloc.h>
50#include <sys/mbuf.h>
51#include <sys/module.h>
52#include <sys/rman.h>
53#include <sys/socket.h>
54#include <sys/sockio.h>
55#include <sys/sysctl.h>
56#include <sys/taskqueue.h>
d9d871af 57#include <sys/eventhandler.h>
d9d871af
MN
58
59#include <net/bpf.h>
60#include <net/ethernet.h>
61#include <net/if.h>
62#include <net/if_arp.h>
63#include <net/if_dl.h>
64#include <net/if_media.h>
2c3d0901 65#include <net/ifq_var.h>
d9d871af
MN
66
67#include <net/if_types.h>
2c3d0901
MN
68#include <net/vlan/if_vlan_var.h>
69#include <net/vlan/if_vlan_ether.h>
d9d871af
MN
70
71#include <netinet/in_systm.h>
72#include <netinet/in.h>
73#include <netinet/if_ether.h>
74#include <netinet/ip.h>
75#include <netinet/ip6.h>
76#include <netinet/tcp.h>
77#include <netinet/udp.h>
78
2c3d0901
MN
79#include <sys/in_cksum.h>
80#include <bus/pci/pcivar.h>
81#include <bus/pci/pcireg.h>
d9d871af
MN
82
83#include "e1000_api.h"
84#include "e1000_82571.h"
85#include "if_em.h"
52535fdf 86#include "ifcap_defines.h" // XXX
d9d871af
MN
87
88/*********************************************************************
89 * Set this to one to display debug statistics
90 *********************************************************************/
91int em_display_debug_stats = 0;
92
93/*********************************************************************
94 * Driver version:
95 *********************************************************************/
96char em_driver_version[] = "6.9.25";
97
98
99/*********************************************************************
100 * PCI Device ID Table
101 *
102 * Used by probe to select devices to load on
103 * Last field stores an index into e1000_strings
104 * Last entry must be all 0s
105 *
106 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
107 *********************************************************************/
108
109static em_vendor_info_t em_vendor_info_array[] =
110{
111 /* Intel(R) PRO/1000 Network Connection */
112 { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0},
113 { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
114 { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0},
115 { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
116 { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0},
117
118 { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0},
119 { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0},
120 { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
121 { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
122 { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0},
123 { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
124 { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
125
126 { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0},
127
128 { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
129 { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
130
131 { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
132 { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
133 { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
134 { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0},
135
136 { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
137 { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
138 { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
139 { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
140 { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
141
142 { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
143 { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
144 { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
145 { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
146 { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
147 { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
148 { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0},
149 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
150 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
151 PCI_ANY_ID, PCI_ANY_ID, 0},
152
153 { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0},
154 { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0},
155 { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0},
156
157 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
158 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
159 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
160 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
161 PCI_ANY_ID, PCI_ANY_ID, 0},
162 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
163 PCI_ANY_ID, PCI_ANY_ID, 0},
164 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
165 PCI_ANY_ID, PCI_ANY_ID, 0},
166 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
167 PCI_ANY_ID, PCI_ANY_ID, 0},
168 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
169 PCI_ANY_ID, PCI_ANY_ID, 0},
170 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
171 PCI_ANY_ID, PCI_ANY_ID, 0},
172 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
173 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0},
174 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
175 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0},
176
177 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0},
178 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0},
179 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0},
180 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0},
181 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
182 PCI_ANY_ID, PCI_ANY_ID, 0},
183 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
184 PCI_ANY_ID, PCI_ANY_ID, 0},
185 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
186 PCI_ANY_ID, PCI_ANY_ID, 0},
187 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
188 PCI_ANY_ID, PCI_ANY_ID, 0},
189 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
190 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
191 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
192 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
193 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
194 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
195 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
196 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0},
197 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
198 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0},
199 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0},
200 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0},
201 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0},
202 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0},
203 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0},
204 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0},
205 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0},
206 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0},
207 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0},
208 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
209 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
210 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0},
211 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
212 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0},
213 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0},
214 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0},
215 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0},
216 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0},
217 /* required last entry */
218 { 0, 0, 0, 0, 0}
219};
220
221/*********************************************************************
222 * Table of branding strings for all supported NICs.
223 *********************************************************************/
224
225static char *em_strings[] = {
226 "Intel(R) PRO/1000 Network Connection"
227};
228
229/*********************************************************************
230 * Function prototypes
231 *********************************************************************/
232static int em_probe(device_t);
233static int em_attach(device_t);
234static int em_detach(device_t);
235static int em_shutdown(device_t);
236static int em_suspend(device_t);
237static int em_resume(device_t);
238static void em_start(struct ifnet *);
239static void em_start_locked(struct ifnet *ifp);
240#if __FreeBSD_version >= 800000
241static int em_mq_start(struct ifnet *, struct mbuf *);
242static int em_mq_start_locked(struct ifnet *, struct mbuf *);
243static void em_qflush(struct ifnet *);
244#endif
2c3d0901 245static int em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
d9d871af
MN
246static void em_init(void *);
247static void em_init_locked(struct adapter *);
248static void em_stop(void *);
249static void em_media_status(struct ifnet *, struct ifmediareq *);
250static int em_media_change(struct ifnet *);
251static void em_identify_hardware(struct adapter *);
252static int em_allocate_pci_resources(struct adapter *);
253static int em_allocate_legacy(struct adapter *adapter);
254static int em_allocate_msix(struct adapter *adapter);
255static int em_setup_msix(struct adapter *);
256static void em_free_pci_resources(struct adapter *);
257static void em_local_timer(void *);
258static int em_hardware_init(struct adapter *);
259static void em_setup_interface(device_t, struct adapter *);
260static void em_setup_transmit_structures(struct adapter *);
261static void em_initialize_transmit_unit(struct adapter *);
262static int em_setup_receive_structures(struct adapter *);
263static void em_initialize_receive_unit(struct adapter *);
264static void em_enable_intr(struct adapter *);
265static void em_disable_intr(struct adapter *);
266static void em_free_transmit_structures(struct adapter *);
267static void em_free_receive_structures(struct adapter *);
268static void em_update_stats_counters(struct adapter *);
269static void em_txeof(struct adapter *);
270static void em_tx_purge(struct adapter *);
271static int em_allocate_receive_structures(struct adapter *);
272static int em_allocate_transmit_structures(struct adapter *);
273static int em_rxeof(struct adapter *, int);
274#ifndef __NO_STRICT_ALIGNMENT
275static int em_fixup_rx(struct adapter *);
276#endif
277static void em_receive_checksum(struct adapter *, struct e1000_rx_desc *,
278 struct mbuf *);
279static void em_transmit_checksum_setup(struct adapter *, struct mbuf *,
280 u32 *, u32 *);
2c3d0901 281#ifdef NET_TSO
d9d871af
MN
282static bool em_tso_setup(struct adapter *, struct mbuf *,
283 u32 *, u32 *);
2c3d0901 284#endif
d9d871af
MN
285static void em_set_promisc(struct adapter *);
286static void em_disable_promisc(struct adapter *);
287static void em_set_multi(struct adapter *);
288static void em_print_hw_stats(struct adapter *);
289static void em_update_link_status(struct adapter *);
290static int em_get_buf(struct adapter *, int);
2c3d0901 291
d9d871af
MN
292static void em_register_vlan(void *, struct ifnet *, u16);
293static void em_unregister_vlan(void *, struct ifnet *, u16);
294static void em_setup_vlan_hw_support(struct adapter *);
2c3d0901 295
d9d871af
MN
296static int em_xmit(struct adapter *, struct mbuf **);
297static void em_smartspeed(struct adapter *);
298static int em_82547_fifo_workaround(struct adapter *, int);
299static void em_82547_update_fifo_head(struct adapter *, int);
300static int em_82547_tx_fifo_reset(struct adapter *);
301static void em_82547_move_tail(void *);
302static int em_dma_malloc(struct adapter *, bus_size_t,
303 struct em_dma_alloc *, int);
304static void em_dma_free(struct adapter *, struct em_dma_alloc *);
305static void em_print_debug_info(struct adapter *);
306static void em_print_nvm_info(struct adapter *);
307static int em_is_valid_ether_addr(u8 *);
308static int em_sysctl_stats(SYSCTL_HANDLER_ARGS);
309static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
310static u32 em_fill_descriptors (bus_addr_t address, u32 length,
311 PDESC_ARRAY desc_array);
312static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
313static void em_add_int_delay_sysctl(struct adapter *, const char *,
314 const char *, struct em_int_delay_info *, int, int);
315/* Management and WOL Support */
316static void em_init_manageability(struct adapter *);
317static void em_release_manageability(struct adapter *);
318static void em_get_hw_control(struct adapter *);
319static void em_release_hw_control(struct adapter *);
320static void em_get_wakeup(device_t);
321static void em_enable_wakeup(device_t);
322static int em_enable_phy_wakeup(struct adapter *);
323
324#ifdef EM_LEGACY_IRQ
325static void em_intr(void *);
326#else /* FAST IRQ */
d9d871af 327static void em_irq_fast(void *);
d9d871af
MN
328
329/* MSIX handlers */
330static void em_msix_tx(void *);
331static void em_msix_rx(void *);
332static void em_msix_link(void *);
333static void em_handle_rx(void *context, int pending);
334static void em_handle_tx(void *context, int pending);
335
336static void em_handle_rxtx(void *context, int pending);
337static void em_handle_link(void *context, int pending);
338static void em_add_rx_process_limit(struct adapter *, const char *,
339 const char *, int *, int);
340#endif /* ~EM_LEGACY_IRQ */
341
342#ifdef DEVICE_POLLING
343static poll_handler_t em_poll;
344#endif /* POLLING */
345
346/*********************************************************************
347 * FreeBSD Device Interface Entry Points
348 *********************************************************************/
349
350static device_method_t em_methods[] = {
351 /* Device interface */
352 DEVMETHOD(device_probe, em_probe),
353 DEVMETHOD(device_attach, em_attach),
354 DEVMETHOD(device_detach, em_detach),
355 DEVMETHOD(device_shutdown, em_shutdown),
356 DEVMETHOD(device_suspend, em_suspend),
357 DEVMETHOD(device_resume, em_resume),
358 {0, 0}
359};
360
361static driver_t em_driver = {
362 "em", em_methods, sizeof(struct adapter),
363};
364
365static devclass_t em_devclass;
366DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
367MODULE_DEPEND(em, pci, 1, 1, 1);
368MODULE_DEPEND(em, ether, 1, 1, 1);
369
370/*********************************************************************
371 * Tunable default values.
372 *********************************************************************/
373
374#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000)
375#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024)
376#define M_TSO_LEN 66
377
378/* Allow common code without TSO */
379#ifndef CSUM_TSO
380#define CSUM_TSO 0
381#endif
382
383static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
384static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
385static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
386static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
387static int em_rxd = EM_DEFAULT_RXD;
388static int em_txd = EM_DEFAULT_TXD;
389static int em_smart_pwr_down = FALSE;
390/* Controls whether promiscuous also shows bad packets */
391static int em_debug_sbp = FALSE;
392/* Local switch for MSI/MSIX */
393static int em_enable_msi = TRUE;
394
395TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
396TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
397TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
398TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
399TUNABLE_INT("hw.em.rxd", &em_rxd);
400TUNABLE_INT("hw.em.txd", &em_txd);
401TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
402TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
403TUNABLE_INT("hw.em.enable_msi", &em_enable_msi);
404
405#ifndef EM_LEGACY_IRQ
406/* How many packets rxeof tries to clean at a time */
407static int em_rx_process_limit = 100;
408TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
409#endif
410
411/* Flow control setting - default to FULL */
412static int em_fc_setting = e1000_fc_full;
413TUNABLE_INT("hw.em.fc_setting", &em_fc_setting);
414
415/*
416** Shadow VFTA table, this is needed because
417** the real vlan filter table gets cleared during
418** a soft reset and the driver needs to be able
419** to repopulate it.
420*/
421static u32 em_shadow_vfta[EM_VFTA_SIZE];
422
423/* Global used in WOL setup with multiport cards */
424static int global_quad_port_a = 0;
425
426/*********************************************************************
427 * Device identification routine
428 *
429 * em_probe determines if the driver should be loaded on
430 * adapter based on PCI vendor/device id of the adapter.
431 *
432 * return BUS_PROBE_DEFAULT on success, positive on failure
433 *********************************************************************/
434
435static int
436em_probe(device_t dev)
437{
438 char adapter_name[60];
439 u16 pci_vendor_id = 0;
440 u16 pci_device_id = 0;
441 u16 pci_subvendor_id = 0;
442 u16 pci_subdevice_id = 0;
443 em_vendor_info_t *ent;
444
445 INIT_DEBUGOUT("em_probe: begin");
446
447 pci_vendor_id = pci_get_vendor(dev);
448 if (pci_vendor_id != EM_VENDOR_ID)
449 return (ENXIO);
450
451 pci_device_id = pci_get_device(dev);
452 pci_subvendor_id = pci_get_subvendor(dev);
453 pci_subdevice_id = pci_get_subdevice(dev);
454
455 ent = em_vendor_info_array;
456 while (ent->vendor_id != 0) {
457 if ((pci_vendor_id == ent->vendor_id) &&
458 (pci_device_id == ent->device_id) &&
459
460 ((pci_subvendor_id == ent->subvendor_id) ||
461 (ent->subvendor_id == PCI_ANY_ID)) &&
462
463 ((pci_subdevice_id == ent->subdevice_id) ||
464 (ent->subdevice_id == PCI_ANY_ID))) {
2c3d0901 465 ksprintf(adapter_name, "%s %s",
d9d871af
MN
466 em_strings[ent->index],
467 em_driver_version);
468 device_set_desc_copy(dev, adapter_name);
469 return (BUS_PROBE_DEFAULT);
470 }
471 ent++;
472 }
473
474 return (ENXIO);
475}
476
477/*********************************************************************
478 * Device initialization routine
479 *
480 * The attach entry point is called when the driver is being loaded.
481 * This routine identifies the type of hardware, allocates all resources
482 * and initializes the hardware.
483 *
484 * return 0 on success, positive on failure
485 *********************************************************************/
486
487static int
488em_attach(device_t dev)
489{
490 struct adapter *adapter;
491 int tsize, rsize;
492 int error = 0;
493
494 INIT_DEBUGOUT("em_attach: begin");
495
496 adapter = device_get_softc(dev);
497 adapter->dev = adapter->osdep.dev = dev;
2c3d0901 498
d9d871af
MN
499 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
500 EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
501 EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
502
503 /* SYSCTL stuff */
2c3d0901
MN
504 sysctl_ctx_init(&adapter->sysctl_ctx);
505 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
506 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
507 device_get_nameunit(adapter->dev),
508 CTLFLAG_RD, 0, "");
509 if (adapter->sysctl_tree == NULL) {
510 device_printf(adapter->dev, "can't add sysctl node\n");
511 error = ENOMEM;
512 goto err_sysctl;
513 }
514
515 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
516 SYSCTL_CHILDREN(adapter->sysctl_tree),
d9d871af
MN
517 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
518 em_sysctl_debug_info, "I", "Debug Information");
519
2c3d0901
MN
520 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
521 SYSCTL_CHILDREN(adapter->sysctl_tree),
d9d871af
MN
522 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
523 em_sysctl_stats, "I", "Statistics");
524
2c3d0901
MN
525 callout_init(&adapter->timer);
526 callout_init(&adapter->tx_fifo_timer);
d9d871af
MN
527
528 /* Determine hardware and mac info */
529 em_identify_hardware(adapter);
530
531 /* Setup PCI resources */
532 if (em_allocate_pci_resources(adapter)) {
533 device_printf(dev, "Allocation of PCI resources failed\n");
534 error = ENXIO;
535 goto err_pci;
536 }
537
538 /*
539 ** For ICH8 and family we need to
540 ** map the flash memory, and this
541 ** must happen after the MAC is
542 ** identified
543 */
544 if ((adapter->hw.mac.type == e1000_ich8lan) ||
545 (adapter->hw.mac.type == e1000_pchlan) ||
546 (adapter->hw.mac.type == e1000_ich9lan) ||
547 (adapter->hw.mac.type == e1000_ich10lan)) {
548 int rid = EM_BAR_TYPE_FLASH;
549 adapter->flash = bus_alloc_resource_any(dev,
550 SYS_RES_MEMORY, &rid, RF_ACTIVE);
551 if (adapter->flash == NULL) {
552 device_printf(dev, "Mapping of Flash failed\n");
553 error = ENXIO;
554 goto err_pci;
555 }
556 /* This is used in the shared code */
557 adapter->hw.flash_address = (u8 *)adapter->flash;
558 adapter->osdep.flash_bus_space_tag =
559 rman_get_bustag(adapter->flash);
560 adapter->osdep.flash_bus_space_handle =
561 rman_get_bushandle(adapter->flash);
562 }
563
564 /* Do Shared Code initialization */
565 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
566 device_printf(dev, "Setup of Shared code failed\n");
567 error = ENXIO;
568 goto err_pci;
569 }
570
571 e1000_get_bus_info(&adapter->hw);
572
573 /* Set up some sysctls for the tunable interrupt delays */
574 em_add_int_delay_sysctl(adapter, "rx_int_delay",
575 "receive interrupt delay in usecs", &adapter->rx_int_delay,
576 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
577 em_add_int_delay_sysctl(adapter, "tx_int_delay",
578 "transmit interrupt delay in usecs", &adapter->tx_int_delay,
579 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
580 if (adapter->hw.mac.type >= e1000_82540) {
581 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
582 "receive interrupt delay limit in usecs",
583 &adapter->rx_abs_int_delay,
584 E1000_REGISTER(&adapter->hw, E1000_RADV),
585 em_rx_abs_int_delay_dflt);
586 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
587 "transmit interrupt delay limit in usecs",
588 &adapter->tx_abs_int_delay,
589 E1000_REGISTER(&adapter->hw, E1000_TADV),
590 em_tx_abs_int_delay_dflt);
591 }
592
593#ifndef EM_LEGACY_IRQ
594 /* Sysctls for limiting the amount of work done in the taskqueue */
595 em_add_rx_process_limit(adapter, "rx_processing_limit",
596 "max number of rx packets to process", &adapter->rx_process_limit,
597 em_rx_process_limit);
598#endif
599
600 /*
601 * Validate number of transmit and receive descriptors. It
602 * must not exceed hardware maximum, and must be multiple
603 * of E1000_DBA_ALIGN.
604 */
605 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
606 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
607 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
608 (em_txd < EM_MIN_TXD)) {
609 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
610 EM_DEFAULT_TXD, em_txd);
611 adapter->num_tx_desc = EM_DEFAULT_TXD;
612 } else
613 adapter->num_tx_desc = em_txd;
614 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
615 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
616 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
617 (em_rxd < EM_MIN_RXD)) {
618 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
619 EM_DEFAULT_RXD, em_rxd);
620 adapter->num_rx_desc = EM_DEFAULT_RXD;
621 } else
622 adapter->num_rx_desc = em_rxd;
623
624 adapter->hw.mac.autoneg = DO_AUTO_NEG;
625 adapter->hw.phy.autoneg_wait_to_complete = FALSE;
626 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
627 adapter->rx_buffer_len = 2048;
628
629 e1000_init_script_state_82541(&adapter->hw, TRUE);
630 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
631
632 /* Copper options */
633 if (adapter->hw.phy.media_type == e1000_media_type_copper) {
634 adapter->hw.phy.mdix = AUTO_ALL_MODES;
635 adapter->hw.phy.disable_polarity_correction = FALSE;
636 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
637 }
638
639 /*
640 * Set the frame limits assuming
641 * standard ethernet sized frames.
642 */
643 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
644 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
645
646 /*
647 * This controls when hardware reports transmit completion
648 * status.
649 */
650 adapter->hw.mac.report_tx_early = 1;
651
652 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
653 EM_DBA_ALIGN);
654
655 /* Allocate Transmit Descriptor ring */
656 if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
657 device_printf(dev, "Unable to allocate tx_desc memory\n");
658 error = ENOMEM;
659 goto err_tx_desc;
660 }
661 adapter->tx_desc_base =
662 (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
663
664 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
665 EM_DBA_ALIGN);
666
667 /* Allocate Receive Descriptor ring */
668 if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
669 device_printf(dev, "Unable to allocate rx_desc memory\n");
670 error = ENOMEM;
671 goto err_rx_desc;
672 }
673 adapter->rx_desc_base =
674 (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
675
676 /*
677 ** Start from a known state, this is
678 ** important in reading the nvm and
679 ** mac from that.
680 */
681 e1000_reset_hw(&adapter->hw);
682
683 /* Make sure we have a good EEPROM before we read from it */
684 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
685 /*
686 ** Some PCI-E parts fail the first check due to
687 ** the link being in sleep state, call it again,
688 ** if it fails a second time its a real issue.
689 */
690 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
691 device_printf(dev,
692 "The EEPROM Checksum Is Not Valid\n");
693 error = EIO;
694 goto err_hw_init;
695 }
696 }
697
698 /* Copy the permanent MAC address out of the EEPROM */
699 if (e1000_read_mac_addr(&adapter->hw) < 0) {
700 device_printf(dev, "EEPROM read error while reading MAC"
701 " address\n");
702 error = EIO;
703 goto err_hw_init;
704 }
705
706 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
707 device_printf(dev, "Invalid MAC address\n");
708 error = EIO;
709 goto err_hw_init;
710 }
711
712 /* Initialize the hardware */
713 if (em_hardware_init(adapter)) {
714 device_printf(dev, "Unable to initialize the hardware\n");
715 error = EIO;
716 goto err_hw_init;
717 }
718
719 /* Allocate transmit descriptors and buffers */
720 if (em_allocate_transmit_structures(adapter)) {
721 device_printf(dev, "Could not setup transmit structures\n");
722 error = ENOMEM;
723 goto err_tx_struct;
724 }
725
726 /* Allocate receive descriptors and buffers */
727 if (em_allocate_receive_structures(adapter)) {
728 device_printf(dev, "Could not setup receive structures\n");
729 error = ENOMEM;
730 goto err_rx_struct;
731 }
732
733 /*
734 ** Do interrupt configuration
735 */
736 if (adapter->msi > 1) /* Do MSI/X */
737 error = em_allocate_msix(adapter);
738 else /* MSI or Legacy */
739 error = em_allocate_legacy(adapter);
740 if (error)
741 goto err_rx_struct;
742
743 /*
744 * Get Wake-on-Lan and Management info for later use
745 */
746 em_get_wakeup(dev);
747
748 /* Setup OS specific network interface */
749 em_setup_interface(dev, adapter);
750
751 /* Initialize statistics */
752 em_update_stats_counters(adapter);
753
754 adapter->hw.mac.get_link_status = 1;
755 em_update_link_status(adapter);
756
757 /* Indicate SOL/IDER usage */
758 if (e1000_check_reset_block(&adapter->hw))
759 device_printf(dev,
760 "PHY reset is blocked due to SOL/IDER session.\n");
761
762 /* Do we need workaround for 82544 PCI-X adapter? */
763 if (adapter->hw.bus.type == e1000_bus_type_pcix &&
764 adapter->hw.mac.type == e1000_82544)
765 adapter->pcix_82544 = TRUE;
766 else
767 adapter->pcix_82544 = FALSE;
768
d9d871af
MN
769 /* Register for VLAN events */
770 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
771 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
772 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
773 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
d9d871af
MN
774
775 /* Non-AMT based hardware can now take control from firmware */
776 if (adapter->has_manage && !adapter->has_amt)
777 em_get_hw_control(adapter);
778
779 /* Tell the stack that the interface is not active */
2c3d0901 780 adapter->ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
d9d871af
MN
781
782 INIT_DEBUGOUT("em_attach: end");
783
784 return (0);
785
786err_rx_struct:
787 em_free_transmit_structures(adapter);
788err_tx_struct:
789err_hw_init:
790 em_release_hw_control(adapter);
791 em_dma_free(adapter, &adapter->rxdma);
792err_rx_desc:
793 em_dma_free(adapter, &adapter->txdma);
794err_tx_desc:
795err_pci:
796 em_free_pci_resources(adapter);
2c3d0901
MN
797err_sysctl:
798 sysctl_ctx_free(&adapter->sysctl_ctx);
d9d871af
MN
799 EM_TX_LOCK_DESTROY(adapter);
800 EM_RX_LOCK_DESTROY(adapter);
801 EM_CORE_LOCK_DESTROY(adapter);
802
803 return (error);
804}
805
806/*********************************************************************
807 * Device removal routine
808 *
809 * The detach entry point is called when the driver is being removed.
810 * This routine stops the adapter and deallocates all the resources
811 * that were allocated for driver operation.
812 *
813 * return 0 on success, positive on failure
814 *********************************************************************/
815
816static int
817em_detach(device_t dev)
818{
819 struct adapter *adapter = device_get_softc(dev);
d9d871af
MN
820
821 INIT_DEBUGOUT("em_detach: begin");
822
823 /* Make sure VLANS are not using driver */
2c3d0901 824 if (adapter->ifp->if_vlantrunks != NULL) {
d9d871af
MN
825 device_printf(dev,"Vlan in use, detach first\n");
826 return (EBUSY);
827 }
828
829#ifdef DEVICE_POLLING
830 if (ifp->if_capenable & IFCAP_POLLING)
831 ether_poll_deregister(ifp);
832#endif
833
834 EM_CORE_LOCK(adapter);
835 EM_TX_LOCK(adapter);
836 adapter->in_detach = 1;
837 em_stop(adapter);
838 e1000_phy_hw_reset(&adapter->hw);
839
840 em_release_manageability(adapter);
841
842 EM_TX_UNLOCK(adapter);
843 EM_CORE_UNLOCK(adapter);
844
d9d871af
MN
845 /* Unregister VLAN events */
846 if (adapter->vlan_attach != NULL)
847 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
848 if (adapter->vlan_detach != NULL)
849 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
d9d871af
MN
850
851 ether_ifdetach(adapter->ifp);
2c3d0901
MN
852 callout_stop(&adapter->timer);
853 callout_stop(&adapter->tx_fifo_timer);
d9d871af
MN
854
855 em_free_pci_resources(adapter);
856 bus_generic_detach(dev);
d9d871af
MN
857
858 em_free_transmit_structures(adapter);
859 em_free_receive_structures(adapter);
860
861 /* Free Transmit Descriptor ring */
862 if (adapter->tx_desc_base) {
863 em_dma_free(adapter, &adapter->txdma);
864 adapter->tx_desc_base = NULL;
865 }
866
867 /* Free Receive Descriptor ring */
868 if (adapter->rx_desc_base) {
869 em_dma_free(adapter, &adapter->rxdma);
870 adapter->rx_desc_base = NULL;
871 }
872
873 em_release_hw_control(adapter);
2c3d0901 874 sysctl_ctx_free(&adapter->sysctl_ctx);
d9d871af
MN
875 EM_TX_LOCK_DESTROY(adapter);
876 EM_RX_LOCK_DESTROY(adapter);
877 EM_CORE_LOCK_DESTROY(adapter);
878
879 return (0);
880}
881
882/*********************************************************************
883 *
884 * Shutdown entry point
885 *
886 **********************************************************************/
887
888static int
889em_shutdown(device_t dev)
890{
891 return em_suspend(dev);
892}
893
894/*
895 * Suspend/resume device methods.
896 */
897static int
898em_suspend(device_t dev)
899{
900 struct adapter *adapter = device_get_softc(dev);
901
902 EM_CORE_LOCK(adapter);
903
904 em_release_manageability(adapter);
905 em_release_hw_control(adapter);
906 em_enable_wakeup(dev);
907
908 EM_CORE_UNLOCK(adapter);
909
910 return bus_generic_suspend(dev);
911}
912
913static int
914em_resume(device_t dev)
915{
916 struct adapter *adapter = device_get_softc(dev);
917 struct ifnet *ifp = adapter->ifp;
918
919 EM_CORE_LOCK(adapter);
920 em_init_locked(adapter);
921 em_init_manageability(adapter);
922 EM_CORE_UNLOCK(adapter);
923 em_start(ifp);
924
925 return bus_generic_resume(dev);
926}
927
928
929/*********************************************************************
930 * Transmit entry point
931 *
932 * em_start is called by the stack to initiate a transmit.
933 * The driver will remain in this routine as long as there are
934 * packets to transmit and transmit resources are available.
935 * In case resources are not available stack is notified and
936 * the packet is requeued.
937 **********************************************************************/
938
2c3d0901 939#if 0
d9d871af
MN
940static int
941em_mq_start_locked(struct ifnet *ifp, struct mbuf *m)
942{
943 struct adapter *adapter = ifp->if_softc;
944 struct mbuf *next;
945 int error = E1000_SUCCESS;
946
947 EM_TX_LOCK_ASSERT(adapter);
948 /* To allow being called from a tasklet */
949 if (m == NULL)
950 goto process;
951
2c3d0901
MN
952 if (((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) !=
953 IFF_RUNNING)
d9d871af
MN
954 || (!adapter->link_active)) {
955 error = drbr_enqueue(ifp, adapter->br, m);
956 return (error);
957 } else if (!drbr_needs_enqueue(ifp, adapter->br) &&
958 (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) {
959 if ((error = em_xmit(adapter, &m)) != 0) {
960 if (m)
961 error = drbr_enqueue(ifp, adapter->br, m);
962 return (error);
963 } else {
964 /*
965 * We've bypassed the buf ring so we need to update
966 * ifp directly
967 */
968 drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
969 /*
970 ** Send a copy of the frame to the BPF
971 ** listener and set the watchdog on.
972 */
973 ETHER_BPF_MTAP(ifp, m);
974 adapter->watchdog_check = TRUE;
975 }
976 } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0)
977 return (error);
978
979process:
980 if (drbr_empty(ifp, adapter->br))
981 return(error);
982 /* Process the queue */
983 while (TRUE) {
2c3d0901 984 if ((ifp->if_flags & IFF_RUNNING) == 0)
d9d871af
MN
985 break;
986 next = drbr_dequeue(ifp, adapter->br);
987 if (next == NULL)
988 break;
989 if ((error = em_xmit(adapter, &next)) != 0) {
990 if (next != NULL)
991 error = drbr_enqueue(ifp, adapter->br, next);
992 break;
993 }
994 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
995 ETHER_BPF_MTAP(ifp, next);
996 /* Set the watchdog */
997 adapter->watchdog_check = TRUE;
998 }
999
1000 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
2c3d0901 1001 ifp->if_flags |= IFF_OACTIVE;
d9d871af
MN
1002
1003 return (error);
1004}
1005
1006/*
1007** Multiqueue capable stack interface, this is not
1008** yet truely multiqueue, but that is coming...
1009*/
1010static int
1011em_mq_start(struct ifnet *ifp, struct mbuf *m)
1012{
1013
1014 struct adapter *adapter = ifp->if_softc;
1015 int error = 0;
1016
1017 if (EM_TX_TRYLOCK(adapter)) {
2c3d0901 1018 if (ifp->if_flags & IFF_RUNNING)
d9d871af
MN
1019 error = em_mq_start_locked(ifp, m);
1020 EM_TX_UNLOCK(adapter);
1021 } else
1022 error = drbr_enqueue(ifp, adapter->br, m);
1023
1024 return (error);
1025}
1026
1027static void
1028em_qflush(struct ifnet *ifp)
1029{
1030 struct mbuf *m;
1031 struct adapter *adapter = (struct adapter *)ifp->if_softc;
1032
1033 EM_TX_LOCK(adapter);
1034 while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL)
1035 m_freem(m);
1036 if_qflush(ifp);
1037 EM_TX_UNLOCK(adapter);
1038}
2c3d0901 1039#endif
d9d871af
MN
1040
1041static void
1042em_start_locked(struct ifnet *ifp)
1043{
1044 struct adapter *adapter = ifp->if_softc;
1045 struct mbuf *m_head;
1046
1047 EM_TX_LOCK_ASSERT(adapter);
1048
2c3d0901
MN
1049 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) !=
1050 IFF_RUNNING)
d9d871af
MN
1051 return;
1052 if (!adapter->link_active)
1053 return;
1054
2c3d0901 1055 while (!ifq_is_empty(&ifp->if_snd)) {
d9d871af 1056
2c3d0901 1057 m_head = ifq_dequeue(&ifp->if_snd, NULL);
d9d871af
MN
1058 if (m_head == NULL)
1059 break;
1060 /*
1061 * Encapsulation can modify our pointer, and or make it
1062 * NULL on failure. In that event, we can't requeue.
1063 */
1064 if (em_xmit(adapter, &m_head)) {
1065 if (m_head == NULL)
1066 break;
2c3d0901
MN
1067 ifp->if_flags |= IFF_OACTIVE;
1068 ifq_prepend(&ifp->if_snd, m_head);
d9d871af
MN
1069 break;
1070 }
1071
1072 /* Send a copy of the frame to the BPF listener */
1073 ETHER_BPF_MTAP(ifp, m_head);
1074
1075 /* Set timeout in case hardware has problems transmitting. */
1076 adapter->watchdog_check = TRUE;
1077 }
1078 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
2c3d0901 1079 ifp->if_flags |= IFF_OACTIVE;
d9d871af
MN
1080
1081 return;
1082}
1083
1084static void
1085em_start(struct ifnet *ifp)
1086{
1087 struct adapter *adapter = ifp->if_softc;
1088
1089 EM_TX_LOCK(adapter);
2c3d0901 1090 if (ifp->if_flags & IFF_RUNNING)
d9d871af
MN
1091 em_start_locked(ifp);
1092 EM_TX_UNLOCK(adapter);
1093}
1094
1095/*********************************************************************
1096 * Ioctl entry point
1097 *
1098 * em_ioctl is called when the user wants to configure the
1099 * interface.
1100 *
1101 * return 0 on success, positive on failure
1102 **********************************************************************/
1103
1104static int
2c3d0901 1105em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred * uc)
d9d871af
MN
1106{
1107 struct adapter *adapter = ifp->if_softc;
1108 struct ifreq *ifr = (struct ifreq *)data;
1109#ifdef INET
1110 struct ifaddr *ifa = (struct ifaddr *)data;
1111#endif
1112 int error = 0;
1113
1114 if (adapter->in_detach)
1115 return (error);
1116
1117 switch (command) {
1118 case SIOCSIFADDR:
1119#ifdef INET
1120 if (ifa->ifa_addr->sa_family == AF_INET) {
1121 /*
1122 * XXX
1123 * Since resetting hardware takes a very long time
1124 * and results in link renegotiation we only
1125 * initialize the hardware only when it is absolutely
1126 * required.
1127 */
1128 ifp->if_flags |= IFF_UP;
2c3d0901 1129 if (!(ifp->if_flags & IFF_RUNNING)) {
d9d871af
MN
1130 EM_CORE_LOCK(adapter);
1131 em_init_locked(adapter);
1132 EM_CORE_UNLOCK(adapter);
1133 }
1134 arp_ifinit(ifp, ifa);
1135 } else
1136#endif
1137 error = ether_ioctl(ifp, command, data);
1138 break;
1139 case SIOCSIFMTU:
1140 {
1141 int max_frame_size;
1142 u16 eeprom_data = 0;
1143
1144 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1145
1146 EM_CORE_LOCK(adapter);
1147 switch (adapter->hw.mac.type) {
1148 case e1000_82573:
1149 /*
1150 * 82573 only supports jumbo frames
1151 * if ASPM is disabled.
1152 */
1153 e1000_read_nvm(&adapter->hw,
1154 NVM_INIT_3GIO_3, 1, &eeprom_data);
1155 if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1156 max_frame_size = ETHER_MAX_LEN;
1157 break;
1158 }
1159 /* Allow Jumbo frames - fall thru */
1160 case e1000_82571:
1161 case e1000_82572:
1162 case e1000_ich9lan:
1163 case e1000_ich10lan:
1164 case e1000_82574:
1165 case e1000_80003es2lan: /* Limit Jumbo Frame size */
1166 max_frame_size = 9234;
1167 break;
1168 case e1000_pchlan:
1169 max_frame_size = 4096;
1170 break;
1171 /* Adapters that do not support jumbo frames */
1172 case e1000_82542:
1173 case e1000_82583:
1174 case e1000_ich8lan:
1175 max_frame_size = ETHER_MAX_LEN;
1176 break;
1177 default:
1178 max_frame_size = MAX_JUMBO_FRAME_SIZE;
1179 }
1180 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1181 ETHER_CRC_LEN) {
1182 EM_CORE_UNLOCK(adapter);
1183 error = EINVAL;
1184 break;
1185 }
1186
1187 ifp->if_mtu = ifr->ifr_mtu;
1188 adapter->max_frame_size =
1189 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1190 em_init_locked(adapter);
1191 EM_CORE_UNLOCK(adapter);
1192 break;
1193 }
1194 case SIOCSIFFLAGS:
1195 IOCTL_DEBUGOUT("ioctl rcv'd:\
1196 SIOCSIFFLAGS (Set Interface Flags)");
1197 EM_CORE_LOCK(adapter);
1198 if (ifp->if_flags & IFF_UP) {
2c3d0901 1199 if ((ifp->if_flags & IFF_RUNNING)) {
d9d871af
MN
1200 if ((ifp->if_flags ^ adapter->if_flags) &
1201 (IFF_PROMISC | IFF_ALLMULTI)) {
1202 em_disable_promisc(adapter);
1203 em_set_promisc(adapter);
1204 }
1205 } else
1206 em_init_locked(adapter);
1207 } else
2c3d0901 1208 if (ifp->if_flags & IFF_RUNNING) {
d9d871af
MN
1209 EM_TX_LOCK(adapter);
1210 em_stop(adapter);
1211 EM_TX_UNLOCK(adapter);
1212 }
1213 adapter->if_flags = ifp->if_flags;
1214 EM_CORE_UNLOCK(adapter);
1215 break;
1216 case SIOCADDMULTI:
1217 case SIOCDELMULTI:
1218 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
2c3d0901 1219 if (ifp->if_flags & IFF_RUNNING) {
d9d871af
MN
1220 EM_CORE_LOCK(adapter);
1221 em_disable_intr(adapter);
1222 em_set_multi(adapter);
1223 if (adapter->hw.mac.type == e1000_82542 &&
1224 adapter->hw.revision_id == E1000_REVISION_2) {
1225 em_initialize_receive_unit(adapter);
1226 }
1227#ifdef DEVICE_POLLING
1228 if (!(ifp->if_capenable & IFCAP_POLLING))
1229#endif
1230 em_enable_intr(adapter);
1231 EM_CORE_UNLOCK(adapter);
1232 }
1233 break;
1234 case SIOCSIFMEDIA:
1235 /* Check SOL/IDER usage */
1236 EM_CORE_LOCK(adapter);
1237 if (e1000_check_reset_block(&adapter->hw)) {
1238 EM_CORE_UNLOCK(adapter);
1239 device_printf(adapter->dev, "Media change is"
1240 " blocked due to SOL/IDER session.\n");
1241 break;
1242 }
1243 EM_CORE_UNLOCK(adapter);
1244 case SIOCGIFMEDIA:
1245 IOCTL_DEBUGOUT("ioctl rcv'd: \
1246 SIOCxIFMEDIA (Get/Set Interface Media)");
1247 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1248 break;
1249 case SIOCSIFCAP:
1250 {
1251 int mask, reinit;
1252
1253 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1254 reinit = 0;
1255 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1256#ifdef DEVICE_POLLING
1257 if (mask & IFCAP_POLLING) {
1258 if (ifr->ifr_reqcap & IFCAP_POLLING) {
1259 error = ether_poll_register(em_poll, ifp);
1260 if (error)
1261 return (error);
1262 EM_CORE_LOCK(adapter);
1263 em_disable_intr(adapter);
1264 ifp->if_capenable |= IFCAP_POLLING;
1265 EM_CORE_UNLOCK(adapter);
1266 } else {
1267 error = ether_poll_deregister(ifp);
1268 /* Enable interrupt even in error case */
1269 EM_CORE_LOCK(adapter);
1270 em_enable_intr(adapter);
1271 ifp->if_capenable &= ~IFCAP_POLLING;
1272 EM_CORE_UNLOCK(adapter);
1273 }
1274 }
1275#endif
1276 if (mask & IFCAP_HWCSUM) {
1277 ifp->if_capenable ^= IFCAP_HWCSUM;
1278 reinit = 1;
1279 }
2c3d0901 1280#ifdef NET_TSO
d9d871af
MN
1281 if (mask & IFCAP_TSO4) {
1282 ifp->if_capenable ^= IFCAP_TSO4;
1283 reinit = 1;
1284 }
1285#endif
1286 if (mask & IFCAP_VLAN_HWTAGGING) {
1287 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1288 reinit = 1;
1289 }
1290
1291 if (mask & IFCAP_VLAN_HWFILTER) {
1292 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1293 reinit = 1;
1294 }
1295
1296 if ((mask & IFCAP_WOL) &&
1297 (ifp->if_capabilities & IFCAP_WOL) != 0) {
1298 if (mask & IFCAP_WOL_MCAST)
1299 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1300 if (mask & IFCAP_WOL_MAGIC)
1301 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1302 }
1303
2c3d0901 1304 if (reinit && (ifp->if_flags & IFF_RUNNING))
d9d871af 1305 em_init(adapter);
2c3d0901 1306#if 0
d9d871af
MN
1307 VLAN_CAPABILITIES(ifp);
1308#endif
1309 break;
1310 }
1311
1312 default:
1313 error = ether_ioctl(ifp, command, data);
1314 break;
1315 }
1316
1317 return (error);
1318}
1319
1320
1321/*********************************************************************
1322 * Init entry point
1323 *
1324 * This routine is used in two ways. It is used by the stack as
1325 * init entry point in network interface structure. It is also used
1326 * by the driver as a hw/sw initialization routine to get to a
1327 * consistent state.
1328 *
1329 * return 0 on success, positive on failure
1330 **********************************************************************/
1331
1332static void
1333em_init_locked(struct adapter *adapter)
1334{
1335 struct ifnet *ifp = adapter->ifp;
1336 device_t dev = adapter->dev;
1337 u32 pba;
1338
1339 INIT_DEBUGOUT("em_init: begin");
1340
1341 EM_CORE_LOCK_ASSERT(adapter);
1342
1343 EM_TX_LOCK(adapter);
1344 em_stop(adapter);
1345 EM_TX_UNLOCK(adapter);
1346
1347 /*
1348 * Packet Buffer Allocation (PBA)
1349 * Writing PBA sets the receive portion of the buffer
1350 * the remainder is used for the transmit buffer.
1351 *
1352 * Devices before the 82547 had a Packet Buffer of 64K.
1353 * Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1354 * After the 82547 the buffer was reduced to 40K.
1355 * Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1356 * Note: default does not leave enough room for Jumbo Frame >10k.
1357 */
1358 switch (adapter->hw.mac.type) {
1359 case e1000_82547:
1360 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1361 if (adapter->max_frame_size > 8192)
1362 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1363 else
1364 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1365 adapter->tx_fifo_head = 0;
1366 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1367 adapter->tx_fifo_size =
1368 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1369 break;
1370 /* Total Packet Buffer on these is 48K */
1371 case e1000_82571:
1372 case e1000_82572:
1373 case e1000_80003es2lan:
1374 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1375 break;
1376 case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1377 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1378 break;
1379 case e1000_82574:
1380 case e1000_82583:
1381 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1382 break;
1383 case e1000_ich9lan:
1384 case e1000_ich10lan:
1385 case e1000_pchlan:
1386 pba = E1000_PBA_10K;
1387 break;
1388 case e1000_ich8lan:
1389 pba = E1000_PBA_8K;
1390 break;
1391 default:
1392 /* Devices before 82547 had a Packet Buffer of 64K. */
1393 if (adapter->max_frame_size > 8192)
1394 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1395 else
1396 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1397 }
1398
1399 INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1400 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1401
1402 /* Get the latest mac address, User can use a LAA */
1403 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1404 ETHER_ADDR_LEN);
1405
1406 /* Put the address into the Receive Address Array */
1407 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1408
1409 /*
1410 * With the 82571 adapter, RAR[0] may be overwritten
1411 * when the other port is reset, we make a duplicate
1412 * in RAR[14] for that eventuality, this assures
1413 * the interface continues to function.
1414 */
1415 if (adapter->hw.mac.type == e1000_82571) {
1416 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1417 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1418 E1000_RAR_ENTRIES - 1);
1419 }
1420
1421 /* Initialize the hardware */
1422 if (em_hardware_init(adapter)) {
1423 device_printf(dev, "Unable to initialize the hardware\n");
1424 return;
1425 }
1426 em_update_link_status(adapter);
1427
1428 /* Setup VLAN support, basic and offload if available */
1429 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1430 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1431 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1432 /* Use real VLAN Filter support */
1433 em_setup_vlan_hw_support(adapter);
1434 else {
1435 u32 ctrl;
1436 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1437 ctrl |= E1000_CTRL_VME;
1438 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1439 }
1440 }
1441
1442 /* Set hardware offload abilities */
1443 ifp->if_hwassist = 0;
1444 if (adapter->hw.mac.type >= e1000_82543) {
1445 if (ifp->if_capenable & IFCAP_TXCSUM)
1446 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
2c3d0901 1447#ifdef NET_TSO
d9d871af
MN
1448 if (ifp->if_capenable & IFCAP_TSO4)
1449 ifp->if_hwassist |= CSUM_TSO;
1450#endif
1451 }
1452
1453 /* Configure for OS presence */
1454 em_init_manageability(adapter);
1455
1456 /* Prepare transmit descriptors and buffers */
1457 em_setup_transmit_structures(adapter);
1458 em_initialize_transmit_unit(adapter);
1459
1460 /* Setup Multicast table */
1461 em_set_multi(adapter);
1462
1463 /* Prepare receive descriptors and buffers */
1464 if (em_setup_receive_structures(adapter)) {
1465 device_printf(dev, "Could not setup receive structures\n");
1466 EM_TX_LOCK(adapter);
1467 em_stop(adapter);
1468 EM_TX_UNLOCK(adapter);
1469 return;
1470 }
1471 em_initialize_receive_unit(adapter);
1472
1473 /* Don't lose promiscuous settings */
1474 em_set_promisc(adapter);
1475
2c3d0901
MN
1476 ifp->if_flags |= IFF_RUNNING;
1477 ifp->if_flags &= ~IFF_OACTIVE;
d9d871af
MN
1478
1479 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1480 e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1481
1482 /* MSI/X configuration for 82574 */
1483 if (adapter->hw.mac.type == e1000_82574) {
1484 int tmp;
1485 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1486 tmp |= E1000_CTRL_EXT_PBA_CLR;
1487 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1488 /*
1489 ** Set the IVAR - interrupt vector routing.
1490 ** Each nibble represents a vector, high bit
1491 ** is enable, other 3 bits are the MSIX table
1492 ** entry, we map RXQ0 to 0, TXQ0 to 1, and
1493 ** Link (other) to 2, hence the magic number.
1494 */
1495 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1496 }
1497
1498#ifdef DEVICE_POLLING
1499 /*
1500 * Only enable interrupts if we are not polling, make sure
1501 * they are off otherwise.
1502 */
1503 if (ifp->if_capenable & IFCAP_POLLING)
1504 em_disable_intr(adapter);
1505 else
1506#endif /* DEVICE_POLLING */
1507 em_enable_intr(adapter);
1508
1509 /* AMT based hardware can now take control from firmware */
1510 if (adapter->has_manage && adapter->has_amt)
1511 em_get_hw_control(adapter);
1512
1513 /* Don't reset the phy next time init gets called */
1514 adapter->hw.phy.reset_disable = TRUE;
1515}
1516
1517static void
1518em_init(void *arg)
1519{
1520 struct adapter *adapter = arg;
1521
1522 EM_CORE_LOCK(adapter);
1523 em_init_locked(adapter);
1524 EM_CORE_UNLOCK(adapter);
1525}
1526
1527
1528#ifdef DEVICE_POLLING
1529/*********************************************************************
1530 *
1531 * Legacy polling routine
1532 *
1533 *********************************************************************/
1534static int
1535em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1536{
1537 struct adapter *adapter = ifp->if_softc;
1538 u32 reg_icr, rx_done = 0;
1539
1540 EM_CORE_LOCK(adapter);
2c3d0901 1541 if ((ifp->if_flags & IFF_RUNNING) == 0) {
d9d871af
MN
1542 EM_CORE_UNLOCK(adapter);
1543 return (rx_done);
1544 }
1545
1546 if (cmd == POLL_AND_CHECK_STATUS) {
1547 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1548 /* Link status change */
1549 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1550 adapter->hw.mac.get_link_status = 1;
1551 em_update_link_status(adapter);
1552 }
1553 if (reg_icr & E1000_ICR_RXO)
1554 adapter->rx_overruns++;
1555 }
1556 EM_CORE_UNLOCK(adapter);
1557
1558 rx_done = em_rxeof(adapter, count);
1559
1560 EM_TX_LOCK(adapter);
1561 em_txeof(adapter);
2c3d0901 1562#if 0
d9d871af
MN
1563 if (!drbr_empty(ifp, adapter->br))
1564 em_mq_start_locked(ifp, NULL);
1565#else
2c3d0901 1566 if (!ifq_is_empty(&ifp->if_snd))
d9d871af
MN
1567 em_start_locked(ifp);
1568#endif
1569 EM_TX_UNLOCK(adapter);
1570 return (rx_done);
1571}
1572#endif /* DEVICE_POLLING */
1573
1574#ifdef EM_LEGACY_IRQ
1575/*********************************************************************
1576 *
1577 * Legacy Interrupt Service routine
1578 *
1579 *********************************************************************/
1580
1581static void
1582em_intr(void *arg)
1583{
1584 struct adapter *adapter = arg;
1585 struct ifnet *ifp = adapter->ifp;
1586 u32 reg_icr;
1587
1588
1589 if (ifp->if_capenable & IFCAP_POLLING)
1590 return;
1591
1592 EM_CORE_LOCK(adapter);
1593 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1594 if (reg_icr & E1000_ICR_RXO)
1595 adapter->rx_overruns++;
1596 if ((reg_icr == 0xffffffff) || (reg_icr == 0)||
1597 (adapter->hw.mac.type >= e1000_82571 &&
1598 (reg_icr & E1000_ICR_INT_ASSERTED) == 0))
1599 goto out;
1600
2c3d0901 1601 if ((ifp->if_flags & IFF_RUNNING) == 0)
d9d871af
MN
1602 goto out;
1603
1604 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1605 callout_stop(&adapter->timer);
1606 adapter->hw.mac.get_link_status = 1;
1607 em_update_link_status(adapter);
1608 /* Deal with TX cruft when link lost */
1609 em_tx_purge(adapter);
1610 callout_reset(&adapter->timer, hz,
1611 em_local_timer, adapter);
1612 goto out;
1613 }
1614
1615 EM_TX_LOCK(adapter);
1616 em_txeof(adapter);
1617 em_rxeof(adapter, -1);
1618 em_txeof(adapter);
2c3d0901
MN
1619 if (ifp->if_flags & IFF_RUNNING &&
1620 !ifq_is_empty(&ifp->if_snd))
d9d871af
MN
1621 em_start_locked(ifp);
1622 EM_TX_UNLOCK(adapter);
1623
1624out:
1625 EM_CORE_UNLOCK(adapter);
1626 return;
1627}
1628
1629#else /* EM_FAST_IRQ, then fast interrupt routines only */
1630
1631static void
1632em_handle_link(void *context, int pending)
1633{
1634 struct adapter *adapter = context;
1635 struct ifnet *ifp = adapter->ifp;
1636
2c3d0901 1637 if (!(ifp->if_flags & IFF_RUNNING))
d9d871af
MN
1638 return;
1639
1640 EM_CORE_LOCK(adapter);
1641 callout_stop(&adapter->timer);
1642 em_update_link_status(adapter);
1643 /* Deal with TX cruft when link lost */
1644 em_tx_purge(adapter);
1645 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1646 EM_CORE_UNLOCK(adapter);
1647}
1648
1649
1650/* Combined RX/TX handler, used by Legacy and MSI */
1651static void
1652em_handle_rxtx(void *context, int pending)
1653{
1654 struct adapter *adapter = context;
1655 struct ifnet *ifp = adapter->ifp;
1656
1657
2c3d0901 1658 if (ifp->if_flags & IFF_RUNNING) {
d9d871af
MN
1659 if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1660 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1661 EM_TX_LOCK(adapter);
1662 em_txeof(adapter);
2c3d0901 1663#if 0
d9d871af
MN
1664 if (!drbr_empty(ifp, adapter->br))
1665 em_mq_start_locked(ifp, NULL);
1666#else
2c3d0901 1667 if (!ifq_is_empty(&ifp->if_snd))
d9d871af
MN
1668 em_start_locked(ifp);
1669#endif
1670 EM_TX_UNLOCK(adapter);
1671 }
1672
1673 em_enable_intr(adapter);
1674}
1675
1676/*********************************************************************
1677 *
1678 * Fast Legacy/MSI Combined Interrupt Service routine
1679 *
1680 *********************************************************************/
d9d871af
MN
1681#define FILTER_STRAY
1682#define FILTER_HANDLED
1683static void
d9d871af
MN
1684em_irq_fast(void *arg)
1685{
1686 struct adapter *adapter = arg;
1687 struct ifnet *ifp;
1688 u32 reg_icr;
1689
1690 ifp = adapter->ifp;
1691
1692 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1693
1694 /* Hot eject? */
1695 if (reg_icr == 0xffffffff)
1696 return FILTER_STRAY;
1697
1698 /* Definitely not our interrupt. */
1699 if (reg_icr == 0x0)
1700 return FILTER_STRAY;
1701
1702 /*
1703 * Starting with the 82571 chip, bit 31 should be used to
1704 * determine whether the interrupt belongs to us.
1705 */
1706 if (adapter->hw.mac.type >= e1000_82571 &&
1707 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1708 return FILTER_STRAY;
1709
1710 /*
1711 * Mask interrupts until the taskqueue is finished running. This is
1712 * cheap, just assume that it is needed. This also works around the
1713 * MSI message reordering errata on certain systems.
1714 */
1715 em_disable_intr(adapter);
1716 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1717
1718 /* Link status change */
1719 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1720 adapter->hw.mac.get_link_status = 1;
2c3d0901 1721 taskqueue_enqueue(taskqueue_swi, &adapter->link_task);
d9d871af
MN
1722 }
1723
1724 if (reg_icr & E1000_ICR_RXO)
1725 adapter->rx_overruns++;
1726 return FILTER_HANDLED;
1727}
1728
1729/*********************************************************************
1730 *
1731 * MSIX Interrupt Service Routines
1732 *
1733 **********************************************************************/
1734#define EM_MSIX_TX 0x00040000
1735#define EM_MSIX_RX 0x00010000
1736#define EM_MSIX_LINK 0x00100000
1737
1738static void
1739em_msix_tx(void *arg)
1740{
1741 struct adapter *adapter = arg;
1742 struct ifnet *ifp = adapter->ifp;
1743
1744 ++adapter->tx_irq;
2c3d0901 1745 if (ifp->if_flags & IFF_RUNNING) {
d9d871af
MN
1746 EM_TX_LOCK(adapter);
1747 em_txeof(adapter);
1748 EM_TX_UNLOCK(adapter);
1749 taskqueue_enqueue(adapter->tq, &adapter->tx_task);
1750 }
1751 /* Reenable this interrupt */
1752 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX);
1753 return;
1754}
1755
1756/*********************************************************************
1757 *
1758 * MSIX RX Interrupt Service routine
1759 *
1760 **********************************************************************/
1761
1762static void
1763em_msix_rx(void *arg)
1764{
1765 struct adapter *adapter = arg;
1766 struct ifnet *ifp = adapter->ifp;
1767
1768 ++adapter->rx_irq;
2c3d0901 1769 if ((ifp->if_flags & IFF_RUNNING) &&
d9d871af
MN
1770 (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1771 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1772 /* Reenable this interrupt */
1773 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX);
1774 return;
1775}
1776
1777/*********************************************************************
1778 *
1779 * MSIX Link Fast Interrupt Service routine
1780 *
1781 **********************************************************************/
1782
1783static void
1784em_msix_link(void *arg)
1785{
1786 struct adapter *adapter = arg;
1787 u32 reg_icr;
1788
1789 ++adapter->link_irq;
1790 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1791
1792 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1793 adapter->hw.mac.get_link_status = 1;
2c3d0901 1794 taskqueue_enqueue(taskqueue_swi, &adapter->link_task);
d9d871af
MN
1795 }
1796 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1797 EM_MSIX_LINK | E1000_IMS_LSC);
1798 return;
1799}
1800
1801static void
1802em_handle_rx(void *context, int pending)
1803{
1804 struct adapter *adapter = context;
1805 struct ifnet *ifp = adapter->ifp;
1806
2c3d0901 1807 if ((ifp->if_flags & IFF_RUNNING) &&
d9d871af
MN
1808 (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1809 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1810
1811}
1812
1813static void
1814em_handle_tx(void *context, int pending)
1815{
1816 struct adapter *adapter = context;
1817 struct ifnet *ifp = adapter->ifp;
1818
2c3d0901 1819 if (ifp->if_flags & IFF_RUNNING) {
d9d871af
MN
1820 if (!EM_TX_TRYLOCK(adapter))
1821 return;
1822 em_txeof(adapter);
2c3d0901 1823#if 0
d9d871af
MN
1824 if (!drbr_empty(ifp, adapter->br))
1825 em_mq_start_locked(ifp, NULL);
1826#else
2c3d0901 1827 if (!ifq_is_empty(&ifp->if_snd))
d9d871af
MN
1828 em_start_locked(ifp);
1829#endif
1830 EM_TX_UNLOCK(adapter);
1831 }
1832}
1833#endif /* EM_FAST_IRQ */
1834
1835/*********************************************************************
1836 *
1837 * Media Ioctl callback
1838 *
1839 * This routine is called whenever the user queries the status of
1840 * the interface using ifconfig.
1841 *
1842 **********************************************************************/
1843static void
1844em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1845{
1846 struct adapter *adapter = ifp->if_softc;
1847 u_char fiber_type = IFM_1000_SX;
1848
1849 INIT_DEBUGOUT("em_media_status: begin");
1850
1851 EM_CORE_LOCK(adapter);
1852 em_update_link_status(adapter);
1853
1854 ifmr->ifm_status = IFM_AVALID;
1855 ifmr->ifm_active = IFM_ETHER;
1856
1857 if (!adapter->link_active) {
1858 EM_CORE_UNLOCK(adapter);
1859 return;
1860 }
1861
1862 ifmr->ifm_status |= IFM_ACTIVE;
1863
1864 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1865 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1866 if (adapter->hw.mac.type == e1000_82545)
1867 fiber_type = IFM_1000_LX;
1868 ifmr->ifm_active |= fiber_type | IFM_FDX;
1869 } else {
1870 switch (adapter->link_speed) {
1871 case 10:
1872 ifmr->ifm_active |= IFM_10_T;
1873 break;
1874 case 100:
1875 ifmr->ifm_active |= IFM_100_TX;
1876 break;
1877 case 1000:
1878 ifmr->ifm_active |= IFM_1000_T;
1879 break;
1880 }
1881 if (adapter->link_duplex == FULL_DUPLEX)
1882 ifmr->ifm_active |= IFM_FDX;
1883 else
1884 ifmr->ifm_active |= IFM_HDX;
1885 }
1886 EM_CORE_UNLOCK(adapter);
1887}
1888
1889/*********************************************************************
1890 *
1891 * Media Ioctl callback
1892 *
1893 * This routine is called when the user changes speed/duplex using
1894 * media/mediopt option with ifconfig.
1895 *
1896 **********************************************************************/
1897static int
1898em_media_change(struct ifnet *ifp)
1899{
1900 struct adapter *adapter = ifp->if_softc;
1901 struct ifmedia *ifm = &adapter->media;
1902
1903 INIT_DEBUGOUT("em_media_change: begin");
1904
1905 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1906 return (EINVAL);
1907
1908 EM_CORE_LOCK(adapter);
1909 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1910 case IFM_AUTO:
1911 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1912 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1913 break;
1914 case IFM_1000_LX:
1915 case IFM_1000_SX:
1916 case IFM_1000_T:
1917 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1918 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1919 break;
1920 case IFM_100_TX:
1921 adapter->hw.mac.autoneg = FALSE;
1922 adapter->hw.phy.autoneg_advertised = 0;
1923 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1924 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1925 else
1926 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1927 break;
1928 case IFM_10_T:
1929 adapter->hw.mac.autoneg = FALSE;
1930 adapter->hw.phy.autoneg_advertised = 0;
1931 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1932 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1933 else
1934 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1935 break;
1936 default:
1937 device_printf(adapter->dev, "Unsupported media type\n");
1938 }
1939
1940 /* As the speed/duplex settings my have changed we need to
1941 * reset the PHY.
1942 */
1943 adapter->hw.phy.reset_disable = FALSE;
1944
1945 em_init_locked(adapter);
1946 EM_CORE_UNLOCK(adapter);
1947
1948 return (0);
1949}
1950
1951/*********************************************************************
1952 *
1953 * This routine maps the mbufs to tx descriptors.
1954 *
1955 * return 0 on success, positive on failure
1956 **********************************************************************/
1957
1958static int
1959em_xmit(struct adapter *adapter, struct mbuf **m_headp)
1960{
1961 bus_dma_segment_t segs[EM_MAX_SCATTER];
1962 bus_dmamap_t map;
1963 struct em_buffer *tx_buffer, *tx_buffer_mapped;
1964 struct e1000_tx_desc *ctxd = NULL;
1965 struct mbuf *m_head;
1966 u32 txd_upper, txd_lower, txd_used, txd_saved;
1967 int nsegs, i, j, first, last = 0;
1968 int error, do_tso, tso_desc = 0;
2c3d0901 1969
d9d871af
MN
1970 m_head = *m_headp;
1971 txd_upper = txd_lower = txd_used = txd_saved = 0;
1972
2c3d0901 1973#ifdef NET_TSO
d9d871af
MN
1974 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1975#else
1976 do_tso = 0;
1977#endif
1978
1979 /*
1980 * Force a cleanup if number of TX descriptors
1981 * available hits the threshold
1982 */
1983 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1984 em_txeof(adapter);
1985 /* Now do we at least have a minimal? */
1986 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
1987 adapter->no_tx_desc_avail1++;
1988 return (ENOBUFS);
1989 }
1990 }
1991
1992
1993 /*
1994 * TSO workaround:
1995 * If an mbuf is only header we need
1996 * to pull 4 bytes of data into it.
1997 */
1998 if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
1999 m_head = m_pullup(m_head, M_TSO_LEN + 4);
2000 *m_headp = m_head;
2001 if (m_head == NULL)
2002 return (ENOBUFS);
2003 }
2004
2005 /*
2006 * Map the packet for DMA
2007 *
2008 * Capture the first descriptor index,
2009 * this descriptor will have the index
2010 * of the EOP which is the only one that
2011 * now gets a DONE bit writeback.
2012 */
2013 first = adapter->next_avail_tx_desc;
2014 tx_buffer = &adapter->tx_buffer_area[first];
2015 tx_buffer_mapped = tx_buffer;
2016 map = tx_buffer->map;
2017
2c3d0901
MN
2018 error = bus_dmamap_load_mbuf_segment(adapter->txtag, map,
2019 *m_headp, segs, EM_MAX_SCATTER, &nsegs, BUS_DMA_NOWAIT);
d9d871af
MN
2020
2021 /*
2022 * There are two types of errors we can (try) to handle:
2023 * - EFBIG means the mbuf chain was too long and bus_dma ran
2024 * out of segments. Defragment the mbuf chain and try again.
2025 * - ENOMEM means bus_dma could not obtain enough bounce buffers
2026 * at this point in time. Defer sending and try again later.
2027 * All other errors, in particular EINVAL, are fatal and prevent the
2028 * mbuf chain from ever going through. Drop it and report error.
2029 */
2030 if (error == EFBIG) {
2031 struct mbuf *m;
2032
2c3d0901 2033 m = m_defrag(*m_headp, MB_DONTWAIT);
d9d871af
MN
2034 if (m == NULL) {
2035 adapter->mbuf_alloc_failed++;
2036 m_freem(*m_headp);
2037 *m_headp = NULL;
2038 return (ENOBUFS);
2039 }
2040 *m_headp = m;
2041
2042 /* Try it again */
2c3d0901
MN
2043 error = bus_dmamap_load_mbuf_segment(adapter->txtag, map,
2044 *m_headp, segs, EM_MAX_SCATTER, &nsegs, BUS_DMA_NOWAIT);
d9d871af
MN
2045
2046 if (error) {
2047 adapter->no_tx_dma_setup++;
2048 m_freem(*m_headp);
2049 *m_headp = NULL;
2050 return (error);
2051 }
2052 } else if (error != 0) {
2053 adapter->no_tx_dma_setup++;
2054 return (error);
2055 }
2056
2057 /*
2058 * TSO Hardware workaround, if this packet is not
2059 * TSO, and is only a single descriptor long, and
2060 * it follows a TSO burst, then we need to add a
2061 * sentinel descriptor to prevent premature writeback.
2062 */
2063 if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
2064 if (nsegs == 1)
2065 tso_desc = TRUE;
2066 adapter->tx_tso = FALSE;
2067 }
2068
2069 if (nsegs > (adapter->num_tx_desc_avail - 2)) {
2070 adapter->no_tx_desc_avail2++;
2071 bus_dmamap_unload(adapter->txtag, map);
2072 return (ENOBUFS);
2073 }
2074 m_head = *m_headp;
2075
2076 /* Do hardware assists */
2c3d0901 2077#ifdef NET_TSO
d9d871af
MN
2078 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2079 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
2080 if (error != TRUE)
2081 return (ENXIO); /* something foobar */
2082 /* we need to make a final sentinel transmit desc */
2083 tso_desc = TRUE;
2084 } else
2085#endif
2086 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2087 em_transmit_checksum_setup(adapter, m_head,
2088 &txd_upper, &txd_lower);
2089
2090 i = adapter->next_avail_tx_desc;
2091 if (adapter->pcix_82544)
2092 txd_saved = i;
2093
2094 /* Set up our transmit descriptors */
2095 for (j = 0; j < nsegs; j++) {
2096 bus_size_t seg_len;
2097 bus_addr_t seg_addr;
2098 /* If adapter is 82544 and on PCIX bus */
2099 if(adapter->pcix_82544) {
2100 DESC_ARRAY desc_array;
2101 u32 array_elements, counter;
2102 /*
2103 * Check the Address and Length combination and
2104 * split the data accordingly
2105 */
2106 array_elements = em_fill_descriptors(segs[j].ds_addr,
2107 segs[j].ds_len, &desc_array);
2108 for (counter = 0; counter < array_elements; counter++) {
2109 if (txd_used == adapter->num_tx_desc_avail) {
2110 adapter->next_avail_tx_desc = txd_saved;
2111 adapter->no_tx_desc_avail2++;
2112 bus_dmamap_unload(adapter->txtag, map);
2113 return (ENOBUFS);
2114 }
2115 tx_buffer = &adapter->tx_buffer_area[i];
2116 ctxd = &adapter->tx_desc_base[i];
2117 ctxd->buffer_addr = htole64(
2118 desc_array.descriptor[counter].address);
2119 ctxd->lower.data = htole32(
2120 (adapter->txd_cmd | txd_lower | (u16)
2121 desc_array.descriptor[counter].length));
2122 ctxd->upper.data =
2123 htole32((txd_upper));
2124 last = i;
2125 if (++i == adapter->num_tx_desc)
2126 i = 0;
2127 tx_buffer->m_head = NULL;
2128 tx_buffer->next_eop = -1;
2129 txd_used++;
2130 }
2131 } else {
2132 tx_buffer = &adapter->tx_buffer_area[i];
2133 ctxd = &adapter->tx_desc_base[i];
2134 seg_addr = segs[j].ds_addr;
2135 seg_len = segs[j].ds_len;
2136 /*
2137 ** TSO Workaround:
2138 ** If this is the last descriptor, we want to
2139 ** split it so we have a small final sentinel
2140 */
2141 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2142 seg_len -= 4;
2143 ctxd->buffer_addr = htole64(seg_addr);
2144 ctxd->lower.data = htole32(
2145 adapter->txd_cmd | txd_lower | seg_len);
2146 ctxd->upper.data =
2147 htole32(txd_upper);
2148 if (++i == adapter->num_tx_desc)
2149 i = 0;
2150 /* Now make the sentinel */
2151 ++txd_used; /* using an extra txd */
2152 ctxd = &adapter->tx_desc_base[i];
2153 tx_buffer = &adapter->tx_buffer_area[i];
2154 ctxd->buffer_addr =
2155 htole64(seg_addr + seg_len);
2156 ctxd->lower.data = htole32(
2157 adapter->txd_cmd | txd_lower | 4);
2158 ctxd->upper.data =
2159 htole32(txd_upper);
2160 last = i;
2161 if (++i == adapter->num_tx_desc)
2162 i = 0;
2163 } else {
2164 ctxd->buffer_addr = htole64(seg_addr);
2165 ctxd->lower.data = htole32(
2166 adapter->txd_cmd | txd_lower | seg_len);
2167 ctxd->upper.data =
2168 htole32(txd_upper);
2169 last = i;
2170 if (++i == adapter->num_tx_desc)
2171 i = 0;
2172 }
2173 tx_buffer->m_head = NULL;
2174 tx_buffer->next_eop = -1;
2175 }
2176 }
2177
2178 adapter->next_avail_tx_desc = i;
2179 if (adapter->pcix_82544)
2180 adapter->num_tx_desc_avail -= txd_used;
2181 else {
2182 adapter->num_tx_desc_avail -= nsegs;
2183 if (tso_desc) /* TSO used an extra for sentinel */
2184 adapter->num_tx_desc_avail -= txd_used;
2185 }
2186
2187 /*
2c3d0901 2188 ** Handle VLAN tag
d9d871af 2189 */
d9d871af
MN
2190 if (m_head->m_flags & M_VLANTAG) {
2191 /* Set the vlan id. */
2192 ctxd->upper.fields.special =
2c3d0901 2193 htole16(m_head->m_pkthdr.ether_vlantag);
d9d871af
MN
2194 /* Tell hardware to add tag */
2195 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
2196 }
2197
2198 tx_buffer->m_head = m_head;
2199 tx_buffer_mapped->map = tx_buffer->map;
2200 tx_buffer->map = map;
2201 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
2202
2203 /*
2204 * Last Descriptor of Packet
2205 * needs End Of Packet (EOP)
2206 * and Report Status (RS)
2207 */
2208 ctxd->lower.data |=
2209 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2210 /*
2211 * Keep track in the first buffer which
2212 * descriptor will be written back
2213 */
2214 tx_buffer = &adapter->tx_buffer_area[first];
2215 tx_buffer->next_eop = last;
2216 adapter->watchdog_time = ticks;
2217
2218 /*
2219 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2220 * that this frame is available to transmit.
2221 */
2222 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2223 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2224 if (adapter->hw.mac.type == e1000_82547 &&
2225 adapter->link_duplex == HALF_DUPLEX)
2226 em_82547_move_tail(adapter);
2227 else {
2228 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
2229 if (adapter->hw.mac.type == e1000_82547)
2230 em_82547_update_fifo_head(adapter,
2231 m_head->m_pkthdr.len);
2232 }
2233
2234 return (0);
2235}
2236
2237/*********************************************************************
2238 *
2239 * 82547 workaround to avoid controller hang in half-duplex environment.
2240 * The workaround is to avoid queuing a large packet that would span
2241 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
2242 * in this case. We do that only when FIFO is quiescent.
2243 *
2244 **********************************************************************/
2245static void
2c3d0901 2246em_82547_move_tail_locked(void *arg)
d9d871af
MN
2247{
2248 struct adapter *adapter = arg;
2c3d0901 2249
d9d871af
MN
2250 struct e1000_tx_desc *tx_desc;
2251 u16 hw_tdt, sw_tdt, length = 0;
2252 bool eop = 0;
2253
2254 EM_TX_LOCK_ASSERT(adapter);
2255
2256 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2257 sw_tdt = adapter->next_avail_tx_desc;
2258
2259 while (hw_tdt != sw_tdt) {
2260 tx_desc = &adapter->tx_desc_base[hw_tdt];
2261 length += tx_desc->lower.flags.length;
2262 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2263 if (++hw_tdt == adapter->num_tx_desc)
2264 hw_tdt = 0;
2265
2266 if (eop) {
2267 if (em_82547_fifo_workaround(adapter, length)) {
2268 adapter->tx_fifo_wrk_cnt++;
2269 callout_reset(&adapter->tx_fifo_timer, 1,
2270 em_82547_move_tail, adapter);
2271 break;
2272 }
2273 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2274 em_82547_update_fifo_head(adapter, length);
2275 length = 0;
2276 }
2277 }
2278}
2279
2c3d0901
MN
2280static void
2281em_82547_move_tail(void *arg)
2282{
2283 struct adapter *adapter = arg;
2284 EM_TX_LOCK(adapter);
2285 em_82547_move_tail_locked(arg);
2286 EM_TX_UNLOCK(adapter);
2287}
2288
d9d871af
MN
2289static int
2290em_82547_fifo_workaround(struct adapter *adapter, int len)
2291{
2292 int fifo_space, fifo_pkt_len;
2293
2294 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2295
2296 if (adapter->link_duplex == HALF_DUPLEX) {
2297 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2298
2299 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2300 if (em_82547_tx_fifo_reset(adapter))
2301 return (0);
2302 else
2303 return (1);
2304 }
2305 }
2306
2307 return (0);
2308}
2309
2310static void
2311em_82547_update_fifo_head(struct adapter *adapter, int len)
2312{
2313 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2314
2315 /* tx_fifo_head is always 16 byte aligned */
2316 adapter->tx_fifo_head += fifo_pkt_len;
2317 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
2318 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2319 }
2320}
2321
2322
2323static int
2324em_82547_tx_fifo_reset(struct adapter *adapter)
2325{
2326 u32 tctl;
2327
2328 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2329 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2330 (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
2331 E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2332 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2333 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2334 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2335 /* Disable TX unit */
2336 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2337 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2338 tctl & ~E1000_TCTL_EN);
2339
2340 /* Reset FIFO pointers */
2341 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2342 adapter->tx_head_addr);
2343 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2344 adapter->tx_head_addr);
2345 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2346 adapter->tx_head_addr);
2347 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2348 adapter->tx_head_addr);
2349
2350 /* Re-enable TX unit */
2351 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2352 E1000_WRITE_FLUSH(&adapter->hw);
2353
2354 adapter->tx_fifo_head = 0;
2355 adapter->tx_fifo_reset_cnt++;
2356
2357 return (TRUE);
2358 }
2359 else {
2360 return (FALSE);
2361 }
2362}
2363
2364static void
2365em_set_promisc(struct adapter *adapter)
2366{
2367 struct ifnet *ifp = adapter->ifp;
2368 u32 reg_rctl;
2369
2370 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2371
2372 if (ifp->if_flags & IFF_PROMISC) {
2373 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2374 /* Turn this on if you want to see bad packets */
2375 if (em_debug_sbp)
2376 reg_rctl |= E1000_RCTL_SBP;
2377 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2378 } else if (ifp->if_flags & IFF_ALLMULTI) {
2379 reg_rctl |= E1000_RCTL_MPE;
2380 reg_rctl &= ~E1000_RCTL_UPE;
2381 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2382 }
2383}
2384
2385static void
2386em_disable_promisc(struct adapter *adapter)
2387{
2388 u32 reg_rctl;
2389
2390 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2391
2392 reg_rctl &= (~E1000_RCTL_UPE);
2393 reg_rctl &= (~E1000_RCTL_MPE);
2394 reg_rctl &= (~E1000_RCTL_SBP);
2395 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2396}
2397
2398
2399/*********************************************************************
2400 * Multicast Update
2401 *
2402 * This routine is called whenever multicast address list is updated.
2403 *
2404 **********************************************************************/
2405
2406static void
2407em_set_multi(struct adapter *adapter)
2408{
2409 struct ifnet *ifp = adapter->ifp;
2410 struct ifmultiaddr *ifma;
2411 u32 reg_rctl = 0;
2412 u8 *mta; /* Multicast array memory */
2413 int mcnt = 0;
2414
2415 IOCTL_DEBUGOUT("em_set_multi: begin");
2416
2417 if (adapter->hw.mac.type == e1000_82542 &&
2418 adapter->hw.revision_id == E1000_REVISION_2) {
2419 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2420 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2421 e1000_pci_clear_mwi(&adapter->hw);
2422 reg_rctl |= E1000_RCTL_RST;
2423 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2424 msec_delay(5);
2425 }
2426
2427 /* Allocate temporary memory to setup array */
2c3d0901 2428 mta = kmalloc(sizeof(u8) *
d9d871af
MN
2429 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
2430 M_DEVBUF, M_NOWAIT | M_ZERO);
2431 if (mta == NULL)
2432 panic("em_set_multi memory failure\n");
2433
2c3d0901 2434#if 0
d9d871af
MN
2435#if __FreeBSD_version < 800000
2436 IF_ADDR_LOCK(ifp);
2437#else
2438 if_maddr_rlock(ifp);
2439#endif
2c3d0901 2440#endif
441d34b2 2441 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
d9d871af
MN
2442 if (ifma->ifma_addr->sa_family != AF_LINK)
2443 continue;
2444
2445 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2446 break;
2447
2448 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2449 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2450 mcnt++;
2451 }
2c3d0901 2452#if 0
d9d871af
MN
2453#if __FreeBSD_version < 800000
2454 IF_ADDR_UNLOCK(ifp);
2455#else
2456 if_maddr_runlock(ifp);
2c3d0901 2457#endif
d9d871af
MN
2458#endif
2459 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2460 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2461 reg_rctl |= E1000_RCTL_MPE;
2462 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2463 } else
2464 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2465
2466 if (adapter->hw.mac.type == e1000_82542 &&
2467 adapter->hw.revision_id == E1000_REVISION_2) {
2468 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2469 reg_rctl &= ~E1000_RCTL_RST;
2470 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2471 msec_delay(5);
2472 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2473 e1000_pci_set_mwi(&adapter->hw);
2474 }
2c3d0901 2475 kfree(mta, M_DEVBUF);
d9d871af
MN
2476}
2477
2478
2479/*********************************************************************
2480 * Timer routine
2481 *
2482 * This routine checks for link status and updates statistics.
2483 *
2484 **********************************************************************/
2485
2486static void
2c3d0901 2487em_local_timer_locked(void *arg)
d9d871af
MN
2488{
2489 struct adapter *adapter = arg;
2490 struct ifnet *ifp = adapter->ifp;
2491
2492 EM_CORE_LOCK_ASSERT(adapter);
2493
2494#ifndef DEVICE_POLLING
2495 taskqueue_enqueue(adapter->tq,
2496 &adapter->rxtx_task);
2497#endif
2498 em_update_link_status(adapter);
2499 em_update_stats_counters(adapter);
2500
2501 /* Reset LAA into RAR[0] on 82571 */
2502 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2503 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2504
2c3d0901 2505 if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
d9d871af
MN
2506 em_print_hw_stats(adapter);
2507
2508 em_smartspeed(adapter);
2509
2510 /*
2511 * We check the watchdog: the time since
2512 * the last TX descriptor was cleaned.
2513 * This implies a functional TX engine.
2514 */
2515 if ((adapter->watchdog_check == TRUE) &&
2516 (ticks - adapter->watchdog_time > EM_WATCHDOG))
2517 goto hung;
2518
2519 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2520 return;
2521hung:
2522 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2c3d0901 2523 adapter->ifp->if_flags &= ~IFF_RUNNING;
d9d871af
MN
2524 adapter->watchdog_events++;
2525 em_init_locked(adapter);
2526}
2527
2c3d0901
MN
2528static void
2529em_local_timer(void *arg)
2530{
2531 struct adapter *adapter = arg;
2532
2533 EM_CORE_LOCK(adapter);
2534 em_local_timer_locked(arg);
2535 EM_CORE_UNLOCK(adapter);
2536}
2537
2538
d9d871af
MN
2539static void
2540em_update_link_status(struct adapter *adapter)
2541{
2542 struct e1000_hw *hw = &adapter->hw;
2543 struct ifnet *ifp = adapter->ifp;
2544 device_t dev = adapter->dev;
2545 u32 link_check = 0;
2546
2547 /* Get the cached link value or read phy for real */
2548 switch (hw->phy.media_type) {
2549 case e1000_media_type_copper:
2550 if (hw->mac.get_link_status) {
2551 /* Do the work to read phy */
2552 e1000_check_for_link(hw);
2553 link_check = !hw->mac.get_link_status;
2554 if (link_check) /* ESB2 fix */
2555 e1000_cfg_on_link_up(hw);
2556 } else
2557 link_check = TRUE;
2558 break;
2559 case e1000_media_type_fiber:
2560 e1000_check_for_link(hw);
2561 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2562 E1000_STATUS_LU);
2563 break;
2564 case e1000_media_type_internal_serdes:
2565 e1000_check_for_link(hw);
2566 link_check = adapter->hw.mac.serdes_has_link;
2567 break;
2568 default:
2569 case e1000_media_type_unknown:
2570 break;
2571 }
2572
2573 /* Now check for a transition */
2574 if (link_check && (adapter->link_active == 0)) {
2575 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2576 &adapter->link_duplex);
2577 /* Check if we must disable SPEED_MODE bit on PCI-E */
2578 if ((adapter->link_speed != SPEED_1000) &&
2579 ((hw->mac.type == e1000_82571) ||
2580 (hw->mac.type == e1000_82572))) {
2581 int tarc0;
2582 tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2583 tarc0 &= ~SPEED_MODE_BIT;
2584 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2585 }
2586 if (bootverbose)
2587 device_printf(dev, "Link is up %d Mbps %s\n",
2588 adapter->link_speed,
2589 ((adapter->link_duplex == FULL_DUPLEX) ?
2590 "Full Duplex" : "Half Duplex"));
2591 adapter->link_active = 1;
2592 adapter->smartspeed = 0;
2593 ifp->if_baudrate = adapter->link_speed * 1000000;
2c3d0901
MN
2594 ifp->if_link_state = LINK_STATE_UP;
2595 if_link_state_change(ifp);
d9d871af
MN
2596 } else if (!link_check && (adapter->link_active == 1)) {
2597 ifp->if_baudrate = adapter->link_speed = 0;
2598 adapter->link_duplex = 0;
2599 if (bootverbose)
2600 device_printf(dev, "Link is Down\n");
2601 adapter->link_active = 0;
2602 /* Link down, disable watchdog */
2603 adapter->watchdog_check = FALSE;
2c3d0901
MN
2604 ifp->if_link_state = LINK_STATE_DOWN;
2605 if_link_state_change(ifp);
d9d871af
MN
2606 }
2607}
2608
2609/*********************************************************************
2610 *
2611 * This routine disables all traffic on the adapter by issuing a
2612 * global reset on the MAC and deallocates TX/RX buffers.
2613 *
2614 * This routine should always be called with BOTH the CORE
2615 * and TX locks.
2616 **********************************************************************/
2617
2618static void
2619em_stop(void *arg)
2620{
2621 struct adapter *adapter = arg;
2622 struct ifnet *ifp = adapter->ifp;
2623
2624 EM_CORE_LOCK_ASSERT(adapter);
2625 EM_TX_LOCK_ASSERT(adapter);
2626
2627 INIT_DEBUGOUT("em_stop: begin");
2628
2629 em_disable_intr(adapter);
2630 callout_stop(&adapter->timer);
2631 callout_stop(&adapter->tx_fifo_timer);
2632
2633 /* Tell the stack that the interface is no longer active */
2c3d0901 2634 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
d9d871af
MN
2635
2636 e1000_reset_hw(&adapter->hw);
2637 if (adapter->hw.mac.type >= e1000_82544)
2638 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2639}
2640
2641
2642/*********************************************************************
2643 *
2644 * Determine hardware revision.
2645 *
2646 **********************************************************************/
2647static void
2648em_identify_hardware(struct adapter *adapter)
2649{
2650 device_t dev = adapter->dev;
2651
2652 /* Make sure our PCI config space has the necessary stuff set */
2653 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2654 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2655 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2656 device_printf(dev, "Memory Access and/or Bus Master bits "
2657 "were not set!\n");
2658 adapter->hw.bus.pci_cmd_word |=
2659 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2660 pci_write_config(dev, PCIR_COMMAND,
2661 adapter->hw.bus.pci_cmd_word, 2);
2662 }
2663
2664 /* Save off the information about this board */
2665 adapter->hw.vendor_id = pci_get_vendor(dev);
2666 adapter->hw.device_id = pci_get_device(dev);
2667 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2668 adapter->hw.subsystem_vendor_id =
2669 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2670 adapter->hw.subsystem_device_id =
2671 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2672
2673 /* Do Shared Code Init and Setup */
2674 if (e1000_set_mac_type(&adapter->hw)) {
2675 device_printf(dev, "Setup init failure\n");
2676 return;
2677 }
2678}
2679
2680static int
2681em_allocate_pci_resources(struct adapter *adapter)
2682{
2683 device_t dev = adapter->dev;
2684 int val, rid, error = E1000_SUCCESS;
2685
2686 rid = PCIR_BAR(0);
2687 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2688 &rid, RF_ACTIVE);
2689 if (adapter->memory == NULL) {
2690 device_printf(dev, "Unable to allocate bus resource: memory\n");
2691 return (ENXIO);
2692 }
2693 adapter->osdep.mem_bus_space_tag =
2694 rman_get_bustag(adapter->memory);
2695 adapter->osdep.mem_bus_space_handle =
2696 rman_get_bushandle(adapter->memory);
2697 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2698
2699 /* Only older adapters use IO mapping */
2700 if ((adapter->hw.mac.type > e1000_82543) &&
2701 (adapter->hw.mac.type < e1000_82571)) {
2702 /* Figure our where our IO BAR is ? */
2703 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2704 val = pci_read_config(dev, rid, 4);
2705 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2706 adapter->io_rid = rid;
2707 break;
2708 }
2709 rid += 4;
2710 /* check for 64bit BAR */
2711 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2712 rid += 4;
2713 }
2714 if (rid >= PCIR_CIS) {
2715 device_printf(dev, "Unable to locate IO BAR\n");
2716 return (ENXIO);
2717 }
2718 adapter->ioport = bus_alloc_resource_any(dev,
2719 SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2720 if (adapter->ioport == NULL) {
2721 device_printf(dev, "Unable to allocate bus resource: "
2722 "ioport\n");
2723 return (ENXIO);
2724 }
2725 adapter->hw.io_base = 0;
2726 adapter->osdep.io_bus_space_tag =
2727 rman_get_bustag(adapter->ioport);
2728 adapter->osdep.io_bus_space_handle =
2729 rman_get_bushandle(adapter->ioport);
2730 }
2731
2732 /*
2733 ** Init the resource arrays
2734 ** used by MSIX setup
2735 */
2736 for (int i = 0; i < 3; i++) {
2737 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2738 adapter->tag[i] = NULL;
2739 adapter->res[i] = NULL;
2740 }
2741
2742 /*
2743 * Setup MSI/X or MSI if PCI Express
2744 */
2745 if (em_enable_msi)
2746 adapter->msi = em_setup_msix(adapter);
2747
2748 adapter->hw.back = &adapter->osdep;
2749
2750 return (error);
2751}
2752
2753/*********************************************************************
2754 *
2755 * Setup the Legacy or MSI Interrupt handler
2756 *
2757 **********************************************************************/
2758int
2759em_allocate_legacy(struct adapter *adapter)
2760{
2761 device_t dev = adapter->dev;
2762 int error;
2763
2764 /* Manually turn off all interrupts */
2765 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2766
2767 /* Legacy RID is 0 */
2768 if (adapter->msi == 0)
2769 adapter->rid[0] = 0;
2770
2771 /* We allocate a single interrupt resource */
2772 adapter->res[0] = bus_alloc_resource_any(dev,
2773 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2774 if (adapter->res[0] == NULL) {
2775 device_printf(dev, "Unable to allocate bus resource: "
2776 "interrupt\n");
2777 return (ENXIO);
2778 }
2779
2780#ifdef EM_LEGACY_IRQ
2781 /* We do Legacy setup */
2782 if ((error = bus_setup_intr(dev, adapter->res[0],
2c3d0901
MN
2783 /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_intr, adapter,
2784 &adapter->tag[0], NULL)) != 0) {
d9d871af
MN
2785 device_printf(dev, "Failed to register interrupt handler");
2786 return (error);
2787 }
2788
2789#else /* FAST_IRQ */
2790 /*
2791 * Try allocating a fast interrupt and the associated deferred
2792 * processing contexts.
2793 */
2794 TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2795 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2c3d0901 2796 adapter->tq = taskqueue_create("em_taskq", M_NOWAIT,
d9d871af 2797 taskqueue_thread_enqueue, &adapter->tq);
2c3d0901 2798 taskqueue_start_threads(&adapter->tq, 1, TDPRI_KERN_DAEMON /*PI_NET*/, -1, "%s taskq",
d9d871af 2799 device_get_nameunit(adapter->dev));
d9d871af 2800 if ((error = bus_setup_intr(dev, adapter->res[0],
f8a09be1 2801 /*INTR_TYPE_NET |*/ 0, em_irq_fast, adapter,
2c3d0901 2802 &adapter->tag[0], NULL)) != 0) {
d9d871af
MN
2803 device_printf(dev, "Failed to register fast interrupt "
2804 "handler: %d\n", error);
2805 taskqueue_free(adapter->tq);
2806 adapter->tq = NULL;
2807 return (error);
2808 }
2809#endif /* EM_LEGACY_IRQ */
2810
2811 return (0);
2812}
2813
2814/*********************************************************************
2815 *
2816 * Setup the MSIX Interrupt handlers
2817 * This is not really Multiqueue, rather
2818 * its just multiple interrupt vectors.
2819 *
2820 **********************************************************************/
2821int
2822em_allocate_msix(struct adapter *adapter)
2823{
2824 device_t dev = adapter->dev;
2825 int error;
2826
2827 /* Make sure all interrupts are disabled */
2828 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2829
2830 /* First get the resources */
2831 for (int i = 0; i < adapter->msi; i++) {
2832 adapter->res[i] = bus_alloc_resource_any(dev,
2833 SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE);
2834 if (adapter->res[i] == NULL) {
2835 device_printf(dev,
2836 "Unable to allocate bus resource: "
2837 "MSIX Interrupt\n");
2838 return (ENXIO);
2839 }
2840 }
2841
2842 /*
2843 * Now allocate deferred processing contexts.
2844 */
2845 TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter);
2846 TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter);
2847 /*
2848 * Handle compatibility for msi case for deferral due to
2849 * trylock failure
2850 */
2851 TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter);
2852 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2c3d0901 2853 adapter->tq = taskqueue_create("em_taskq", M_NOWAIT,
d9d871af 2854 taskqueue_thread_enqueue, &adapter->tq);
2c3d0901 2855 taskqueue_start_threads(&adapter->tq, 1, TDPRI_KERN_DAEMON /*PI_NET*/, -1, "%s taskq",
d9d871af
MN
2856 device_get_nameunit(adapter->dev));
2857
2858 /*
2859 * And setup the interrupt handlers
2860 */
2861
2862 /* First slot to RX */
2863 if ((error = bus_setup_intr(dev, adapter->res[0],
2c3d0901
MN
2864 /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_msix_rx, adapter,
2865 &adapter->tag[0], NULL)) != 0) {
d9d871af
MN
2866 device_printf(dev, "Failed to register RX handler");
2867 return (error);
2868 }
2869
2870 /* Next TX */
2871 if ((error = bus_setup_intr(dev, adapter->res[1],
2c3d0901
MN
2872 /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_msix_tx, adapter,
2873 &adapter->tag[1], NULL)) != 0) {
d9d871af
MN
2874 device_printf(dev, "Failed to register TX handler");
2875 return (error);
2876 }
2877
2878 /* And Link */
2879 if ((error = bus_setup_intr(dev, adapter->res[2],
2c3d0901
MN
2880 /*INTR_TYPE_NET |*/ INTR_MPSAFE, em_msix_link, adapter,
2881 &adapter->tag[2], NULL)) != 0) {
d9d871af
MN
2882 device_printf(dev, "Failed to register TX handler");
2883 return (error);
2884 }
2885
2886 return (0);
2887}
2888
2889
2890static void
2891em_free_pci_resources(struct adapter *adapter)
2892{
2893 device_t dev = adapter->dev;
2894
2895 /* Make sure the for loop below runs once */
2896 if (adapter->msi == 0)
2897 adapter->msi = 1;
2898
2899 /*
2900 * First release all the interrupt resources:
2901 * notice that since these are just kept
2902 * in an array we can do the same logic
2903 * whether its MSIX or just legacy.
2904 */
2905 for (int i = 0; i < adapter->msi; i++) {
2906 if (adapter->tag[i] != NULL) {
2907 bus_teardown_intr(dev, adapter->res[i],
2908 adapter->tag[i]);
2909 adapter->tag[i] = NULL;
2910 }
2911 if (adapter->res[i] != NULL) {
2912 bus_release_resource(dev, SYS_RES_IRQ,
2913 adapter->rid[i], adapter->res[i]);
2914 }
2915 }
2916
2917 if (adapter->msi)
2918 pci_release_msi(dev);
2919
2920 if (adapter->msix != NULL)
2921 bus_release_resource(dev, SYS_RES_MEMORY,
2922 PCIR_BAR(EM_MSIX_BAR), adapter->msix);
2923
2924 if (adapter->memory != NULL)
2925 bus_release_resource(dev, SYS_RES_MEMORY,
2926 PCIR_BAR(0), adapter->memory);
2927
2928 if (adapter->flash != NULL)
2929 bus_release_resource(dev, SYS_RES_MEMORY,
2930 EM_FLASH, adapter->flash);
2931
2932 if (adapter->ioport != NULL)
2933 bus_release_resource(dev, SYS_RES_IOPORT,
2934 adapter->io_rid, adapter->ioport);
2935}
2936
2937/*
2938 * Setup MSI or MSI/X
2939 */
2940static int
2941em_setup_msix(struct adapter *adapter)
2942{
2943 device_t dev = adapter->dev;
2944 int val = 0;
2945
2946 if (adapter->hw.mac.type < e1000_82571)
2947 return (0);
2948
2949 /* Setup MSI/X for Hartwell */
2950 if (adapter->hw.mac.type == e1000_82574) {
2951 /* Map the MSIX BAR */
2952 int rid = PCIR_BAR(EM_MSIX_BAR);
2953 adapter->msix = bus_alloc_resource_any(dev,
2954 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2955 if (!adapter->msix) {
2956 /* May not be enabled */
2957 device_printf(adapter->dev,
2958 "Unable to map MSIX table \n");
2959 goto msi;
2960 }
2961 val = pci_msix_count(dev);
2962 /*
2963 ** 82574 can be configured for 5 but
2964 ** we limit use to 3.
2965 */
2966 if (val > 3) val = 3;
2967 if ((val) && pci_alloc_msix(dev, &val) == 0) {
2968 device_printf(adapter->dev,"Using MSIX interrupts\n");
2969 return (val);
2970 }
2971 }
2972msi:
2973 val = pci_msi_count(dev);
2974 if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2975 adapter->msi = 1;
2976 device_printf(adapter->dev,"Using MSI interrupt\n");
2977 return (val);
2978 }
2979 return (0);
2980}
2981
2982/*********************************************************************
2983 *
2984 * Initialize the hardware to a configuration
2985 * as specified by the adapter structure.
2986 *
2987 **********************************************************************/
2988static int
2989em_hardware_init(struct adapter *adapter)
2990{
2991 device_t dev = adapter->dev;
2992 u16 rx_buffer_size;
2993
2994 INIT_DEBUGOUT("em_hardware_init: begin");
2995
2996 /* Issue a global reset */
2997 e1000_reset_hw(&adapter->hw);
2998
2999 /* When hardware is reset, fifo_head is also reset */
3000 adapter->tx_fifo_head = 0;
3001
3002 /* Set up smart power down as default off on newer adapters. */
3003 if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 ||
3004 adapter->hw.mac.type == e1000_82572)) {
3005 u16 phy_tmp = 0;
3006
3007 /* Speed up time to link by disabling smart power down. */
3008 e1000_read_phy_reg(&adapter->hw,
3009 IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3010 phy_tmp &= ~IGP02E1000_PM_SPD;
3011 e1000_write_phy_reg(&adapter->hw,
3012 IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3013 }
3014
3015 /*
3016 * These parameters control the automatic generation (Tx) and
3017 * response (Rx) to Ethernet PAUSE frames.
3018 * - High water mark should allow for at least two frames to be
3019 * received after sending an XOFF.
3020 * - Low water mark works best when it is very near the high water mark.
3021 * This allows the receiver to restart by sending XON when it has
3022 * drained a bit. Here we use an arbitary value of 1500 which will
3023 * restart after one full frame is pulled from the buffer. There
3024 * could be several smaller frames in the buffer and if so they will
3025 * not trigger the XON until their total number reduces the buffer
3026 * by 1500.
3027 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3028 */
3029 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
3030 0xffff) << 10 );
3031
3032 adapter->hw.fc.high_water = rx_buffer_size -
3033 roundup2(adapter->max_frame_size, 1024);
3034 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
3035
3036 if (adapter->hw.mac.type == e1000_80003es2lan)
3037 adapter->hw.fc.pause_time = 0xFFFF;
3038 else
3039 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
3040 adapter->hw.fc.send_xon = TRUE;
3041
3042 /* Set Flow control, use the tunable location if sane */
3043 if ((em_fc_setting >= 0) || (em_fc_setting < 4))
3044 adapter->hw.fc.requested_mode = em_fc_setting;
3045 else
3046 adapter->hw.fc.requested_mode = e1000_fc_none;
3047
3048 /* Override - workaround for PCHLAN issue */
3049 if (adapter->hw.mac.type == e1000_pchlan)
3050 adapter->hw.fc.requested_mode = e1000_fc_rx_pause;
3051
3052 if (e1000_init_hw(&adapter->hw) < 0) {
3053 device_printf(dev, "Hardware Initialization Failed\n");
3054 return (EIO);
3055 }
3056
3057 e1000_check_for_link(&adapter->hw);
3058
3059 return (0);
3060}
3061
3062/*********************************************************************
3063 *
3064 * Setup networking device structure and register an interface.
3065 *
3066 **********************************************************************/
3067static void
3068em_setup_interface(device_t dev, struct adapter *adapter)
3069{
3070 struct ifnet *ifp;
3071
3072 INIT_DEBUGOUT("em_setup_interface: begin");
3073
2c3d0901 3074 ifp = adapter->ifp = &adapter->arpcom.ac_if;
d9d871af
MN
3075 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3076 ifp->if_mtu = ETHERMTU;
3077 ifp->if_init = em_init;
3078 ifp->if_softc = adapter;
3079 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3080 ifp->if_ioctl = em_ioctl;
3081 ifp->if_start = em_start;
2c3d0901
MN
3082 ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
3083 ifq_set_ready(&ifp->if_snd);
d9d871af 3084
2c3d0901 3085 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
d9d871af
MN
3086
3087 ifp->if_capabilities = ifp->if_capenable = 0;
3088
3089#if __FreeBSD_version >= 800000
3090 /* Multiqueue tx functions */
3091 ifp->if_transmit = em_mq_start;
3092 ifp->if_qflush = em_qflush;
3093 adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx);
3094#endif
3095 if (adapter->hw.mac.type >= e1000_82543) {
3096 int version_cap;
d9d871af 3097 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
d9d871af
MN
3098 ifp->if_capabilities |= version_cap;
3099 ifp->if_capenable |= version_cap;
3100 }
3101
2c3d0901 3102#ifdef NET_TSO
d9d871af
MN
3103 /* Identify TSO capable adapters */
3104 if ((adapter->hw.mac.type > e1000_82544) &&
3105 (adapter->hw.mac.type != e1000_82547))
3106 ifp->if_capabilities |= IFCAP_TSO4;
3107 /*
3108 * By default only enable on PCI-E, this
3109 * can be overriden by ifconfig.
3110 */
3111 if (adapter->hw.mac.type >= e1000_82571)
3112 ifp->if_capenable |= IFCAP_TSO4;
3113#endif
3114 /*
3115 * Tell the upper layer(s) we
3116 * support full VLAN capability
3117 */
3118 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3119 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3120 ifp->if_capenable |= (IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING);
3121
3122 /*
3123 ** Dont turn this on by default, if vlans are
3124 ** created on another pseudo device (eg. lagg)
3125 ** then vlan events are not passed thru, breaking
3126 ** operation, but with HW FILTER off it works. If
3127 ** using vlans directly on the em driver you can
3128 ** enable this and get full hardware tag filtering.
3129 */
3130 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3131
3132#ifdef DEVICE_POLLING
3133 ifp->if_capabilities |= IFCAP_POLLING;
3134#endif
3135
3136 /* Limit WOL to MAGIC, not clear others are used */
3137 if (adapter->wol) {
3138 ifp->if_capabilities |= IFCAP_WOL_MAGIC;
3139 ifp->if_capenable |= IFCAP_WOL_MAGIC;
3140 }
3141
3142 /*
3143 * Specify the media types supported by this adapter and register
3144 * callbacks to update media and link information
3145 */
3146 ifmedia_init(&adapter->media, IFM_IMASK,
3147 em_media_change, em_media_status);
3148 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3149 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3150 u_char fiber_type = IFM_1000_SX; /* default type */
3151
3152 if (adapter->hw.mac.type == e1000_82545)
3153 fiber_type = IFM_1000_LX;
3154 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3155 0, NULL);
3156 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3157 } else {
3158 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3159 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3160 0, NULL);
3161 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3162 0, NULL);
3163 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3164 0, NULL);
3165 if (adapter->hw.phy.type != e1000_phy_ife) {
3166 ifmedia_add(&adapter->media,
3167 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3168 ifmedia_add(&adapter->media,
3169 IFM_ETHER | IFM_1000_T, 0, NULL);
3170 }
3171 }
3172 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3173 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3174}
3175
3176
3177/*********************************************************************
3178 *
3179 * Workaround for SmartSpeed on 82541 and 82547 controllers
3180 *
3181 **********************************************************************/
3182static void
3183em_smartspeed(struct adapter *adapter)
3184{
3185 u16 phy_tmp;
3186
3187 if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
3188 adapter->hw.mac.autoneg == 0 ||
3189 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
3190 return;
3191
3192 if (adapter->smartspeed == 0) {
3193 /* If Master/Slave config fault is asserted twice,
3194 * we assume back-to-back */
3195 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3196 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
3197 return;
3198 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3199 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
3200 e1000_read_phy_reg(&adapter->hw,
3201 PHY_1000T_CTRL, &phy_tmp);
3202 if(phy_tmp & CR_1000T_MS_ENABLE) {
3203 phy_tmp &= ~CR_1000T_MS_ENABLE;
3204 e1000_write_phy_reg(&adapter->hw,
3205 PHY_1000T_CTRL, phy_tmp);
3206 adapter->smartspeed++;
3207 if(adapter->hw.mac.autoneg &&
3208 !e1000_copper_link_autoneg(&adapter->hw) &&
3209 !e1000_read_phy_reg(&adapter->hw,
3210 PHY_CONTROL, &phy_tmp)) {
3211 phy_tmp |= (MII_CR_AUTO_NEG_EN |
3212 MII_CR_RESTART_AUTO_NEG);
3213 e1000_write_phy_reg(&adapter->hw,
3214 PHY_CONTROL, phy_tmp);
3215 }
3216 }
3217 }
3218 return;
3219 } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
3220 /* If still no link, perhaps using 2/3 pair cable */
3221 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
3222 phy_tmp |= CR_1000T_MS_ENABLE;
3223 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
3224 if(adapter->hw.mac.autoneg &&
3225 !e1000_copper_link_autoneg(&adapter->hw) &&
3226 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
3227 phy_tmp |= (MII_CR_AUTO_NEG_EN |
3228 MII_CR_RESTART_AUTO_NEG);
3229 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
3230 }
3231 }
3232 /* Restart process after EM_SMARTSPEED_MAX iterations */
3233 if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
3234 adapter->smartspeed = 0;
3235}
3236
3237
3238/*
3239 * Manage DMA'able memory.
3240 */
3241static void
3242em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3243{
3244 if (error)
3245 return;
3246 *(bus_addr_t *) arg = segs[0].ds_addr;
3247}
3248
3249static int
3250em_dma_malloc(struct adapter *adapter, bus_size_t size,
3251 struct em_dma_alloc *dma, int mapflags)
3252{
3253 int error;
3254
2c3d0901 3255 error = bus_dma_tag_create(NULL, /* parent */
d9d871af
MN
3256 EM_DBA_ALIGN, 0, /* alignment, bounds */
3257 BUS_SPACE_MAXADDR, /* lowaddr */
3258 BUS_SPACE_MAXADDR, /* highaddr */
3259 NULL, NULL, /* filter, filterarg */
3260 size, /* maxsize */
3261 1, /* nsegments */
3262 size, /* maxsegsize */
3263 0, /* flags */
d9d871af
MN
3264 &dma->dma_tag);
3265 if (error) {
3266 device_printf(adapter->dev,
3267 "%s: bus_dma_tag_create failed: %d\n",
3268 __func__, error);
3269 goto fail_0;
3270 }
3271
3272 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3273 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3274 if (error) {
3275 device_printf(adapter->dev,
3276 "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3277 __func__, (uintmax_t)size, error);
3278 goto fail_2;
3279 }
3280
3281 dma->dma_paddr = 0;
3282 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3283 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3284 if (error || dma->dma_paddr == 0) {
3285 device_printf(adapter->dev,
3286 "%s: bus_dmamap_load failed: %d\n",
3287 __func__, error);
3288 goto fail_3;
3289 }
3290
3291 return (0);
3292
3293fail_3:
3294 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3295fail_2:
3296 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3297 bus_dma_tag_destroy(dma->dma_tag);
3298fail_0:
3299 dma->dma_map = NULL;
3300 dma->dma_tag = NULL;
3301
3302 return (error);
3303}
3304
3305static void
3306em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3307{
3308 if (dma->dma_tag == NULL)
3309 return;
3310 if (dma->dma_map != NULL) {
3311 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3312 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3313 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3314 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3315 dma->dma_map = NULL;
3316 }
3317 bus_dma_tag_destroy(dma->dma_tag);
3318 dma->dma_tag = NULL;
3319}
3320
3321
3322/*********************************************************************
3323 *
3324 * Allocate memory for tx_buffer structures. The tx_buffer stores all
3325 * the information needed to transmit a packet on the wire.
3326 *
3327 **********************************************************************/
3328static int
3329em_allocate_transmit_structures(struct adapter *adapter)
3330{
3331 device_t dev = adapter->dev;
3332 struct em_buffer *tx_buffer;
3333 int error;
3334
3335 /*
3336 * Create DMA tags for tx descriptors
3337 */
2c3d0901 3338 if ((error = bus_dma_tag_create(NULL, /* parent */
d9d871af
MN
3339 1, 0, /* alignment, bounds */
3340 BUS_SPACE_MAXADDR, /* lowaddr */
3341 BUS_SPACE_MAXADDR, /* highaddr */
3342 NULL, NULL, /* filter, filterarg */
3343 EM_TSO_SIZE, /* maxsize */
3344 EM_MAX_SCATTER, /* nsegments */
3345 EM_TSO_SEG_SIZE, /* maxsegsize */
3346 0, /* flags */
d9d871af
MN
3347 &adapter->txtag)) != 0) {
3348 device_printf(dev, "Unable to allocate TX DMA tag\n");
3349 goto fail;
3350 }
3351
2c3d0901 3352 adapter->tx_buffer_area = kmalloc(sizeof(struct em_buffer) *
d9d871af
MN
3353 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3354 if (adapter->tx_buffer_area == NULL) {
3355 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3356 error = ENOMEM;
3357 goto fail;
3358 }
3359
3360 /* Create the descriptor buffer dma maps */
3361 for (int i = 0; i < adapter->num_tx_desc; i++) {
3362 tx_buffer = &adapter->tx_buffer_area[i];
3363 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
3364 if (error != 0) {
3365 device_printf(dev, "Unable to create TX DMA map\n");
3366 goto fail;
3367 }
3368 tx_buffer->next_eop = -1;
3369 }
3370
3371 return (0);
3372fail:
3373 em_free_transmit_structures(adapter);
3374 return (error);
3375}
3376
3377/*********************************************************************
3378 *
3379 * (Re)Initialize transmit structures.
3380 *
3381 **********************************************************************/
3382static void
3383em_setup_transmit_structures(struct adapter *adapter)
3384{
3385 struct em_buffer *tx_buffer;
3386
3387 /* Clear the old ring contents */
3388 bzero(adapter->tx_desc_base,
3389 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3390
3391 /* Free any existing TX buffers */
3392 for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3393 tx_buffer = &adapter->tx_buffer_area[i];
3394 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3395 BUS_DMASYNC_POSTWRITE);
3396 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
3397 m_freem(tx_buffer->m_head);
3398 tx_buffer->m_head = NULL;
3399 tx_buffer->next_eop = -1;
3400 }
3401
3402 /* Reset state */
3403 adapter->next_avail_tx_desc = 0;
3404 adapter->next_tx_to_clean = 0;
3405 adapter->num_tx_desc_avail = adapter->num_tx_desc;
3406
3407 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3408 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3409
3410 return;
3411}
3412
3413/*********************************************************************
3414 *
3415 * Enable transmit unit.
3416 *
3417 **********************************************************************/
3418static void
3419em_initialize_transmit_unit(struct adapter *adapter)
3420{
3421 u32 tctl, tarc, tipg = 0;
3422 u64 bus_addr;
3423
3424 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3425 /* Setup the Base and Length of the Tx Descriptor Ring */
3426 bus_addr = adapter->txdma.dma_paddr;
3427 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
3428 adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3429 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
3430 (u32)(bus_addr >> 32));
3431 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
3432 (u32)bus_addr);
3433 /* Setup the HW Tx Head and Tail descriptor pointers */
3434 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
3435 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
3436
3437 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3438 E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
3439 E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
3440
3441 /* Set the default values for the Tx Inter Packet Gap timer */
3442 switch (adapter->hw.mac.type) {
3443 case e1000_82542:
3444 tipg = DEFAULT_82542_TIPG_IPGT;
3445 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3446 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3447 break;
3448 case e1000_80003es2lan:
3449 tipg = DEFAULT_82543_TIPG_IPGR1;
3450 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3451 E1000_TIPG_IPGR2_SHIFT;
3452 break;
3453 default:
3454 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3455 (adapter->hw.phy.media_type ==
3456 e1000_media_type_internal_serdes))
3457 tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3458 else
3459 tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3460 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3461 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3462 }
3463
3464 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3465 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3466 if(adapter->hw.mac.type >= e1000_82540)
3467 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3468 adapter->tx_abs_int_delay.value);
3469
3470 if ((adapter->hw.mac.type == e1000_82571) ||
3471 (adapter->hw.mac.type == e1000_82572)) {
3472 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3473 tarc |= SPEED_MODE_BIT;
3474 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3475 } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3476 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3477 tarc |= 1;
3478 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3479 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3480 tarc |= 1;
3481 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3482 }
3483
3484 /* Program the Transmit Control Register */
3485 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3486 tctl &= ~E1000_TCTL_CT;
3487 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3488 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3489
3490 if (adapter->hw.mac.type >= e1000_82571)
3491 tctl |= E1000_TCTL_MULR;
3492
3493 /* This write will effectively turn on the transmit unit. */
3494 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3495
3496 /* Setup Transmit Descriptor Base Settings */
3497 adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3498
3499 if (adapter->tx_int_delay.value > 0)
3500 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3501}
3502
3503/*********************************************************************
3504 *
3505 * Free all transmit related data structures.
3506 *
3507 **********************************************************************/
3508static void
3509em_free_transmit_structures(struct adapter *adapter)
3510{
3511 struct em_buffer *tx_buffer;
3512
3513 INIT_DEBUGOUT("free_transmit_structures: begin");
3514
3515 if (adapter->tx_buffer_area != NULL) {
3516 for (int i = 0; i < adapter->num_tx_desc; i++) {
3517 tx_buffer = &adapter->tx_buffer_area[i];
3518 if (tx_buffer->m_head != NULL) {
3519 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3520 BUS_DMASYNC_POSTWRITE);
3521 bus_dmamap_unload(adapter->txtag,
3522 tx_buffer->map);
3523 m_freem(tx_buffer->m_head);
3524 tx_buffer->m_head = NULL;
3525 } else if (tx_buffer->map != NULL)
3526 bus_dmamap_unload(adapter->txtag,
3527 tx_buffer->map);
3528 if (tx_buffer->map != NULL) {
3529 bus_dmamap_destroy(adapter->txtag,
3530 tx_buffer->map);
3531 tx_buffer->map = NULL;
3532 }
3533 }
3534 }
3535 if (adapter->tx_buffer_area != NULL) {
2c3d0901 3536 kfree(adapter->tx_buffer_area, M_DEVBUF);
d9d871af
MN
3537 adapter->tx_buffer_area = NULL;
3538 }
3539 if (adapter->txtag != NULL) {
3540 bus_dma_tag_destroy(adapter->txtag);
3541 adapter->txtag = NULL;
3542 }
3543#if __FreeBSD_version >= 800000
3544 if (adapter->br != NULL)
3545 buf_ring_free(adapter->br, M_DEVBUF);
3546#endif
3547}
3548
3549/*********************************************************************
3550 *
3551 * The offload context needs to be set when we transfer the first
3552 * packet of a particular protocol (TCP/UDP). This routine has been
3553 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3554 *
3555 * Added back the old method of keeping the current context type
3556 * and not setting if unnecessary, as this is reported to be a
3557 * big performance win. -jfv
3558 **********************************************************************/
3559static void
3560em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
3561 u32 *txd_upper, u32 *txd_lower)
3562{
3563 struct e1000_context_desc *TXD = NULL;
3564 struct em_buffer *tx_buffer;
3565 struct ether_vlan_header *eh;
3566 struct ip *ip = NULL;
3567 struct ip6_hdr *ip6;
3568 int curr_txd, ehdrlen;
3569 u32 cmd, hdr_len, ip_hlen;
3570 u16 etype;
3571 u8 ipproto;
3572
3573
3574 cmd = hdr_len = ipproto = 0;
3575 curr_txd = adapter->next_avail_tx_desc;
3576
3577 /*
3578 * Determine where frame payload starts.
3579 * Jump over vlan headers if already present,
3580 * helpful for QinQ too.
3581 */
3582 eh = mtod(mp, struct ether_vlan_header *);
3583 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3584 etype = ntohs(eh->evl_proto);
3585 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3586 } else {
3587 etype = ntohs(eh->evl_encap_proto);
3588 ehdrlen = ETHER_HDR_LEN;
3589 }
3590
3591 /*
3592 * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3593 * TODO: Support SCTP too when it hits the tree.
3594 */
3595 switch (etype) {
3596 case ETHERTYPE_IP:
3597 ip = (struct ip *)(mp->m_data + ehdrlen);
3598 ip_hlen = ip->ip_hl << 2;
3599
3600 /* Setup of IP header checksum. */
3601 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3602 /*
3603 * Start offset for header checksum calculation.
3604 * End offset for header checksum calculation.
3605 * Offset of place to put the checksum.
3606 */
3607 TXD = (struct e1000_context_desc *)
3608 &adapter->tx_desc_base[curr_txd];
3609 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3610 TXD->lower_setup.ip_fields.ipcse =
3611 htole16(ehdrlen + ip_hlen);
3612 TXD->lower_setup.ip_fields.ipcso =
3613 ehdrlen + offsetof(struct ip, ip_sum);
3614 cmd |= E1000_TXD_CMD_IP;
3615 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3616 }
3617
3618 if (mp->m_len < ehdrlen + ip_hlen)
3619 return; /* failure */
3620
3621 hdr_len = ehdrlen + ip_hlen;
3622 ipproto = ip->ip_p;
3623
3624 break;
3625 case ETHERTYPE_IPV6:
3626 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3627 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3628
3629 if (mp->m_len < ehdrlen + ip_hlen)
3630 return; /* failure */
3631
3632 /* IPv6 doesn't have a header checksum. */
3633
3634 hdr_len = ehdrlen + ip_hlen;
3635 ipproto = ip6->ip6_nxt;
3636
3637 break;
3638 default:
3639 *txd_upper = 0;
3640 *txd_lower = 0;
3641 return;
3642 }
3643
3644 switch (ipproto) {
3645 case IPPROTO_TCP:
3646 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3647 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3648 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3649 /* no need for context if already set */
3650 if (adapter->last_hw_offload == CSUM_TCP)
3651 return;
3652 adapter->last_hw_offload = CSUM_TCP;
3653 /*
3654 * Start offset for payload checksum calculation.
3655 * End offset for payload checksum calculation.
3656 * Offset of place to put the checksum.
3657 */
3658 TXD = (struct e1000_context_desc *)
3659 &adapter->tx_desc_base[curr_txd];
3660 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3661 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3662 TXD->upper_setup.tcp_fields.tucso =
3663 hdr_len + offsetof(struct tcphdr, th_sum);
3664 cmd |= E1000_TXD_CMD_TCP;
3665 }
3666 break;
3667 case IPPROTO_UDP:
3668 {
3669 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3670 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3671 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3672 /* no need for context if already set */
3673 if (adapter->last_hw_offload == CSUM_UDP)
3674 return;
3675 adapter->last_hw_offload = CSUM_UDP;
3676 /*
3677 * Start offset for header checksum calculation.
3678 * End offset for header checksum calculation.
3679 * Offset of place to put the checksum.
3680 */
3681 TXD = (struct e1000_context_desc *)
3682 &adapter->tx_desc_base[curr_txd];
3683 TXD->upper_setup.tcp_fields.tucss = hdr_len;
3684 TXD->upper_setup.tcp_fields.tucse = htole16(0);
3685 TXD->upper_setup.tcp_fields.tucso =
3686 hdr_len + offsetof(struct udphdr, uh_sum);
3687 }
3688 /* Fall Thru */
3689 }
3690 default:
3691 break;
3692 }
3693
3694 TXD->tcp_seg_setup.data = htole32(0);
3695 TXD->cmd_and_length =
3696 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3697 tx_buffer = &adapter->tx_buffer_area[curr_txd];
3698 tx_buffer->m_head = NULL;
3699 tx_buffer->next_eop = -1;
3700
3701 if (++curr_txd == adapter->num_tx_desc)
3702 curr_txd = 0;
3703
3704 adapter->num_tx_desc_avail--;
3705 adapter->next_avail_tx_desc = curr_txd;
3706}
3707
3708
2c3d0901 3709#ifdef NET_TSO
d9d871af
MN
3710/**********************************************************************
3711 *
3712 * Setup work for hardware segmentation offload (TSO)
3713 *
3714 **********************************************************************/
3715static bool
3716em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper,
3717 u32 *txd_lower)
3718{
3719 struct e1000_context_desc *TXD;
3720 struct em_buffer *tx_buffer;
3721 struct ether_vlan_header *eh;
3722 struct ip *ip;
3723 struct ip6_hdr *ip6;
3724 struct tcphdr *th;
3725 int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6;
3726 u16 etype;
3727
3728 /*
3729 * This function could/should be extended to support IP/IPv6
3730 * fragmentation as well. But as they say, one step at a time.
3731 */
3732
3733 /*
3734 * Determine where frame payload starts.
3735 * Jump over vlan headers if already present,
3736 * helpful for QinQ too.
3737 */
3738 eh = mtod(mp, struct ether_vlan_header *);
3739 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3740 etype = ntohs(eh->evl_proto);
3741 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3742 } else {
3743 etype = ntohs(eh->evl_encap_proto);
3744 ehdrlen = ETHER_HDR_LEN;
3745 }
3746
3747 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3748 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3749 return FALSE; /* -1 */
3750
3751 /*
3752 * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3753 * TODO: Support SCTP too when it hits the tree.
3754 */
3755 switch (etype) {
3756 case ETHERTYPE_IP:
3757 isip6 = 0;
3758 ip = (struct ip *)(mp->m_data + ehdrlen);
3759 if (ip->ip_p != IPPROTO_TCP)
3760 return FALSE; /* 0 */
3761 ip->ip_len = 0;
3762 ip->ip_sum = 0;
3763 ip_hlen = ip->ip_hl << 2;
3764 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3765 return FALSE; /* -1 */
3766 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3767#if 1
3768 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3769 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3770#else
3771 th->th_sum = mp->m_pkthdr.csum_data;
3772#endif
3773 break;
3774 case ETHERTYPE_IPV6:
3775 isip6 = 1;
3776 return FALSE; /* Not supported yet. */
3777 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3778 if (ip6->ip6_nxt != IPPROTO_TCP)
3779 return FALSE; /* 0 */
3780 ip6->ip6_plen = 0;
3781 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3782 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3783 return FALSE; /* -1 */
3784 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3785#if 0
3786 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3787 htons(IPPROTO_TCP)); /* XXX: function notyet. */
3788#else
3789 th->th_sum = mp->m_pkthdr.csum_data;
3790#endif
3791 break;
3792 default:
3793 return FALSE;
3794 }
3795 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3796
3797 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
3798 E1000_TXD_DTYP_D | /* Data descr type */
3799 E1000_TXD_CMD_TSE); /* Do TSE on this packet */
3800
3801 /* IP and/or TCP header checksum calculation and insertion. */
3802 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3803 E1000_TXD_POPTS_TXSM) << 8;
3804
3805 curr_txd = adapter->next_avail_tx_desc;
3806 tx_buffer = &adapter->tx_buffer_area[curr_txd];
3807 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3808
3809 /* IPv6 doesn't have a header checksum. */
3810 if (!isip6) {
3811 /*
3812 * Start offset for header checksum calculation.
3813 * End offset for header checksum calculation.
3814 * Offset of place put the checksum.
3815 */
3816 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3817 TXD->lower_setup.ip_fields.ipcse =
3818 htole16(ehdrlen + ip_hlen - 1);
3819 TXD->lower_setup.ip_fields.ipcso =
3820 ehdrlen + offsetof(struct ip, ip_sum);
3821 }
3822 /*
3823 * Start offset for payload checksum calculation.
3824 * End offset for payload checksum calculation.
3825 * Offset of place to put the checksum.
3826 */
3827 TXD->upper_setup.tcp_fields.tucss =
3828 ehdrlen + ip_hlen;
3829 TXD->upper_setup.tcp_fields.tucse = 0;
3830 TXD->upper_setup.tcp_fields.tucso =
3831 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3832 /*
3833 * Payload size per packet w/o any headers.
3834 * Length of all headers up to payload.
3835 */
3836 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3837 TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3838
3839 TXD->cmd_and_length = htole32(adapter->txd_cmd |
3840 E1000_TXD_CMD_DEXT | /* Extended descr */
3841 E1000_TXD_CMD_TSE | /* TSE context */
3842 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */
3843 E1000_TXD_CMD_TCP | /* Do TCP checksum */
3844 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3845
3846 tx_buffer->m_head = NULL;
3847 tx_buffer->next_eop = -1;
3848
3849 if (++curr_txd == adapter->num_tx_desc)
3850 curr_txd = 0;
3851
3852 adapter->num_tx_desc_avail--;
3853 adapter->next_avail_tx_desc = curr_txd;
3854 adapter->tx_tso = TRUE;
3855
3856 return TRUE;
3857}
3858
2c3d0901 3859#endif
d9d871af
MN
3860
3861/**********************************************************************
3862 *
3863 * Examine each tx_buffer in the used queue. If the hardware is done
3864 * processing the packet then free associated resources. The
3865 * tx_buffer is put back on the free queue.
3866 *
3867 **********************************************************************/
3868static void
3869em_txeof(struct adapter *adapter)
3870{
3871 int first, last, done, num_avail;
3872 struct em_buffer *tx_buffer;
3873 struct e1000_tx_desc *tx_desc, *eop_desc;
3874 struct ifnet *ifp = adapter->ifp;
3875
3876 EM_TX_LOCK_ASSERT(adapter);
3877
3878 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3879 return;
3880
3881 num_avail = adapter->num_tx_desc_avail;
3882 first = adapter->next_tx_to_clean;
3883 tx_desc = &adapter->tx_desc_base[first];
3884 tx_buffer = &adapter->tx_buffer_area[first];
3885 last = tx_buffer->next_eop;
3886 eop_desc = &adapter->tx_desc_base[last];
3887
3888 /*
3889 * What this does is get the index of the
3890 * first descriptor AFTER the EOP of the
3891 * first packet, that way we can do the
3892 * simple comparison on the inner while loop.
3893 */
3894 if (++last == adapter->num_tx_desc)
3895 last = 0;
3896 done = last;
3897
3898 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3899 BUS_DMASYNC_POSTREAD);
3900
3901 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3902 /* We clean the range of the packet */
3903 while (first != done) {
3904 tx_desc->upper.data = 0;
3905 tx_desc->lower.data = 0;
3906 tx_desc->buffer_addr = 0;
3907 ++num_avail;
3908
3909 if (tx_buffer->m_head) {
3910 ifp->if_opackets++;
3911 bus_dmamap_sync(adapter->txtag,
3912 tx_buffer->map,
3913 BUS_DMASYNC_POSTWRITE);
3914 bus_dmamap_unload(adapter->txtag,
3915 tx_buffer->map);
3916
3917 m_freem(tx_buffer->m_head);
3918 tx_buffer->m_head = NULL;
3919 }
3920 tx_buffer->next_eop = -1;
3921 adapter->watchdog_time = ticks;
3922
3923 if (++first == adapter->num_tx_desc)
3924 first = 0;
3925
3926 tx_buffer = &adapter->tx_buffer_area[first];
3927 tx_desc = &adapter->tx_desc_base[first];
3928 }
3929 /* See if we can continue to the next packet */
3930 last = tx_buffer->next_eop;
3931 if (last != -1) {
3932 eop_desc = &adapter->tx_desc_base[last];
3933 /* Get new done point */
3934 if (++last == adapter->num_tx_desc) last = 0;
3935 done = last;
3936 } else
3937 break;
3938 }
3939 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3940 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3941
3942 adapter->next_tx_to_clean = first;
3943
3944 /*
2c3d0901 3945 * If we have enough room, clear IFF_OACTIVE to
d9d871af
MN
3946 * tell the stack that it is OK to send packets.
3947 * If there are no pending descriptors, clear the watchdog.
3948 */
3949 if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2c3d0901 3950 ifp->if_flags &= ~IFF_OACTIVE;
d9d871af
MN
3951 if (num_avail == adapter->num_tx_desc) {
3952 adapter->watchdog_check = FALSE;