projects / dragonfly.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | history | HEAD
ixgbe: Rework ixgbe_xmit()
[dragonfly.git] / sys / dev / netif / ixgbe / ixgbe.c
CommitLineData
9407f759
FT		 1/******************************************************************************
2
3 Copyright (c) 2001-2012, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
ce451236 33/*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
9407f759
FT		 34
35#include "opt_inet.h"
36#include "opt_inet6.h"
37
38#include "ixgbe.h"
39
40/*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43int ixgbe_display_debug_stats = 0;
44
45/*********************************************************************
46 * Driver version
47 *********************************************************************/
ce451236 48char ixgbe_driver_version[] = "2.4.8";
9407f759
FT		 49
50/*********************************************************************
51 * PCI Device ID Table
52 *
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
56 *
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
59
60static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
61{
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
ce451236 81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
9407f759
FT		 82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
ce451236 84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
9407f759
FT		 85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
87 {0, 0, 0, 0, 0}
88};
89
90/*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
93
94static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
96};
97
98/*********************************************************************
99 * Function prototypes
100 *********************************************************************/
101static int ixgbe_probe(device_t);
102static int ixgbe_attach(device_t);
103static int ixgbe_detach(device_t);
104static int ixgbe_shutdown(device_t);
105static void ixgbe_start(struct ifnet *);
106static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107#if 0 /* __FreeBSD_version >= 800000 */
108static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111static void ixgbe_qflush(struct ifnet *);
112#endif
113static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114static void ixgbe_init(void *);
115static void ixgbe_init_locked(struct adapter *);
116static void ixgbe_stop(void *);
117static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118static int ixgbe_media_change(struct ifnet *);
119static void ixgbe_identify_hardware(struct adapter *);
120static int ixgbe_allocate_pci_resources(struct adapter *);
121static int ixgbe_allocate_msix(struct adapter *);
122static int ixgbe_allocate_legacy(struct adapter *);
123static int ixgbe_allocate_queues(struct adapter *);
124#if 0 /* HAVE_MSIX */
125static int ixgbe_setup_msix(struct adapter *);
126#endif
127static void ixgbe_free_pci_resources(struct adapter *);
128static void ixgbe_local_timer(void *);
129static int ixgbe_setup_interface(device_t, struct adapter *);
130static void ixgbe_config_link(struct adapter *);
131
132static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133static int ixgbe_setup_transmit_structures(struct adapter *);
134static void ixgbe_setup_transmit_ring(struct tx_ring *);
135static void ixgbe_initialize_transmit_units(struct adapter *);
136static void ixgbe_free_transmit_structures(struct adapter *);
137static void ixgbe_free_transmit_buffers(struct tx_ring *);
138
139static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140static int ixgbe_setup_receive_structures(struct adapter *);
141static int ixgbe_setup_receive_ring(struct rx_ring *);
142static void ixgbe_initialize_receive_units(struct adapter *);
143static void ixgbe_free_receive_structures(struct adapter *);
144static void ixgbe_free_receive_buffers(struct rx_ring *);
145#if 0 /* NET_LRO */
146static void ixgbe_setup_hw_rsc(struct rx_ring *);
147#endif
148
149static void ixgbe_enable_intr(struct adapter *);
150static void ixgbe_disable_intr(struct adapter *);
151static void ixgbe_update_stats_counters(struct adapter *);
152static bool ixgbe_txeof(struct tx_ring *);
153static bool ixgbe_rxeof(struct ix_queue *, int);
154static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
155static void ixgbe_set_promisc(struct adapter *);
156static void ixgbe_set_multi(struct adapter *);
157static void ixgbe_update_link_status(struct adapter *);
158static void ixgbe_refresh_mbufs(struct rx_ring *, int);
159static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
160static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
161static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
162static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
163static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
164 struct ixgbe_dma_alloc *, int);
165static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
166static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
167 const char *, int *, int);
168static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
169#if 0 /* NET_TSO */
170static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
171#endif
172static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173static void ixgbe_configure_ivars(struct adapter *);
174static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
175
176static void ixgbe_setup_vlan_hw_support(struct adapter *);
177static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
179
180static void ixgbe_add_hw_stats(struct adapter *adapter);
181
182static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
184 struct mbuf *, u32);
185
186/* Support for pluggable optic modules */
187static bool ixgbe_sfp_probe(struct adapter *);
188static void ixgbe_setup_optics(struct adapter *);
189
190/* Legacy (single vector interrupt handler */
191static void ixgbe_legacy_irq(void *);
192
193/* The MSI/X Interrupt handlers */
194static void ixgbe_msix_que(void *);
195static void ixgbe_msix_link(void *);
196
197/* Deferred interrupt tasklets */
198static void ixgbe_handle_que(void *, int);
199static void ixgbe_handle_link(void *, int);
200static void ixgbe_handle_msf(void *, int);
201static void ixgbe_handle_mod(void *, int);
202
203#ifdef IXGBE_FDIR
204static void ixgbe_atr(struct tx_ring *, struct mbuf *);
205static void ixgbe_reinit_fdir(void *, int);
206#endif
207
208/*********************************************************************
209 * FreeBSD Device Interface Entry Points
210 *********************************************************************/
211
212static device_method_t ixgbe_methods[] = {
213 /* Device interface */
214 DEVMETHOD(device_probe, ixgbe_probe),
215 DEVMETHOD(device_attach, ixgbe_attach),
216 DEVMETHOD(device_detach, ixgbe_detach),
217 DEVMETHOD(device_shutdown, ixgbe_shutdown),
218 {0, 0}
219};
220
221static driver_t ixgbe_driver = {
222 "ix", ixgbe_methods, sizeof(struct adapter),
223};
224
225devclass_t ixgbe_devclass;
226DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
227
228MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
229MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230
231/*
232** TUNEABLE PARAMETERS:
233*/
234
235/*
236** AIM: Adaptive Interrupt Moderation
237** which means that the interrupt rate
238** is varied over time based on the
239** traffic for that interrupt vector
240*/
241static int ixgbe_enable_aim = TRUE;
242TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
243
244static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
245TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
246
247/* How many packets rxeof tries to clean at a time */
248static int ixgbe_rx_process_limit = 128;
249TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
250
9407f759
FT		 251/*
252** Smart speed setting, default to on
253** this only works as a compile option
254** right now as its during attach, set
255** this to 'ixgbe_smart_speed_off' to
256** disable.
257*/
258static int ixgbe_smart_speed = ixgbe_smart_speed_on;
259
7e665ea1
FT		 260static int ixgbe_msi_enable = 1;
261TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
262
9407f759
FT		 263/*
264 * MSIX should be the default for best performance,
265 * but this allows it to be forced off for testing.
266 */
267static int ixgbe_enable_msix = 1;
268TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
269
270/*
271 * Header split: this causes the hardware to DMA
272 * the header into a separate mbuf from the payload,
273 * it can be a performance win in some workloads, but
274 * in others it actually hurts, its off by default.
275 */
276static int ixgbe_header_split = FALSE;
277TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
278
279/*
280 * Number of Queues, can be set to 0,
281 * it then autoconfigures based on the
282 * number of cpus with a max of 8. This
283 * can be overriden manually here.
284 */
285static int ixgbe_num_queues = 0;
286TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
287
288/*
289** Number of TX descriptors per ring,
290** setting higher than RX as this seems
291** the better performing choice.
292*/
293static int ixgbe_txd = PERFORM_TXD;
294TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
295
296/* Number of RX descriptors per ring */
297static int ixgbe_rxd = PERFORM_RXD;
298TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
299
300/* Keep running tab on them for sanity check */
301static int ixgbe_total_ports;
302
303#ifdef IXGBE_FDIR
304/*
305** For Flow Director: this is the
306** number of TX packets we sample
307** for the filter pool, this means
308** every 20th packet will be probed.
309**
310** This feature can be disabled by
311** setting this to 0.
312*/
313static int atr_sample_rate = 20;
314/*
315** Flow Director actually 'steals'
316** part of the packet buffer as its
317** filter pool, this variable controls
318** how much it uses:
319** 0 = 64K, 1 = 128K, 2 = 256K
320*/
321static int fdir_pballoc = 1;
322#endif
323
324#ifdef DEV_NETMAP
325/*
326 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
327 * be a reference on how to implement netmap support in a driver.
328 * Additional comments are in ixgbe_netmap.h .
329 *
330 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
331 * that extend the standard driver.
332 */
333#include <dev/netmap/ixgbe_netmap.h>
334#endif /* DEV_NETMAP */
335
336/*********************************************************************
337 * Device identification routine
338 *
339 * ixgbe_probe determines if the driver should be loaded on
340 * adapter based on PCI vendor/device id of the adapter.
341 *
342 * return BUS_PROBE_DEFAULT on success, positive on failure
343 *********************************************************************/
344
345static int
346ixgbe_probe(device_t dev)
347{
348 ixgbe_vendor_info_t *ent;
349
350 u16 pci_vendor_id = 0;
351 u16 pci_device_id = 0;
352 u16 pci_subvendor_id = 0;
353 u16 pci_subdevice_id = 0;
354 char adapter_name[256];
355
356 INIT_DEBUGOUT("ixgbe_probe: begin");
357
358 pci_vendor_id = pci_get_vendor(dev);
359 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
360 return (ENXIO);
361
362 pci_device_id = pci_get_device(dev);
363 pci_subvendor_id = pci_get_subvendor(dev);
364 pci_subdevice_id = pci_get_subdevice(dev);
365
366 ent = ixgbe_vendor_info_array;
367 while (ent->vendor_id != 0) {
368 if ((pci_vendor_id == ent->vendor_id) &&
369 (pci_device_id == ent->device_id) &&
370
371 ((pci_subvendor_id == ent->subvendor_id) ||
372 (ent->subvendor_id == 0)) &&
373
374 ((pci_subdevice_id == ent->subdevice_id) ||
375 (ent->subdevice_id == 0))) {
376 ksprintf(adapter_name, "%s, Version - %s",
377 ixgbe_strings[ent->index],
378 ixgbe_driver_version);
379 device_set_desc_copy(dev, adapter_name);
380 ++ixgbe_total_ports;
381 return (BUS_PROBE_DEFAULT);
382 }
383 ent++;
384 }
385 return (ENXIO);
386}
387
388/*********************************************************************
389 * Device initialization routine
390 *
391 * The attach entry point is called when the driver is being loaded.
392 * This routine identifies the type of hardware, allocates all resources
393 * and initializes the hardware.
394 *
395 * return 0 on success, positive on failure
396 *********************************************************************/
397
398static int
399ixgbe_attach(device_t dev)
400{
401 struct adapter *adapter;
402 struct ixgbe_hw *hw;
403 int error = 0;
404 u16 csum;
405 u32 ctrl_ext;
406
407 INIT_DEBUGOUT("ixgbe_attach: begin");
408
409 if (resource_disabled("ixgbe", device_get_unit(dev))) {
410 device_printf(dev, "Disabled by device hint\n");
411 return (ENXIO);
412 }
413
414 /* Allocate, clear, and link in our adapter structure */
415 adapter = device_get_softc(dev);
416 adapter->dev = adapter->osdep.dev = dev;
417 hw = &adapter->hw;
418
419 /* Core Lock Init*/
420 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
9407f759
FT		 421
422 /* SYSCTL APIs */
423
424 sysctl_ctx_init(&adapter->sysctl_ctx);
425 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
426 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
427 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
428 if (adapter->sysctl_tree == NULL) {
429 device_printf(adapter->dev, "can't add sysctl node\n");
430 return (EINVAL);
431 }
432 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
433 SYSCTL_CHILDREN(adapter->sysctl_tree),
434 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
435 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
436
437 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
438 SYSCTL_CHILDREN(adapter->sysctl_tree),
439 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
440 &ixgbe_enable_aim, 1, "Interrupt Moderation");
441
442 /*
443 ** Allow a kind of speed control by forcing the autoneg
444 ** advertised speed list to only a certain value, this
445 ** supports 1G on 82599 devices, and 100Mb on x540.
446 */
447 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
448 SYSCTL_CHILDREN(adapter->sysctl_tree),
449 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
450 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
451
452 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
453 SYSCTL_CHILDREN(adapter->sysctl_tree),
454 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
455 0, ixgbe_set_thermal_test, "I", "Thermal Test");
456
457 /* Set up the timer callout */
9d1692ad 458 callout_init_mp(&adapter->timer);
9407f759
FT		 459
460 /* Determine hardware revision */
461 ixgbe_identify_hardware(adapter);
462
463 /* Do base PCI setup - map BAR0 */
464 if (ixgbe_allocate_pci_resources(adapter)) {
465 device_printf(dev, "Allocation of PCI resources failed\n");
466 error = ENXIO;
467 goto err_out;
468 }
469
470 /* Do descriptor calc and sanity checks */
471 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
472 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
473 device_printf(dev, "TXD config issue, using default!\n");
474 adapter->num_tx_desc = DEFAULT_TXD;
475 } else
476 adapter->num_tx_desc = ixgbe_txd;
477
478 /*
479 ** With many RX rings it is easy to exceed the
480 ** system mbuf allocation. Tuning nmbclusters
481 ** can alleviate this.
482 */
483 if (nmbclusters > 0 ) {
484 int s;
485 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
486 if (s > nmbclusters) {
487 device_printf(dev, "RX Descriptors exceed "
488 "system mbuf max, using default instead!\n");
489 ixgbe_rxd = DEFAULT_RXD;
490 }
491 }
492
493 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
494 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
495 device_printf(dev, "RXD config issue, using default!\n");
496 adapter->num_rx_desc = DEFAULT_RXD;
497 } else
498 adapter->num_rx_desc = ixgbe_rxd;
499
500 /* Allocate our TX/RX Queues */
501 if (ixgbe_allocate_queues(adapter)) {
502 error = ENOMEM;
503 goto err_out;
504 }
505
506 /* Allocate multicast array memory. */
507 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
508 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
509 if (adapter->mta == NULL) {
510 device_printf(dev, "Can not allocate multicast setup array\n");
511 error = ENOMEM;
512 goto err_late;
513 }
514
515 /* Initialize the shared code */
516 error = ixgbe_init_shared_code(hw);
517 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
518 /*
519 ** No optics in this port, set up
520 ** so the timer routine will probe
521 ** for later insertion.
522 */
523 adapter->sfp_probe = TRUE;
524 error = 0;
525 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
526 device_printf(dev,"Unsupported SFP+ module detected!\n");
527 error = EIO;
528 goto err_late;
529 } else if (error) {
530 device_printf(dev,"Unable to initialize the shared code\n");
531 error = EIO;
532 goto err_late;
533 }
534
535 /* Make sure we have a good EEPROM before we read from it */
536 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
537 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
538 error = EIO;
539 goto err_late;
540 }
541
9407f759 542 error = ixgbe_init_hw(hw);
ce451236
FT		 543 switch (error) {
544 case IXGBE_ERR_EEPROM_VERSION:
9407f759
FT		 545 device_printf(dev, "This device is a pre-production adapter/"
546 "LOM. Please be aware there may be issues associated "
547 "with your hardware.\n If you are experiencing problems "
548 "please contact your Intel or hardware representative "
549 "who provided you with this hardware.\n");
ce451236
FT		 550 break;
551 case IXGBE_ERR_SFP_NOT_SUPPORTED:
9407f759 552 device_printf(dev,"Unsupported SFP+ Module\n");
9407f759
FT		 553 error = EIO;
554 device_printf(dev,"Hardware Initialization Failure\n");
555 goto err_late;
ce451236
FT		 556 case IXGBE_ERR_SFP_NOT_PRESENT:
557 device_printf(dev,"No SFP+ Module found\n");
558 /* falls thru */
559 default:
560 break;
9407f759
FT		 561 }
562
563 /* Detect and set physical type */
564 ixgbe_setup_optics(adapter);
565
566 if ((adapter->msix > 1) && (ixgbe_enable_msix))
567 error = ixgbe_allocate_msix(adapter);
568 else
569 error = ixgbe_allocate_legacy(adapter);
570 if (error)
571 goto err_late;
572
573 /* Setup OS specific network interface */
574 if (ixgbe_setup_interface(dev, adapter) != 0)
575 goto err_late;
576
577 /* Sysctl for limiting the amount of work done in the taskqueue */
578 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
579 "max number of rx packets to process", &adapter->rx_process_limit,
580 ixgbe_rx_process_limit);
581
582 /* Initialize statistics */
583 ixgbe_update_stats_counters(adapter);
584
585 /* Register for VLAN events */
586 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
587 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
588 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
589 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
590
591 /* Print PCIE bus type/speed/width info */
592 ixgbe_get_bus_info(hw);
593 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
594 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
595 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
596 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
597 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
598 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
599 ("Unknown"));
600
601 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
602 (hw->bus.speed == ixgbe_bus_speed_2500)) {
603 device_printf(dev, "PCI-Express bandwidth available"
604 " for this card\n is not sufficient for"
605 " optimal performance.\n");
606 device_printf(dev, "For optimal performance a x8 "
607 "PCIE, or x4 PCIE 2 slot is required.\n");
608 }
609
610 /* let hardware know driver is loaded */
611 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
612 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
613 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
614
615 ixgbe_add_hw_stats(adapter);
616
617#ifdef DEV_NETMAP
618 ixgbe_netmap_attach(adapter);
619#endif /* DEV_NETMAP */
620 INIT_DEBUGOUT("ixgbe_attach: end");
621 return (0);
622err_late:
623 ixgbe_free_transmit_structures(adapter);
624 ixgbe_free_receive_structures(adapter);
625err_out:
626 if (adapter->ifp != NULL)
627 if_free(adapter->ifp);
628 ixgbe_free_pci_resources(adapter);
629 kfree(adapter->mta, M_DEVBUF);
630 return (error);
631
632}
633
634/*********************************************************************
635 * Device removal routine
636 *
637 * The detach entry point is called when the driver is being removed.
638 * This routine stops the adapter and deallocates all the resources
639 * that were allocated for driver operation.
640 *
641 * return 0 on success, positive on failure
642 *********************************************************************/
643
644static int
645ixgbe_detach(device_t dev)
646{
647 struct adapter *adapter = device_get_softc(dev);
648 struct ix_queue *que = adapter->queues;
649 u32 ctrl_ext;
650
651 INIT_DEBUGOUT("ixgbe_detach: begin");
652
653#ifdef NET_VLAN
654 /* Make sure VLANS are not using driver */
655 if (adapter->ifp->if_vlantrunk != NULL) {
656 device_printf(dev,"Vlan in use, detach first\n");
657 return (EBUSY);
658 }
659#endif
660
661 IXGBE_CORE_LOCK(adapter);
662 ixgbe_stop(adapter);
663 IXGBE_CORE_UNLOCK(adapter);
664
665 for (int i = 0; i < adapter->num_queues; i++, que++) {
666 if (que->tq) {
667 taskqueue_drain(que->tq, &que->que_task);
668 taskqueue_free(que->tq);
669 }
670 }
671
672 /* Drain the Link queue */
673 if (adapter->tq) {
674 taskqueue_drain(adapter->tq, &adapter->link_task);
675 taskqueue_drain(adapter->tq, &adapter->mod_task);
676 taskqueue_drain(adapter->tq, &adapter->msf_task);
677#ifdef IXGBE_FDIR
678 taskqueue_drain(adapter->tq, &adapter->fdir_task);
679#endif
680 taskqueue_free(adapter->tq);
681 }
682
683 /* let hardware know driver is unloading */
684 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
685 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
686 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
687
688 /* Unregister VLAN events */
689#ifdef NET_VLAN
690 if (adapter->vlan_attach != NULL)
691 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
692 if (adapter->vlan_detach != NULL)
693 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
694#endif
695
696 ether_ifdetach(adapter->ifp);
697 callout_stop(&adapter->timer);
9407f759
FT		 698#ifdef DEV_NETMAP
699 netmap_detach(adapter->ifp);
700#endif /* DEV_NETMAP */
701 ixgbe_free_pci_resources(adapter);
702 bus_generic_detach(dev);
703 if_free(adapter->ifp);
704
705 ixgbe_free_transmit_structures(adapter);
706 ixgbe_free_receive_structures(adapter);
707 kfree(adapter->mta, M_DEVBUF);
708 sysctl_ctx_free(&adapter->sysctl_ctx);
709
9407f759
FT		 710 IXGBE_CORE_LOCK_DESTROY(adapter);
711 return (0);
712}
713
714/*********************************************************************
715 *
716 * Shutdown entry point
717 *
718 **********************************************************************/
719
720static int
721ixgbe_shutdown(device_t dev)
722{
723 struct adapter *adapter = device_get_softc(dev);
724 IXGBE_CORE_LOCK(adapter);
725 ixgbe_stop(adapter);
726 IXGBE_CORE_UNLOCK(adapter);
727 return (0);
728}
729
730
731/*********************************************************************
732 * Transmit entry point
733 *
734 * ixgbe_start is called by the stack to initiate a transmit.
735 * The driver will remain in this routine as long as there are
736 * packets to transmit and transmit resources are available.
737 * In case resources are not available stack is notified and
738 * the packet is requeued.
739 **********************************************************************/
740
741static void
742ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
743{
744 struct mbuf *m_head;
745 struct adapter *adapter = txr->adapter;
746
747 IXGBE_TX_LOCK_ASSERT(txr);
748
749 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
750 return;
751 if (!adapter->link_active)
752 return;
753
754 while (!ifq_is_empty(&ifp->if_snd)) {
755 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
756 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
757 break;
758 }
759
760 m_head = ifq_dequeue(&ifp->if_snd, NULL);
761 if (m_head == NULL)
762 break;
763
764 if (ixgbe_xmit(txr, &m_head)) {
765#if 0 /* XXX: prepend to an ALTQ queue ? */
766 if (m_head != NULL)
767 IF_PREPEND(&ifp->if_snd, m_head);
768#endif
769 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
770 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
771 break;
772 }
773 /* Send a copy of the frame to the BPF listener */
774 ETHER_BPF_MTAP(ifp, m_head);
775
776 /* Set watchdog on */
777 txr->watchdog_time = ticks;
778 txr->queue_status = IXGBE_QUEUE_WORKING;
779
780 }
781 return;
782}
783
784/*
785 * Legacy TX start - called by the stack, this
786 * always uses the first tx ring, and should
787 * not be used with multiqueue tx enabled.
788 */
789static void
790ixgbe_start(struct ifnet *ifp)
791{
792 struct adapter *adapter = ifp->if_softc;
793 struct tx_ring *txr = adapter->tx_rings;
794
795 if (ifp->if_flags & IFF_RUNNING) {
796 IXGBE_TX_LOCK(txr);
797 ixgbe_start_locked(txr, ifp);
798 IXGBE_TX_UNLOCK(txr);
799 }
800 return;
801}
802
803#if 0 /* __FreeBSD_version >= 800000 */
804/*
805** Multiqueue Transmit driver
806**
807*/
808static int
809ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
810{
811 struct adapter *adapter = ifp->if_softc;
812 struct ix_queue *que;
813 struct tx_ring *txr;
814 int i = 0, err = 0;
815
816 /* Which queue to use */
817 if ((m->m_flags & M_FLOWID) != 0)
818 i = m->m_pkthdr.flowid % adapter->num_queues;
819 else
820 i = curcpu % adapter->num_queues;
821
822 txr = &adapter->tx_rings[i];
823 que = &adapter->queues[i];
824
825 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
826 IXGBE_TX_TRYLOCK(txr)) {
827 err = ixgbe_mq_start_locked(ifp, txr, m);
828 IXGBE_TX_UNLOCK(txr);
829 } else {
830 err = drbr_enqueue(ifp, txr->br, m);
831 taskqueue_enqueue(que->tq, &que->que_task);
832 }
833
834 return (err);
835}
836
837static int
838ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
839{
840 struct adapter *adapter = txr->adapter;
841 struct mbuf *next;
842 int enqueued, err = 0;
843
844 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
845 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
846 adapter->link_active == 0) {
847 if (m != NULL)
848 err = drbr_enqueue(ifp, txr->br, m);
849 return (err);
850 }
851
852 enqueued = 0;
853 if (m == NULL) {
854 next = drbr_dequeue(ifp, txr->br);
855 } else if (drbr_needs_enqueue(ifp, txr->br)) {
856 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
857 return (err);
858 next = drbr_dequeue(ifp, txr->br);
859 } else
860 next = m;
861
862 /* Process the queue */
863 while (next != NULL) {
864 if ((err = ixgbe_xmit(txr, &next)) != 0) {
865 if (next != NULL)
866 err = drbr_enqueue(ifp, txr->br, next);
867 break;
868 }
869 enqueued++;
870 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
871 /* Send a copy of the frame to the BPF listener */
872 ETHER_BPF_MTAP(ifp, next);
873 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
874 break;
875 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
876 ixgbe_txeof(txr);
877 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
878 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
879 break;
880 }
881 next = drbr_dequeue(ifp, txr->br);
882 }
883
884 if (enqueued > 0) {
885 /* Set watchdog on */
886 txr->queue_status |= IXGBE_QUEUE_WORKING;
887 txr->watchdog_time = ticks;
888 }
889
890 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
891 ixgbe_txeof(txr);
892
893 return (err);
894}
895
896/*
897** Flush all ring buffers
898*/
899static void
900ixgbe_qflush(struct ifnet *ifp)
901{
902 struct adapter *adapter = ifp->if_softc;
903 struct tx_ring *txr = adapter->tx_rings;
904 struct mbuf *m;
905
906 for (int i = 0; i < adapter->num_queues; i++, txr++) {
907 IXGBE_TX_LOCK(txr);
908 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
909 m_freem(m);
910 IXGBE_TX_UNLOCK(txr);
911 }
912 if_qflush(ifp);
913}
914#endif /* __FreeBSD_version >= 800000 */
915
916/*********************************************************************
917 * Ioctl entry point
918 *
919 * ixgbe_ioctl is called when the user wants to configure the
920 * interface.
921 *
922 * return 0 on success, positive on failure
923 **********************************************************************/
924
925static int
926ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
927{
928 struct adapter *adapter = ifp->if_softc;
929 struct ifreq *ifr = (struct ifreq *) data;
930#if defined(INET) || defined(INET6)
931 struct ifaddr *ifa = (struct ifaddr *)data;
932 bool avoid_reset = FALSE;
933#endif
934 int error = 0;
935
936 switch (command) {
937
938 case SIOCSIFADDR:
939#ifdef INET
940 if (ifa->ifa_addr->sa_family == AF_INET)
941 avoid_reset = TRUE;
942#endif
943#ifdef INET6
944 if (ifa->ifa_addr->sa_family == AF_INET6)
945 avoid_reset = TRUE;
946#endif
947#if defined(INET) || defined(INET6)
948 /*
949 ** Calling init results in link renegotiation,
950 ** so we avoid doing it when possible.
951 */
952 if (avoid_reset) {
953 ifp->if_flags |= IFF_UP;
954 if (!(ifp->if_flags & IFF_RUNNING))
955 ixgbe_init(adapter);
956 if (!(ifp->if_flags & IFF_NOARP))
957 arp_ifinit(ifp, ifa);
958 } else
959 error = ether_ioctl(ifp, command, data);
960#endif
961 break;
962 case SIOCSIFMTU:
963 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
964 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
965 error = EINVAL;
966 } else {
967 IXGBE_CORE_LOCK(adapter);
968 ifp->if_mtu = ifr->ifr_mtu;
969 adapter->max_frame_size =
970 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
971 ixgbe_init_locked(adapter);
972 IXGBE_CORE_UNLOCK(adapter);
973 }
974 break;
975 case SIOCSIFFLAGS:
976 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
977 IXGBE_CORE_LOCK(adapter);
978 if (ifp->if_flags & IFF_UP) {
979 if ((ifp->if_flags & IFF_RUNNING)) {
980 if ((ifp->if_flags ^ adapter->if_flags) &
981 (IFF_PROMISC | IFF_ALLMULTI)) {
982 ixgbe_set_promisc(adapter);
983 }
984 } else
985 ixgbe_init_locked(adapter);
986 } else
987 if (ifp->if_flags & IFF_RUNNING)
988 ixgbe_stop(adapter);
989 adapter->if_flags = ifp->if_flags;
990 IXGBE_CORE_UNLOCK(adapter);
991 break;
992 case SIOCADDMULTI:
993 case SIOCDELMULTI:
994 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
995 if (ifp->if_flags & IFF_RUNNING) {
996 IXGBE_CORE_LOCK(adapter);
997 ixgbe_disable_intr(adapter);
998 ixgbe_set_multi(adapter);
999 ixgbe_enable_intr(adapter);
1000 IXGBE_CORE_UNLOCK(adapter);
1001 }
1002 break;
1003 case SIOCSIFMEDIA:
1004 case SIOCGIFMEDIA:
1005 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1006 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1007 break;
1008 case SIOCSIFCAP:
1009 {
1010 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1011 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1012 if (mask & IFCAP_HWCSUM)
1013 ifp->if_capenable ^= IFCAP_HWCSUM;
1014#if 0 /* NET_TSO */
1015 if (mask & IFCAP_TSO4)
1016 ifp->if_capenable ^= IFCAP_TSO4;
1017 if (mask & IFCAP_TSO6)
1018 ifp->if_capenable ^= IFCAP_TSO6;
1019#endif
1020#if 0 /* NET_LRO */
1021 if (mask & IFCAP_LRO)
1022 ifp->if_capenable ^= IFCAP_LRO;
1023#endif
1024 if (mask & IFCAP_VLAN_HWTAGGING)
1025 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1026 if (mask & IFCAP_VLAN_HWFILTER)
1027 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1028#if 0 /* NET_TSO */
1029 if (mask & IFCAP_VLAN_HWTSO)
1030 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1031#endif
1032 if (ifp->if_flags & IFF_RUNNING) {
1033 IXGBE_CORE_LOCK(adapter);
1034 ixgbe_init_locked(adapter);
1035 IXGBE_CORE_UNLOCK(adapter);
1036 }
1037#if 0
1038 VLAN_CAPABILITIES(ifp);
1039#endif
1040 break;
1041 }
1042
1043 default:
1044 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1045 error = ether_ioctl(ifp, command, data);
1046 break;
1047 }
1048
1049 return (error);
1050}
1051
1052/*********************************************************************
1053 * Init entry point
1054 *
1055 * This routine is used in two ways. It is used by the stack as
1056 * init entry point in network interface structure. It is also used
1057 * by the driver as a hw/sw initialization routine to get to a
1058 * consistent state.
1059 *
1060 * return 0 on success, positive on failure
1061 **********************************************************************/
1062#define IXGBE_MHADD_MFS_SHIFT 16
1063
1064static void
1065ixgbe_init_locked(struct adapter *adapter)
1066{
1067 struct ifnet *ifp = adapter->ifp;
1068 device_t dev = adapter->dev;
1069 struct ixgbe_hw *hw = &adapter->hw;
1070 u32 k, txdctl, mhadd, gpie;
1071 u32 rxdctl, rxctrl;
1072
1073 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1074 INIT_DEBUGOUT("ixgbe_init: begin");
1075 hw->adapter_stopped = FALSE;
1076 ixgbe_stop_adapter(hw);
1077 callout_stop(&adapter->timer);
1078
1079 /* reprogram the RAR[0] in case user changed it. */
1080 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1081
1082 /* Get the latest mac address, User can use a LAA */
1083 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1084 IXGBE_ETH_LENGTH_OF_ADDRESS);
1085 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1086 hw->addr_ctrl.rar_used_count = 1;
1087
1088 /* Set the various hardware offload abilities */
1089 ifp->if_hwassist = 0;
1090#if 0 /* NET_TSO */
1091 if (ifp->if_capenable & IFCAP_TSO)
1092 ifp->if_hwassist |= CSUM_TSO;
1093#endif
1094 if (ifp->if_capenable & IFCAP_TXCSUM) {
1095 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1096#if 0
1097 if (hw->mac.type != ixgbe_mac_82598EB)
1098 ifp->if_hwassist |= CSUM_SCTP;
1099#endif
1100 }
1101
1102 /* Prepare transmit descriptors and buffers */
1103 if (ixgbe_setup_transmit_structures(adapter)) {
1104 device_printf(dev,"Could not setup transmit structures\n");
1105 ixgbe_stop(adapter);
1106 return;
1107 }
1108
1109 ixgbe_init_hw(hw);
1110 ixgbe_initialize_transmit_units(adapter);
1111
1112 /* Setup Multicast table */
1113 ixgbe_set_multi(adapter);
1114
1115 /*
1116 ** Determine the correct mbuf pool
1117 ** for doing jumbo/headersplit
1118 */
1119 if (adapter->max_frame_size <= 2048)
1120 adapter->rx_mbuf_sz = MCLBYTES;
1121 else if (adapter->max_frame_size <= 4096)
1122 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1123 else if (adapter->max_frame_size <= 9216)
1124 adapter->rx_mbuf_sz = MJUM9BYTES;
1125 else
1126 adapter->rx_mbuf_sz = MJUM16BYTES;
1127
1128 /* Prepare receive descriptors and buffers */
1129 if (ixgbe_setup_receive_structures(adapter)) {
1130 device_printf(dev,"Could not setup receive structures\n");
1131 ixgbe_stop(adapter);
1132 return;
1133 }
1134
1135 /* Configure RX settings */
1136 ixgbe_initialize_receive_units(adapter);
1137
1138 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1139
1140 /* Enable Fan Failure Interrupt */
1141 gpie |= IXGBE_SDP1_GPIEN;
1142
1143 /* Add for Module detection */
1144 if (hw->mac.type == ixgbe_mac_82599EB)
1145 gpie |= IXGBE_SDP2_GPIEN;
1146
1147 /* Thermal Failure Detection */
1148 if (hw->mac.type == ixgbe_mac_X540)
1149 gpie |= IXGBE_SDP0_GPIEN;
1150
1151 if (adapter->msix > 1) {
1152 /* Enable Enhanced MSIX mode */
1153 gpie |= IXGBE_GPIE_MSIX_MODE;
1154 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1155 IXGBE_GPIE_OCD;
1156 }
1157 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1158
1159 /* Set MTU size */
1160 if (ifp->if_mtu > ETHERMTU) {
1161 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1162 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1163 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1164 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1165 }
1166
1167 /* Now enable all the queues */
1168
1169 for (int i = 0; i < adapter->num_queues; i++) {
1170 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1171 txdctl |= IXGBE_TXDCTL_ENABLE;
1172 /* Set WTHRESH to 8, burst writeback */
1173 txdctl |= (8 << 16);
1174 /*
1175 * When the internal queue falls below PTHRESH (32),
1176 * start prefetching as long as there are at least
1177 * HTHRESH (1) buffers ready. The values are taken
1178 * from the Intel linux driver 3.8.21.
1179 * Prefetching enables tx line rate even with 1 queue.
1180 */
1181 txdctl |= (32 << 0) | (1 << 8);
1182 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1183 }
1184
1185 for (int i = 0; i < adapter->num_queues; i++) {
1186 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1187 if (hw->mac.type == ixgbe_mac_82598EB) {
1188 /*
1189 ** PTHRESH = 21
1190 ** HTHRESH = 4
1191 ** WTHRESH = 8
1192 */
1193 rxdctl &= ~0x3FFFFF;
1194 rxdctl |= 0x080420;
1195 }
1196 rxdctl |= IXGBE_RXDCTL_ENABLE;
1197 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1198 for (k = 0; k < 10; k++) {
1199 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1200 IXGBE_RXDCTL_ENABLE)
1201 break;
1202 else
1203 msec_delay(1);
1204 }
1205 wmb();
1206#ifdef DEV_NETMAP
1207 /*
1208 * In netmap mode, we must preserve the buffers made
1209 * available to userspace before the if_init()
1210 * (this is true by default on the TX side, because
1211 * init makes all buffers available to userspace).
1212 *
1213 * netmap_reset() and the device specific routines
1214 * (e.g. ixgbe_setup_receive_rings()) map these
1215 * buffers at the end of the NIC ring, so here we
1216 * must set the RDT (tail) register to make sure
1217 * they are not overwritten.
1218 *
1219 * In this driver the NIC ring starts at RDH = 0,
1220 * RDT points to the last slot available for reception (?),
1221 * so RDT = num_rx_desc - 1 means the whole ring is available.
1222 */
1223 if (ifp->if_capenable & IFCAP_NETMAP) {
1224 struct netmap_adapter *na = NA(adapter->ifp);
1225 struct netmap_kring *kring = &na->rx_rings[i];
1226 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1227
1228 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1229 } else
1230#endif /* DEV_NETMAP */
1231 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1232 }
1233
1234 /* Set up VLAN support and filter */
1235 ixgbe_setup_vlan_hw_support(adapter);
1236
1237 /* Enable Receive engine */
1238 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1239 if (hw->mac.type == ixgbe_mac_82598EB)
1240 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1241 rxctrl |= IXGBE_RXCTRL_RXEN;
1242 ixgbe_enable_rx_dma(hw, rxctrl);
1243
1244 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1245
1246 /* Set up MSI/X routing */
1247 if (ixgbe_enable_msix) {
1248 ixgbe_configure_ivars(adapter);
1249 /* Set up auto-mask */
1250 if (hw->mac.type == ixgbe_mac_82598EB)
1251 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1252 else {
1253 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1254 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1255 }
1256 } else { /* Simple settings for Legacy/MSI */
1257 ixgbe_set_ivar(adapter, 0, 0, 0);
1258 ixgbe_set_ivar(adapter, 0, 0, 1);
1259 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1260 }
1261
1262#ifdef IXGBE_FDIR
1263 /* Init Flow director */
1264 if (hw->mac.type != ixgbe_mac_82598EB) {
ce451236 1265 u32 hdrm = 32 << fdir_pballoc;
9407f759
FT		 1266
1267 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1268 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1269 }
1270#endif
1271
1272 /*
1273 ** Check on any SFP devices that
1274 ** need to be kick-started
1275 */
1276 if (hw->phy.type == ixgbe_phy_none) {
1277 int err = hw->phy.ops.identify(hw);
1278 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1279 device_printf(dev,
1280 "Unsupported SFP+ module type was detected.\n");
1281 return;
1282 }
1283 }
1284
1285 /* Set moderation on the Link interrupt */
1286 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1287
1288 /* Config/Enable Link */
1289 ixgbe_config_link(adapter);
1290
ce451236
FT		 1291 /* Hardware Packet Buffer & Flow Control setup */
1292 {
1293 u32 rxpb, frame, size, tmp;
1294
1295 frame = adapter->max_frame_size;
1296
1297 /* Calculate High Water */
1298 if (hw->mac.type == ixgbe_mac_X540)
1299 tmp = IXGBE_DV_X540(frame, frame);
1300 else
1301 tmp = IXGBE_DV(frame, frame);
1302 size = IXGBE_BT2KB(tmp);
1303 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1304 hw->fc.high_water[0] = rxpb - size;
1305
1306 /* Now calculate Low Water */
1307 if (hw->mac.type == ixgbe_mac_X540)
1308 tmp = IXGBE_LOW_DV_X540(frame);
1309 else
1310 tmp = IXGBE_LOW_DV(frame);
1311 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1312
1313 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1314 hw->fc.pause_time = IXGBE_FC_PAUSE;
1315 hw->fc.send_xon = TRUE;
1316 }
1317 /* Initialize the FC settings */
1318 ixgbe_start_hw(hw);
1319
9407f759
FT		 1320 /* And now turn on interrupts */
1321 ixgbe_enable_intr(adapter);
1322
1323 /* Now inform the stack we're ready */
1324 ifp->if_flags |= IFF_RUNNING;
1325 ifp->if_flags &= ~IFF_OACTIVE;
1326
1327 return;
1328}
1329
1330static void
1331ixgbe_init(void *arg)
1332{
1333 struct adapter *adapter = arg;
1334
1335 IXGBE_CORE_LOCK(adapter);
1336 ixgbe_init_locked(adapter);
1337 IXGBE_CORE_UNLOCK(adapter);
1338 return;
1339}
1340
1341
1342/*
1343**
1344** MSIX Interrupt Handlers and Tasklets
1345**
1346*/
1347
1348static inline void
1349ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1350{
1351 struct ixgbe_hw *hw = &adapter->hw;
1352 u64 queue = (u64)(1 << vector);
1353 u32 mask;
1354
1355 if (hw->mac.type == ixgbe_mac_82598EB) {
1356 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1357 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1358 } else {
1359 mask = (queue & 0xFFFFFFFF);
1360 if (mask)
1361 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1362 mask = (queue >> 32);
1363 if (mask)
1364 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1365 }
1366}
1367
1368static inline void
1369ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1370{
1371 struct ixgbe_hw *hw = &adapter->hw;
1372 u64 queue = (u64)(1 << vector);
1373 u32 mask;
1374
1375 if (hw->mac.type == ixgbe_mac_82598EB) {
1376 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1377 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1378 } else {
1379 mask = (queue & 0xFFFFFFFF);
1380 if (mask)
1381 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1382 mask = (queue >> 32);
1383 if (mask)
1384 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1385 }
1386}
1387
1388static inline void
1389ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1390{
1391 u32 mask;
1392
1393 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1394 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1395 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1396 } else {
1397 mask = (queues & 0xFFFFFFFF);
1398 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1399 mask = (queues >> 32);
1400 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1401 }
1402}
1403
1404
1405static void
1406ixgbe_handle_que(void *context, int pending)
1407{
1408 struct ix_queue *que = context;
1409 struct adapter *adapter = que->adapter;
1410 struct tx_ring *txr = que->txr;
1411 struct ifnet *ifp = adapter->ifp;
1412 bool more;
1413
1414 if (ifp->if_flags & IFF_RUNNING) {
1415 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1416 IXGBE_TX_LOCK(txr);
1417 ixgbe_txeof(txr);
1418#if 0 /*__FreeBSD_version >= 800000*/
1419 if (!drbr_empty(ifp, txr->br))
1420 ixgbe_mq_start_locked(ifp, txr, NULL);
1421#else
1422 if (!ifq_is_empty(&ifp->if_snd))
1423 ixgbe_start_locked(txr, ifp);
1424#endif
1425 IXGBE_TX_UNLOCK(txr);
1426 if (more) {
1427 taskqueue_enqueue(que->tq, &que->que_task);
1428 return;
1429 }
1430 }
1431
1432 /* Reenable this interrupt */
1433 ixgbe_enable_queue(adapter, que->msix);
1434 return;
1435}
1436
1437
1438/*********************************************************************
1439 *
1440 * Legacy Interrupt Service routine
1441 *
1442 **********************************************************************/
1443
1444static void
1445ixgbe_legacy_irq(void *arg)
1446{
1447 struct ix_queue *que = arg;
1448 struct adapter *adapter = que->adapter;
1449 struct ixgbe_hw *hw = &adapter->hw;
1450 struct tx_ring *txr = adapter->tx_rings;
1451 bool more_tx, more_rx;
1452 u32 reg_eicr, loop = MAX_LOOP;
1453
1454
1455 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1456
1457 ++que->irqs;
1458 if (reg_eicr == 0) {
1459 ixgbe_enable_intr(adapter);
1460 return;
1461 }
1462
1463 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1464
1465 IXGBE_TX_LOCK(txr);
1466 do {
1467 more_tx = ixgbe_txeof(txr);
1468 } while (loop-- && more_tx);
1469 IXGBE_TX_UNLOCK(txr);
1470
1471 if (more_rx || more_tx)
1472 taskqueue_enqueue(que->tq, &que->que_task);
1473
1474 /* Check for fan failure */
1475 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1476 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1477 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1478 "REPLACE IMMEDIATELY!!\n");
1479 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1480 }
1481
1482 /* Link status change */
1483 if (reg_eicr & IXGBE_EICR_LSC)
1484 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1485
1486 ixgbe_enable_intr(adapter);
1487 return;
1488}
1489
1490
1491/*********************************************************************
1492 *
1493 * MSIX Queue Interrupt Service routine
1494 *
1495 **********************************************************************/
1496void
1497ixgbe_msix_que(void *arg)
1498{
1499 struct ix_queue *que = arg;
1500 struct adapter *adapter = que->adapter;
1501 struct tx_ring *txr = que->txr;
1502 struct rx_ring *rxr = que->rxr;
1503 bool more_tx, more_rx;
1504 u32 newitr = 0;
1505
1506 ixgbe_disable_queue(adapter, que->msix);
1507 ++que->irqs;
1508
1509 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1510
1511 IXGBE_TX_LOCK(txr);
1512 more_tx = ixgbe_txeof(txr);
1513 /*
1514 ** Make certain that if the stack
1515 ** has anything queued the task gets
1516 ** scheduled to handle it.
1517 */
1518#if 0
1519#if __FreeBSD_version < 800000
1520 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1521#else
1522 if (!drbr_empty(adapter->ifp, txr->br))
1523#endif
1524#endif
1525 if (!ifq_is_empty(&adapter->ifp->if_snd))
1526 more_tx = 1;
1527 IXGBE_TX_UNLOCK(txr);
1528
1529 /* Do AIM now? */
1530
1531 if (ixgbe_enable_aim == FALSE)
1532 goto no_calc;
1533 /*
1534 ** Do Adaptive Interrupt Moderation:
1535 ** - Write out last calculated setting
1536 ** - Calculate based on average size over
1537 ** the last interval.
1538 */
1539 if (que->eitr_setting)
1540 IXGBE_WRITE_REG(&adapter->hw,
1541 IXGBE_EITR(que->msix), que->eitr_setting);
1542
1543 que->eitr_setting = 0;
1544
1545 /* Idle, do nothing */
1546 if ((txr->bytes == 0) && (rxr->bytes == 0))
1547 goto no_calc;
1548
1549 if ((txr->bytes) && (txr->packets))
1550 newitr = txr->bytes/txr->packets;
1551 if ((rxr->bytes) && (rxr->packets))
1552 newitr = max(newitr,
1553 (rxr->bytes / rxr->packets));
1554 newitr += 24; /* account for hardware frame, crc */
1555
1556 /* set an upper boundary */
1557 newitr = min(newitr, 3000);
1558
1559 /* Be nice to the mid range */
1560 if ((newitr > 300) && (newitr < 1200))
1561 newitr = (newitr / 3);
1562 else
1563 newitr = (newitr / 2);
1564
1565 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1566 newitr |= newitr << 16;
1567 else
1568 newitr |= IXGBE_EITR_CNT_WDIS;
1569
1570 /* save for next interrupt */
1571 que->eitr_setting = newitr;
1572
1573 /* Reset state */
1574 txr->bytes = 0;
1575 txr->packets = 0;
1576 rxr->bytes = 0;
1577 rxr->packets = 0;
1578
1579no_calc:
1580 if (more_tx || more_rx)
1581 taskqueue_enqueue(que->tq, &que->que_task);
1582 else /* Reenable this interrupt */
1583 ixgbe_enable_queue(adapter, que->msix);
1584 return;
1585}
1586
1587
1588static void
1589ixgbe_msix_link(void *arg)
1590{
1591 struct adapter *adapter = arg;
1592 struct ixgbe_hw *hw = &adapter->hw;
1593 u32 reg_eicr;
1594
1595 ++adapter->link_irq;
1596
1597 /* First get the cause */
1598 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1599 /* Clear interrupt with write */
1600 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1601
1602 /* Link status change */
1603 if (reg_eicr & IXGBE_EICR_LSC)
1604 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1605
1606 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1607#ifdef IXGBE_FDIR
1608 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1609 /* This is probably overkill :) */
1610 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1611 return;
ce451236
FT		 1612 /* Disable the interrupt */
1613 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
9407f759
FT		 1614 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1615 } else
1616#endif
1617 if (reg_eicr & IXGBE_EICR_ECC) {
1618 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1619 "Please Reboot!!\n");
1620 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1621 } else
1622
1623 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1624 /* Clear the interrupt */
1625 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1626 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1627 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1628 /* Clear the interrupt */
1629 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1630 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1631 }
1632 }
1633
1634 /* Check for fan failure */
1635 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1636 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1637 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1638 "REPLACE IMMEDIATELY!!\n");
1639 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1640 }
1641
1642 /* Check for over temp condition */
1643 if ((hw->mac.type == ixgbe_mac_X540) &&
1644 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1645 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1646 "PHY IS SHUT DOWN!!\n");
1647 device_printf(adapter->dev, "System shutdown required\n");
1648 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1649 }
1650
1651 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1652 return;
1653}
1654
1655/*********************************************************************
1656 *
1657 * Media Ioctl callback
1658 *
1659 * This routine is called whenever the user queries the status of
1660 * the interface using ifconfig.
1661 *
1662 **********************************************************************/
1663static void
1664ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1665{
1666 struct adapter *adapter = ifp->if_softc;
1667
1668 INIT_DEBUGOUT("ixgbe_media_status: begin");
1669 IXGBE_CORE_LOCK(adapter);
1670 ixgbe_update_link_status(adapter);
1671
1672 ifmr->ifm_status = IFM_AVALID;
1673 ifmr->ifm_active = IFM_ETHER;
1674
1675 if (!adapter->link_active) {
1676 IXGBE_CORE_UNLOCK(adapter);
1677 return;
1678 }
1679
1680 ifmr->ifm_status |= IFM_ACTIVE;
1681
1682 switch (adapter->link_speed) {
1683 case IXGBE_LINK_SPEED_100_FULL:
1684 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1685 break;
1686 case IXGBE_LINK_SPEED_1GB_FULL:
1687 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1688 break;
1689 case IXGBE_LINK_SPEED_10GB_FULL:
1690 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1691 break;
1692 }
1693
1694 IXGBE_CORE_UNLOCK(adapter);
1695
1696 return;
1697}
1698
1699/*********************************************************************
1700 *
1701 * Media Ioctl callback
1702 *
1703 * This routine is called when the user changes speed/duplex using
1704 * media/mediopt option with ifconfig.
1705 *
1706 **********************************************************************/
1707static int
1708ixgbe_media_change(struct ifnet * ifp)
1709{
1710 struct adapter *adapter = ifp->if_softc;
1711 struct ifmedia *ifm = &adapter->media;
1712
1713 INIT_DEBUGOUT("ixgbe_media_change: begin");
1714
1715 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1716 return (EINVAL);
1717
1718 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1719 case IFM_AUTO:
1720 adapter->hw.phy.autoneg_advertised =
1721 IXGBE_LINK_SPEED_100_FULL |
1722 IXGBE_LINK_SPEED_1GB_FULL |
1723 IXGBE_LINK_SPEED_10GB_FULL;
1724 break;
1725 default:
1726 device_printf(adapter->dev, "Only auto media type\n");
1727 return (EINVAL);
1728 }
1729
1730 return (0);
1731}
1732
1733/*********************************************************************
1734 *
1735 * This routine maps the mbufs to tx descriptors, allowing the
1736 * TX engine to transmit the packets.
1737 * - return 0 on success, positive on failure
1738 *
1739 **********************************************************************/
1740
1741static int
1742ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1743{
1744 struct adapter *adapter = txr->adapter;
1745 u32 olinfo_status = 0, cmd_type_len;
1746 u32 paylen = 0;
a1a9516b 1747 int i, j, error, nsegs, maxsegs;
9407f759
FT		 1748 int first, last = 0;
1749 struct mbuf *m_head;
a1a9516b 1750 bus_dma_segment_t segs[adapter->num_segs];
9407f759
FT		 1751 bus_dmamap_t map;
1752 struct ixgbe_tx_buf *txbuf;
1753 union ixgbe_adv_tx_desc *txd = NULL;
1754
1755 m_head = *m_headp;
1756
1757 /* Basic descriptor defines */
1758 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1759 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1760
1761 if (m_head->m_flags & M_VLANTAG)
1762 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1763
1764 /*
1765 * Important to capture the first descriptor
1766 * used because it will contain the index of
1767 * the one we tell the hardware to report back
1768 */
1769 first = txr->next_avail_desc;
1770 txbuf = &txr->tx_buffers[first];
1771 map = txbuf->map;
1772
1773 /*
1774 * Map the packet for DMA.
1775 */
a1a9516b
FT		 1776 maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1777 if (maxsegs > adapter->num_segs)
1778 maxsegs = adapter->num_segs;
1779
1780 error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1781 segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1782 if (error) {
1783 if (error == ENOBUFS)
9407f759 1784 adapter->mbuf_defrag_failed++;
a1a9516b 1785 else
9407f759 1786 adapter->no_tx_dma_setup++;
a1a9516b 1787
9407f759
FT		 1788 m_freem(*m_headp);
1789 *m_headp = NULL;
1790 return (error);
1791 }
1792
1793 /* Make certain there are enough descriptors */
1794 if (nsegs > txr->tx_avail - 2) {
1795 txr->no_desc_avail++;
1796 error = ENOBUFS;
1797 goto xmit_fail;
1798 }
1799 m_head = *m_headp;
1800
1801 /*
1802 ** Set up the appropriate offload context
1803 ** this becomes the first descriptor of
1804 ** a packet.
1805 */
1806#if 0 /* NET_TSO */
1807 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1808 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1809 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1810 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1811 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1812 ++adapter->tso_tx;
1813 } else
1814 return (ENXIO);
1815 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1816#endif
1817 if (ixgbe_tx_ctx_setup(txr, m_head))
1818 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1819
1820#ifdef IXGBE_IEEE1588
1821 /* This is changing soon to an mtag detection */
1822 if (we detect this mbuf has a TSTAMP mtag)
1823 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1824#endif
1825
1826#ifdef IXGBE_FDIR
1827 /* Do the flow director magic */
1828 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1829 ++txr->atr_count;
1830 if (txr->atr_count >= atr_sample_rate) {
1831 ixgbe_atr(txr, m_head);
1832 txr->atr_count = 0;
1833 }
1834 }
1835#endif
1836 /* Record payload length */
1837 if (paylen == 0)
1838 olinfo_status |= m_head->m_pkthdr.len <<
1839 IXGBE_ADVTXD_PAYLEN_SHIFT;
1840
1841 i = txr->next_avail_desc;
1842 for (j = 0; j < nsegs; j++) {
1843 bus_size_t seglen;
1844 bus_addr_t segaddr;
1845
1846 txbuf = &txr->tx_buffers[i];
1847 txd = &txr->tx_base[i];
1848 seglen = segs[j].ds_len;
1849 segaddr = htole64(segs[j].ds_addr);
1850
1851 txd->read.buffer_addr = segaddr;
1852 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1853 cmd_type_len |seglen);
1854 txd->read.olinfo_status = htole32(olinfo_status);
1855 last = i; /* descriptor that will get completion IRQ */
1856
1857 if (++i == adapter->num_tx_desc)
1858 i = 0;
1859
1860 txbuf->m_head = NULL;
1861 txbuf->eop_index = -1;
1862 }
1863
1864 txd->read.cmd_type_len |=
1865 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1866 txr->tx_avail -= nsegs;
1867 txr->next_avail_desc = i;
1868
1869 txbuf->m_head = m_head;
1870 /* Swap the dma map between the first and last descriptor */
1871 txr->tx_buffers[first].map = txbuf->map;
1872 txbuf->map = map;
1873 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1874
1875 /* Set the index of the descriptor that will be marked done */
1876 txbuf = &txr->tx_buffers[first];
1877 txbuf->eop_index = last;
1878
1879 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1880 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1881 /*
1882 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1883 * hardware that this frame is available to transmit.
1884 */
1885 ++txr->total_packets;
1886 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1887
1888 return (0);
1889
1890xmit_fail:
1891 bus_dmamap_unload(txr->txtag, txbuf->map);
1892 return (error);
1893
1894}
1895
1896static void
1897ixgbe_set_promisc(struct adapter *adapter)
1898{
1899 u_int32_t reg_rctl;
1900 struct ifnet *ifp = adapter->ifp;
1901
1902 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1903 reg_rctl &= (~IXGBE_FCTRL_UPE);
1904 reg_rctl &= (~IXGBE_FCTRL_MPE);
1905 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1906
1907 if (ifp->if_flags & IFF_PROMISC) {
1908 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1909 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1910 } else if (ifp->if_flags & IFF_ALLMULTI) {
1911 reg_rctl |= IXGBE_FCTRL_MPE;
1912 reg_rctl &= ~IXGBE_FCTRL_UPE;
1913 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1914 }
1915 return;
1916}
1917
1918
1919/*********************************************************************
1920 * Multicast Update
1921 *
1922 * This routine is called whenever multicast address list is updated.
1923 *
1924 **********************************************************************/
1925#define IXGBE_RAR_ENTRIES 16
1926
1927static void
1928ixgbe_set_multi(struct adapter *adapter)
1929{
1930 u32 fctrl;
1931 u8 *mta;
1932 u8 *update_ptr;
1933 struct ifmultiaddr *ifma;
1934 int mcnt = 0;
1935 struct ifnet *ifp = adapter->ifp;
1936
1937 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1938
1939 mta = adapter->mta;
1940 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1941 MAX_NUM_MULTICAST_ADDRESSES);
1942
1943 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1944 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1945 if (ifp->if_flags & IFF_PROMISC)
1946 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1947 else if (ifp->if_flags & IFF_ALLMULTI) {
1948 fctrl |= IXGBE_FCTRL_MPE;
1949 fctrl &= ~IXGBE_FCTRL_UPE;
1950 } else
1951 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1952
1953 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1954
9407f759
FT		 1955 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1956 if (ifma->ifma_addr->sa_family != AF_LINK)
1957 continue;
1958 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1959 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1960 IXGBE_ETH_LENGTH_OF_ADDRESS);
1961 mcnt++;
1962 }
9407f759
FT		 1963
1964 update_ptr = mta;
1965 ixgbe_update_mc_addr_list(&adapter->hw,
1966 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1967
1968 return;
1969}
1970
1971/*
1972 * This is an iterator function now needed by the multicast
1973 * shared code. It simply feeds the shared code routine the
1974 * addresses in the array of ixgbe_set_multi() one by one.
1975 */
1976static u8 *
1977ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1978{
1979 u8 *addr = *update_ptr;
1980 u8 *newptr;
1981 *vmdq = 0;
1982
1983 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1984 *update_ptr = newptr;
1985 return addr;
1986}
1987
1988
1989/*********************************************************************
1990 * Timer routine
1991 *
1992 * This routine checks for link status,updates statistics,
1993 * and runs the watchdog check.
1994 *
1995 **********************************************************************/
1996
1997static void
1998ixgbe_local_timer(void *arg)
1999{
2000 struct adapter *adapter = arg;
2001 device_t dev = adapter->dev;
2002 struct ifnet *ifp = adapter->ifp;
2003 struct ix_queue *que = adapter->queues;
2004 struct tx_ring *txr = adapter->tx_rings;
2005 int hung, busy, paused;
2006
fe609108 2007 IXGBE_CORE_LOCK(adapter);
9407f759
FT		 2008 hung = busy = paused = 0;
2009
2010 /* Check for pluggable optics */
2011 if (adapter->sfp_probe)
2012 if (!ixgbe_sfp_probe(adapter))
2013 goto out; /* Nothing to do */
2014
2015 ixgbe_update_link_status(adapter);
2016 ixgbe_update_stats_counters(adapter);
2017
2018 /*
2019 * If the interface has been paused
2020 * then don't do the watchdog check
2021 */
2022 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2023 paused = 1;
2024
2025 /*
2026 ** Check the TX queues status
2027 ** - central locked handling of OACTIVE
2028 ** - watchdog only if all queues show hung
2029 */
2030 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2031 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2032 (paused == 0))
2033 ++hung;
2034 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2035 ++busy;
2036 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2037 taskqueue_enqueue(que->tq, &que->que_task);
2038 }
2039 /* Only truely watchdog if all queues show hung */
2040 if (hung == adapter->num_queues)
2041 goto watchdog;
2042 /* Only turn off the stack flow when ALL are depleted */
2043 if (busy == adapter->num_queues)
2044 ifp->if_flags |= IFF_OACTIVE;
2045 else if ((ifp->if_flags & IFF_OACTIVE) &&
2046 (busy < adapter->num_queues))
2047 ifp->if_flags &= ~IFF_OACTIVE;
2048
2049out:
2050 ixgbe_rearm_queues(adapter, adapter->que_mask);
2051 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
fe609108 2052 IXGBE_CORE_UNLOCK(adapter);
9407f759
FT		 2053 return;
2054
2055watchdog:
2056 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2057 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2058 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2059 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2060 device_printf(dev,"TX(%d) desc avail = %d,"
2061 "Next TX to Clean = %d\n",
2062 txr->me, txr->tx_avail, txr->next_to_clean);
2063 adapter->ifp->if_flags &= ~IFF_RUNNING;
2064 adapter->watchdog_events++;
2065 ixgbe_init_locked(adapter);
2066
fe609108 2067 IXGBE_CORE_UNLOCK(adapter);
9407f759
FT		 2068}
2069
2070/*
2071** Note: this routine updates the OS on the link state
2072** the real check of the hardware only happens with
2073** a link interrupt.
2074*/
2075static void
2076ixgbe_update_link_status(struct adapter *adapter)
2077{
2078 struct ifnet *ifp = adapter->ifp;
2079 struct tx_ring *txr = adapter->tx_rings;
2080 device_t dev = adapter->dev;
2081
2082
2083 if (adapter->link_up){
2084 if (adapter->link_active == FALSE) {
2085 if (bootverbose)
2086 device_printf(dev,"Link is up %d Gbps %s \n",
2087 ((adapter->link_speed == 128)? 10:1),
2088 "Full Duplex");
2089 adapter->link_active = TRUE;
ce451236
FT		 2090 /* Update any Flow Control changes */
2091 ixgbe_fc_enable(&adapter->hw);
9407f759
FT		 2092 ifp->if_link_state = LINK_STATE_UP;
2093 if_link_state_change(ifp);
2094 }
2095 } else { /* Link down */
2096 if (adapter->link_active == TRUE) {
2097 if (bootverbose)
2098 device_printf(dev,"Link is Down\n");
2099 ifp->if_link_state = LINK_STATE_DOWN;
2100 if_link_state_change(ifp);
2101 adapter->link_active = FALSE;
2102 for (int i = 0; i < adapter->num_queues;
2103 i++, txr++)
2104 txr->queue_status = IXGBE_QUEUE_IDLE;
2105 }
2106 }
2107
2108 return;
2109}
2110
2111
2112/*********************************************************************
2113 *
2114 * This routine disables all traffic on the adapter by issuing a
2115 * global reset on the MAC and deallocates TX/RX buffers.
2116 *
2117 **********************************************************************/
2118
2119static void
2120ixgbe_stop(void *arg)
2121{
2122 struct ifnet *ifp;
2123 struct adapter *adapter = arg;
2124 struct ixgbe_hw *hw = &adapter->hw;
2125 ifp = adapter->ifp;
2126
2127 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2128
2129 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2130 ixgbe_disable_intr(adapter);
2131 callout_stop(&adapter->timer);
2132
2133 /* Let the stack know...*/
2134 ifp->if_flags &= ~IFF_RUNNING;
2135 ifp->if_flags |= IFF_OACTIVE;
2136
2137 ixgbe_reset_hw(hw);
2138 hw->adapter_stopped = FALSE;
2139 ixgbe_stop_adapter(hw);
2140 /* Turn off the laser */
2141 if (hw->phy.multispeed_fiber)
2142 ixgbe_disable_tx_laser(hw);
2143
2144 /* reprogram the RAR[0] in case user changed it. */
2145 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2146
2147 return;
2148}
2149
2150
2151/*********************************************************************
2152 *
2153 * Determine hardware revision.
2154 *
2155 **********************************************************************/
2156static void
2157ixgbe_identify_hardware(struct adapter *adapter)
2158{
2159 device_t dev = adapter->dev;
2160 struct ixgbe_hw *hw = &adapter->hw;
2161
2162 /* Save off the information about this board */
2163 hw->vendor_id = pci_get_vendor(dev);
2164 hw->device_id = pci_get_device(dev);
2165 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2166 hw->subsystem_vendor_id =
2167 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2168 hw->subsystem_device_id =
2169 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2170
2171 /* We need this here to set the num_segs below */
2172 ixgbe_set_mac_type(hw);
2173
2174 /* Pick up the 82599 and VF settings */
2175 if (hw->mac.type != ixgbe_mac_82598EB) {
2176 hw->phy.smart_speed = ixgbe_smart_speed;
2177 adapter->num_segs = IXGBE_82599_SCATTER;
2178 } else
2179 adapter->num_segs = IXGBE_82598_SCATTER;
2180
2181 return;
2182}
2183
2184/*********************************************************************
2185 *
2186 * Determine optic type
2187 *
2188 **********************************************************************/
2189static void
2190ixgbe_setup_optics(struct adapter *adapter)
2191{
2192 struct ixgbe_hw *hw = &adapter->hw;
2193 int layer;
2194
2195 layer = ixgbe_get_supported_physical_layer(hw);
2196
2197 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2198 adapter->optics = IFM_10G_T;
2199 return;
2200 }
2201
2202 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2203 adapter->optics = IFM_1000_T;
2204 return;
2205 }
2206
2207 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2208 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2209 adapter->optics = IFM_10G_LR;
2210 return;
2211 }
2212
2213 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2214 adapter->optics = IFM_10G_SR;
2215 return;
2216 }
2217
2218 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2219 adapter->optics = IFM_10G_TWINAX;
2220 return;
2221 }
2222
2223 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2224 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2225 adapter->optics = IFM_10G_CX4;
2226 return;
2227 }
2228
2229 /* If we get here just set the default */
2230 adapter->optics = IFM_ETHER | IFM_AUTO;
2231 return;
2232}
2233
2234/*********************************************************************
2235 *
2236 * Setup the Legacy or MSI Interrupt handler
2237 *
2238 **********************************************************************/
2239static int
2240ixgbe_allocate_legacy(struct adapter *adapter)
2241{
2242 device_t dev = adapter->dev;
2243 struct ix_queue *que = adapter->queues;
2244 int error, rid = 0;
7e665ea1 2245 unsigned int intr_flags;
9407f759
FT		 2246
2247 /* MSI RID at 1 */
2248 if (adapter->msix == 1)
2249 rid = 1;
2250
7e665ea1
FT		 2251 /* Try allocating a MSI interrupt first */
2252 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2253 &rid, &intr_flags);
2254
9407f759
FT		 2255 /* We allocate a single interrupt resource */
2256 adapter->res = bus_alloc_resource_any(dev,
7e665ea1 2257 SYS_RES_IRQ, &rid, intr_flags);
9407f759
FT		 2258 if (adapter->res == NULL) {
2259 device_printf(dev, "Unable to allocate bus resource: "
2260 "interrupt\n");
2261 return (ENXIO);
2262 }
2263
2264 /*
2265 * Try allocating a fast interrupt and the associated deferred
2266 * processing contexts.
2267 */
2268 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2269 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2270 taskqueue_thread_enqueue, &que->tq);
2271 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2272 device_get_nameunit(adapter->dev));
2273
2274 /* Tasklets for Link, SFP and Multispeed Fiber */
2275 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2276 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2277 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2278#ifdef IXGBE_FDIR
2279 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2280#endif
2281 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2282 taskqueue_thread_enqueue, &adapter->tq);
2283 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2284 device_get_nameunit(adapter->dev));
2285
2286 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2287 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2288 device_printf(dev, "Failed to register fast interrupt "
2289 "handler: %d\n", error);
2290 taskqueue_free(que->tq);
2291 taskqueue_free(adapter->tq);
2292 que->tq = NULL;
2293 adapter->tq = NULL;
2294 return (error);
2295 }
2296 /* For simplicity in the handlers */
2297 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2298
2299 return (0);
2300}
2301
2302
2303/*********************************************************************
2304 *
2305 * Setup MSIX Interrupt resources and handlers
2306 *
2307 **********************************************************************/
2308static int
2309ixgbe_allocate_msix(struct adapter *adapter)
2310{
2311 device_t dev = adapter->dev;
2312 struct ix_queue *que = adapter->queues;
2313 int error, rid, vector = 0;
2314
2315 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2316 rid = vector + 1;
2317 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2318 RF_SHAREABLE | RF_ACTIVE);
2319 if (que->res == NULL) {
2320 device_printf(dev,"Unable to allocate"
2321 " bus resource: que interrupt [%d]\n", vector);
2322 return (ENXIO);
2323 }
2324 /* Set the handler function */
2325 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2326 ixgbe_msix_que, que, &que->tag, &que->serializer);
2327 if (error) {
2328 que->res = NULL;
2329 device_printf(dev, "Failed to register QUE handler");
2330 return (error);
2331 }
2332#if 0 /* __FreeBSD_version >= 800504 */
2333 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2334#endif
2335 que->msix = vector;
2336 adapter->que_mask |= (u64)(1 << que->msix);
2337 /*
2338 ** Bind the msix vector, and thus the
2339 ** ring to the corresponding cpu.
2340 */
2341#if 0 /* XXX */
2342 if (adapter->num_queues > 1)
2343 bus_bind_intr(dev, que->res, i);
2344#endif
2345
2346 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2347 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2348 taskqueue_thread_enqueue, &que->tq);
2349 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2350 device_get_nameunit(adapter->dev));
2351 }
2352
2353 /* and Link */
2354 rid = vector + 1;
2355 adapter->res = bus_alloc_resource_any(dev,
2356 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2357 if (!adapter->res) {
2358 device_printf(dev,"Unable to allocate"
2359 " bus resource: Link interrupt [%d]\n", rid);
2360 return (ENXIO);
2361 }
2362 /* Set the link handler function */
2363 error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2364 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2365 if (error) {
2366 adapter->res = NULL;
2367 device_printf(dev, "Failed to register LINK handler");
2368 return (error);
2369 }
2370#if 0 /* __FreeBSD_version >= 800504 */
2371 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2372#endif
2373 adapter->linkvec = vector;
2374 /* Tasklets for Link, SFP and Multispeed Fiber */
2375 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2376 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2377 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2378#ifdef IXGBE_FDIR
2379 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2380#endif
2381 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2382 taskqueue_thread_enqueue, &adapter->tq);
2383 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2384 device_get_nameunit(adapter->dev));
2385
2386 return (0);
2387}
2388
2389#if 0 /* HAVE_MSIX */
2390/*
2391 * Setup Either MSI/X or MSI
2392 */
2393static int
2394ixgbe_setup_msix(struct adapter *adapter)
2395{
2396 device_t dev = adapter->dev;
2397 int rid, want, queues, msgs;
2398
2399 /* Override by tuneable */
2400 if (ixgbe_enable_msix == 0)
2401 goto msi;
2402
2403 /* First try MSI/X */
2404 rid = PCIR_BAR(MSIX_82598_BAR);
2405 adapter->msix_mem = bus_alloc_resource_any(dev,
2406 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2407 if (!adapter->msix_mem) {
2408 rid += 4; /* 82599 maps in higher BAR */
2409 adapter->msix_mem = bus_alloc_resource_any(dev,
2410 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2411 }
2412 if (!adapter->msix_mem) {
2413 /* May not be enabled */
2414 device_printf(adapter->dev,
2415 "Unable to map MSIX table \n");
2416 goto msi;
2417 }
2418
2419 msgs = pci_msix_count(dev);
2420 if (msgs == 0) { /* system has msix disabled */
2421 bus_release_resource(dev, SYS_RES_MEMORY,
2422 rid, adapter->msix_mem);
2423 adapter->msix_mem = NULL;
2424 goto msi;
2425 }
2426
2427 /* Figure out a reasonable auto config value */
2428 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2429
2430 if (ixgbe_num_queues != 0)
2431 queues = ixgbe_num_queues;
2432 /* Set max queues to 8 when autoconfiguring */
2433 else if ((ixgbe_num_queues == 0) && (queues > 8))
2434 queues = 8;
2435
2436 /*
2437 ** Want one vector (RX/TX pair) per queue
2438 ** plus an additional for Link.
2439 */
2440 want = queues + 1;
2441 if (msgs >= want)
2442 msgs = want;
2443 else {
2444 device_printf(adapter->dev,
2445 "MSIX Configuration Problem, "
2446 "%d vectors but %d queues wanted!\n",
2447 msgs, want);
2448 return (0); /* Will go to Legacy setup */
2449 }
2450 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2451 device_printf(adapter->dev,
2452 "Using MSIX interrupts with %d vectors\n", msgs);
2453 adapter->num_queues = queues;
2454 return (msgs);
2455 }
2456msi:
2457 msgs = pci_msi_count(dev);
2458 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2459 device_printf(adapter->dev,"Using an MSI interrupt\n");
2460 else
2461 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2462 return (msgs);
2463}
2464#endif
2465
2466
2467static int
2468ixgbe_allocate_pci_resources(struct adapter *adapter)
2469{
2470 int rid;
2471 device_t dev = adapter->dev;
2472
2473 rid = PCIR_BAR(0);
2474 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2475 &rid, RF_ACTIVE);
2476
2477 if (!(adapter->pci_mem)) {
2478 device_printf(dev,"Unable to allocate bus resource: memory\n");
2479 return (ENXIO);
2480 }
2481
2482 adapter->osdep.mem_bus_space_tag =
2483 rman_get_bustag(adapter->pci_mem);
2484 adapter->osdep.mem_bus_space_handle =
2485 rman_get_bushandle(adapter->pci_mem);
2486 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2487
2488 /* Legacy defaults */
2489 adapter->num_queues = 1;
2490 adapter->hw.back = &adapter->osdep;
2491
2492 /*
2493 ** Now setup MSI or MSI/X, should
2494 ** return us the number of supported
2495 ** vectors. (Will be 1 for MSI)
2496 */
2497#if 0 /* HAVE_MSIX */
2498 adapter->msix = ixgbe_setup_msix(adapter);
2499#endif
2500 return (0);
2501}
2502
2503static void
2504ixgbe_free_pci_resources(struct adapter * adapter)
2505{
2506 struct ix_queue *que = adapter->queues;
2507 device_t dev = adapter->dev;
2508 int rid, memrid;
2509
2510 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2511 memrid = PCIR_BAR(MSIX_82598_BAR);
2512 else
2513 memrid = PCIR_BAR(MSIX_82599_BAR);
2514
2515 /*
2516 ** There is a slight possibility of a failure mode
2517 ** in attach that will result in entering this function
2518 ** before interrupt resources have been initialized, and
2519 ** in that case we do not want to execute the loops below
2520 ** We can detect this reliably by the state of the adapter
2521 ** res pointer.
2522 */
2523 if (adapter->res == NULL)
2524 goto mem;
2525
2526 /*
2527 ** Release all msix queue resources:
2528 */
2529 for (int i = 0; i < adapter->num_queues; i++, que++) {
2530 rid = que->msix + 1;
2531 if (que->tag != NULL) {
2532 bus_teardown_intr(dev, que->res, que->tag);
2533 que->tag = NULL;
2534 }
2535 if (que->res != NULL)
2536 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2537 }
2538
2539
2540 /* Clean the Legacy or Link interrupt last */
2541 if (adapter->linkvec) /* we are doing MSIX */
2542 rid = adapter->linkvec + 1;
2543 else
2544 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2545
2546 if (adapter->tag != NULL) {
2547 bus_teardown_intr(dev, adapter->res, adapter->tag);
2548 adapter->tag = NULL;
2549 }
2550 if (adapter->res != NULL)
2551 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
7e665ea1
FT		 2552 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2553 pci_release_msi(adapter->dev);
9407f759
FT		 2554
2555mem:
2556 if (adapter->msix)
2557 pci_release_msi(dev);
2558
2559 if (adapter->msix_mem != NULL)
2560 bus_release_resource(dev, SYS_RES_MEMORY,
2561 memrid, adapter->msix_mem);
2562
2563 if (adapter->pci_mem != NULL)
2564 bus_release_resource(dev, SYS_RES_MEMORY,
2565 PCIR_BAR(0), adapter->pci_mem);
2566
2567 return;
2568}
2569
2570/*********************************************************************
2571 *
2572 * Setup networking device structure and register an interface.
2573 *
2574 **********************************************************************/
2575static int
2576ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2577{
2578 struct ixgbe_hw *hw = &adapter->hw;
2579 struct ifnet *ifp;
2580
2581 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2582
2583 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2584 if (ifp == NULL) {
2585 device_printf(dev, "can not allocate ifnet structure\n");
2586 return (-1);
2587 }
2588 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2589 ifp->if_baudrate = 1000000000;
2590 ifp->if_init = ixgbe_init;
2591 ifp->if_softc = adapter;
2592 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2593 ifp->if_ioctl = ixgbe_ioctl;
2594 ifp->if_start = ixgbe_start;
2595#if 0 /* __FreeBSD_version >= 800000 */
2596 ifp->if_transmit = ixgbe_mq_start;
2597 ifp->if_qflush = ixgbe_qflush;
2598#endif
2599 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2600
2601 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2602
2603 adapter->max_frame_size =
2604 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2605
2606 /*
2607 * Tell the upper layer(s) we support long frames.
2608 */
2609 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2610
2611#if 0 /* NET_TSO */
2612 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2613#endif
2614 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2615 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2616 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2617#if 0 /* NET_TSO */
2618 | IFCAP_VLAN_HWTSO
2619#endif
2620 | IFCAP_VLAN_MTU;
2621 ifp->if_capenable = ifp->if_capabilities;
2622
2623 /* Don't enable LRO by default */
2624#if 0 /* NET_LRO */
2625 ifp->if_capabilities |= IFCAP_LRO;
2626#endif
2627
2628 /*
2629 ** Don't turn this on by default, if vlans are
2630 ** created on another pseudo device (eg. lagg)
2631 ** then vlan events are not passed thru, breaking
2632 ** operation, but with HW FILTER off it works. If
2633 ** using vlans directly on the ixgbe driver you can
2634 ** enable this and get full hardware tag filtering.
2635 */
2636 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2637
2638 /*
2639 * Specify the media types supported by this adapter and register
2640 * callbacks to update media and link information
2641 */
2642 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2643 ixgbe_media_status);
2644 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2645 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2646 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2647 ifmedia_add(&adapter->media,
2648 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2649 ifmedia_add(&adapter->media,
2650 IFM_ETHER | IFM_1000_T, 0, NULL);
2651 }
2652 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2653 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2654
2655 return (0);
2656}
2657
2658static void
2659ixgbe_config_link(struct adapter *adapter)
2660{
2661 struct ixgbe_hw *hw = &adapter->hw;
2662 u32 autoneg, err = 0;
2663 bool sfp, negotiate;
2664
2665 sfp = ixgbe_is_sfp(hw);
2666
2667 if (sfp) {
2668 if (hw->phy.multispeed_fiber) {
2669 hw->mac.ops.setup_sfp(hw);
2670 ixgbe_enable_tx_laser(hw);
2671 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2672 } else
2673 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2674 } else {
2675 if (hw->mac.ops.check_link)
2676 err = ixgbe_check_link(hw, &autoneg,
2677 &adapter->link_up, FALSE);
2678 if (err)
2679 goto out;
2680 autoneg = hw->phy.autoneg_advertised;
2681 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2682 err = hw->mac.ops.get_link_capabilities(hw,
2683 &autoneg, &negotiate);
2684 if (err)
2685 goto out;
2686 if (hw->mac.ops.setup_link)
2687 err = hw->mac.ops.setup_link(hw, autoneg,
2688 negotiate, adapter->link_up);
2689 }
2690out:
2691 return;
2692}
2693
2694/********************************************************************
2695 * Manage DMA'able memory.
2696 *******************************************************************/
2697static void
2698ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2699{
2700 if (error)
2701 return;
2702 *(bus_addr_t *) arg = segs->ds_addr;
2703 return;
2704}
2705
2706static int
2707ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2708 struct ixgbe_dma_alloc *dma, int mapflags)
2709{
2710 device_t dev = adapter->dev;
2711 int r;
2712
2713 r = bus_dma_tag_create(NULL, /* parent */
2714 DBA_ALIGN, 0, /* alignment, bounds */
2715 BUS_SPACE_MAXADDR, /* lowaddr */
2716 BUS_SPACE_MAXADDR, /* highaddr */
2717 NULL, NULL, /* filter, filterarg */
2718 size, /* maxsize */
2719 1, /* nsegments */
2720 size, /* maxsegsize */
2721 BUS_DMA_ALLOCNOW, /* flags */
2722 &dma->dma_tag);
2723 if (r != 0) {
2724 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2725 "error %u\n", r);
2726 goto fail_0;
2727 }
2728 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2729 BUS_DMA_NOWAIT, &dma->dma_map);
2730 if (r != 0) {
2731 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2732 "error %u\n", r);
2733 goto fail_1;
2734 }
2735 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2736 size,
2737 ixgbe_dmamap_cb,
2738 &dma->dma_paddr,
2739 mapflags | BUS_DMA_NOWAIT);
2740 if (r != 0) {
2741 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2742 "error %u\n", r);
2743 goto fail_2;
2744 }
2745 dma->dma_size = size;
2746 return (0);
2747fail_2:
2748 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2749fail_1:
2750 bus_dma_tag_destroy(dma->dma_tag);
2751fail_0:
2752 dma->dma_map = NULL;
2753 dma->dma_tag = NULL;
2754 return (r);
2755}
2756
2757static void
2758ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2759{
2760 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2761 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2762 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2763 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2764 bus_dma_tag_destroy(dma->dma_tag);
2765}
2766
2767
2768/*********************************************************************
2769 *
2770 * Allocate memory for the transmit and receive rings, and then
2771 * the descriptors associated with each, called only once at attach.
2772 *
2773 **********************************************************************/
2774static int
2775ixgbe_allocate_queues(struct adapter *adapter)
2776{
2777 device_t dev = adapter->dev;
2778 struct ix_queue *que;
2779 struct tx_ring *txr;
2780 struct rx_ring *rxr;
2781 int rsize, tsize, error = IXGBE_SUCCESS;
2782 int txconf = 0, rxconf = 0;
2783
2784 /* First allocate the top level queue structs */
2785 if (!(adapter->queues =
2786 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2787 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2788 device_printf(dev, "Unable to allocate queue memory\n");
2789 error = ENOMEM;
2790 goto fail;
2791 }
2792
2793 /* First allocate the TX ring struct memory */
2794 if (!(adapter->tx_rings =
2795 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2796 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2797 device_printf(dev, "Unable to allocate TX ring memory\n");
2798 error = ENOMEM;
2799 goto tx_fail;
2800 }
2801
2802 /* Next allocate the RX */
2803 if (!(adapter->rx_rings =
2804 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2805 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2806 device_printf(dev, "Unable to allocate RX ring memory\n");
2807 error = ENOMEM;
2808 goto rx_fail;
2809 }
2810
2811 /* For the ring itself */
2812 tsize = roundup2(adapter->num_tx_desc *
2813 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2814
2815 /*
2816 * Now set up the TX queues, txconf is needed to handle the
2817 * possibility that things fail midcourse and we need to
2818 * undo memory gracefully
2819 */
2820 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2821 /* Set up some basics */
2822 txr = &adapter->tx_rings[i];
2823 txr->adapter = adapter;
2824 txr->me = i;
2825
2826 /* Initialize the TX side lock */
2827 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2828 device_get_nameunit(dev), txr->me);
2829 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2830
2831 if (ixgbe_dma_malloc(adapter, tsize,
2832 &txr->txdma, BUS_DMA_NOWAIT)) {
2833 device_printf(dev,
2834 "Unable to allocate TX Descriptor memory\n");
2835 error = ENOMEM;
2836 goto err_tx_desc;
2837 }
2838 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2839 bzero((void *)txr->tx_base, tsize);
2840
2841 /* Now allocate transmit buffers for the ring */
2842 if (ixgbe_allocate_transmit_buffers(txr)) {
2843 device_printf(dev,
2844 "Critical Failure setting up transmit buffers\n");
2845 error = ENOMEM;
2846 goto err_tx_desc;
2847 }
2848#if 0 /* __FreeBSD_version >= 800000 */
2849 /* Allocate a buf ring */
2850 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2851 M_WAITOK, &txr->tx_mtx);
2852 if (txr->br == NULL) {
2853 device_printf(dev,
2854 "Critical Failure setting up buf ring\n");
2855 error = ENOMEM;
2856 goto err_tx_desc;
2857 }
2858#endif
2859 }
2860
2861 /*
2862 * Next the RX queues...
2863 */
2864 rsize = roundup2(adapter->num_rx_desc *
2865 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2866 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2867 rxr = &adapter->rx_rings[i];
2868 /* Set up some basics */
2869 rxr->adapter = adapter;
2870 rxr->me = i;
2871
2872 /* Initialize the RX side lock */
2873 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2874 device_get_nameunit(dev), rxr->me);
2875 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2876
2877 if (ixgbe_dma_malloc(adapter, rsize,
2878 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2879 device_printf(dev,
2880 "Unable to allocate RxDescriptor memory\n");
2881 error = ENOMEM;
2882 goto err_rx_desc;
2883 }
2884 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2885 bzero((void *)rxr->rx_base, rsize);
2886
2887 /* Allocate receive buffers for the ring*/
2888 if (ixgbe_allocate_receive_buffers(rxr)) {
2889 device_printf(dev,
2890 "Critical Failure setting up receive buffers\n");
2891 error = ENOMEM;
2892 goto err_rx_desc;
2893 }
2894 }
2895
2896 /*
2897 ** Finally set up the queue holding structs
2898 */
2899 for (int i = 0; i < adapter->num_queues; i++) {
2900 que = &adapter->queues[i];
2901 que->adapter = adapter;
2902 que->txr = &adapter->tx_rings[i];
2903 que->rxr = &adapter->rx_rings[i];
2904 }
2905
2906 return (0);
2907
2908err_rx_desc:
2909 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2910 ixgbe_dma_free(adapter, &rxr->rxdma);
2911err_tx_desc:
2912 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2913 ixgbe_dma_free(adapter, &txr->txdma);
2914 kfree(adapter->rx_rings, M_DEVBUF);
2915rx_fail:
2916 kfree(adapter->tx_rings, M_DEVBUF);
2917tx_fail:
2918 kfree(adapter->queues, M_DEVBUF);
2919fail:
2920 return (error);
2921}
2922
2923/*********************************************************************
2924 *
2925 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2926 * the information needed to transmit a packet on the wire. This is
2927 * called only once at attach, setup is done every reset.
2928 *
2929 **********************************************************************/
2930static int
2931ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2932{
2933 struct adapter *adapter = txr->adapter;
2934 device_t dev = adapter->dev;
2935 struct ixgbe_tx_buf *txbuf;
2936 int error, i;
2937
2938 /*
2939 * Setup DMA descriptor areas.
2940 */
2941 if ((error = bus_dma_tag_create(
2942 NULL, /* parent */
2943 1, 0, /* alignment, bounds */
2944 BUS_SPACE_MAXADDR, /* lowaddr */
2945 BUS_SPACE_MAXADDR, /* highaddr */
2946 NULL, NULL, /* filter, filterarg */
2947 IXGBE_TSO_SIZE, /* maxsize */
2948 adapter->num_segs, /* nsegments */
2949 PAGE_SIZE, /* maxsegsize */
2950 0, /* flags */
2951 &txr->txtag))) {
2952 device_printf(dev,"Unable to allocate TX DMA tag\n");
2953 goto fail;
2954 }
2955
2956 if (!(txr->tx_buffers =
2957 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2958 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2959 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2960 error = ENOMEM;
2961 goto fail;
2962 }
2963
2964 /* Create the descriptor buffer dma maps */
2965 txbuf = txr->tx_buffers;
2966 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2967 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2968 if (error != 0) {
2969 device_printf(dev, "Unable to create TX DMA map\n");
2970 goto fail;
2971 }
2972 }
2973
2974 return 0;
2975fail:
2976 /* We free all, it handles case where we are in the middle */
2977 ixgbe_free_transmit_structures(adapter);
2978 return (error);
2979}
2980
2981/*********************************************************************
2982 *
2983 * Initialize a transmit ring.
2984 *
2985 **********************************************************************/
2986static void
2987ixgbe_setup_transmit_ring(struct tx_ring *txr)
2988{
2989 struct adapter *adapter = txr->adapter;
2990 struct ixgbe_tx_buf *txbuf;
2991 int i;
2992#ifdef DEV_NETMAP
2993 struct netmap_adapter *na = NA(adapter->ifp);
2994 struct netmap_slot *slot;
2995#endif /* DEV_NETMAP */
2996
2997 /* Clear the old ring contents */
2998 IXGBE_TX_LOCK(txr);
2999#ifdef DEV_NETMAP
3000 /*
3001 * (under lock): if in netmap mode, do some consistency
3002 * checks and set slot to entry 0 of the netmap ring.
3003 */
3004 slot = netmap_reset(na, NR_TX, txr->me, 0);
3005#endif /* DEV_NETMAP */
3006 bzero((void *)txr->tx_base,
3007 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3008 /* Reset indices */
3009 txr->next_avail_desc = 0;
3010 txr->next_to_clean = 0;
3011
3012 /* Free any existing tx buffers. */
3013 txbuf = txr->tx_buffers;
3014 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3015 if (txbuf->m_head != NULL) {
3016 bus_dmamap_sync(txr->txtag, txbuf->map,
3017 BUS_DMASYNC_POSTWRITE);
3018 bus_dmamap_unload(txr->txtag, txbuf->map);
3019 m_freem(txbuf->m_head);
3020 txbuf->m_head = NULL;
3021 }
3022#ifdef DEV_NETMAP
3023 /*
3024 * In netmap mode, set the map for the packet buffer.
3025 * NOTE: Some drivers (not this one) also need to set
3026 * the physical buffer address in the NIC ring.
3027 * Slots in the netmap ring (indexed by "si") are
3028 * kring->nkr_hwofs positions "ahead" wrt the
3029 * corresponding slot in the NIC ring. In some drivers
3030 * (not here) nkr_hwofs can be negative. Function
3031 * netmap_idx_n2k() handles wraparounds properly.
3032 */
3033 if (slot) {
3034 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3035 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3036 }
3037#endif /* DEV_NETMAP */
3038 /* Clear the EOP index */
3039 txbuf->eop_index = -1;
3040 }
3041
3042#ifdef IXGBE_FDIR
3043 /* Set the rate at which we sample packets */
3044 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3045 txr->atr_sample = atr_sample_rate;
3046#endif
3047
3048 /* Set number of descriptors available */
3049 txr->tx_avail = adapter->num_tx_desc;
3050
3051 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3052 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3053 IXGBE_TX_UNLOCK(txr);
3054}
3055
3056/*********************************************************************
3057 *
3058 * Initialize all transmit rings.
3059 *
3060 **********************************************************************/
3061static int
3062ixgbe_setup_transmit_structures(struct adapter *adapter)
3063{
3064 struct tx_ring *txr = adapter->tx_rings;
3065
3066 for (int i = 0; i < adapter->num_queues; i++, txr++)
3067 ixgbe_setup_transmit_ring(txr);
3068
3069 return (0);
3070}
3071
3072/*********************************************************************
3073 *
3074 * Enable transmit unit.
3075 *
3076 **********************************************************************/
3077static void
3078ixgbe_initialize_transmit_units(struct adapter *adapter)
3079{
3080 struct tx_ring *txr = adapter->tx_rings;
3081 struct ixgbe_hw *hw = &adapter->hw;
3082
3083 /* Setup the Base and Length of the Tx Descriptor Ring */
3084
3085 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3086 u64 tdba = txr->txdma.dma_paddr;
3087 u32 txctrl;
3088
3089 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3090 (tdba & 0x00000000ffffffffULL));
3091 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3092 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3093 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3094
3095 /* Setup the HW Tx Head and Tail descriptor pointers */
3096 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3097 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3098
3099 /* Setup Transmit Descriptor Cmd Settings */
3100 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3101 txr->queue_status = IXGBE_QUEUE_IDLE;
3102
3103 /* Disable Head Writeback */
3104 switch (hw->mac.type) {
3105 case ixgbe_mac_82598EB:
3106 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3107 break;
3108 case ixgbe_mac_82599EB:
3109 case ixgbe_mac_X540:
3110 default:
3111 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3112 break;
3113 }
ce451236 3114 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
9407f759
FT		 3115 switch (hw->mac.type) {
3116 case ixgbe_mac_82598EB:
3117 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3118 break;
3119 case ixgbe_mac_82599EB:
3120 case ixgbe_mac_X540:
3121 default:
3122 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3123 break;
3124 }
3125
3126 }
3127
3128 if (hw->mac.type != ixgbe_mac_82598EB) {
3129 u32 dmatxctl, rttdcs;
3130 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3131 dmatxctl |= IXGBE_DMATXCTL_TE;
3132 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3133 /* Disable arbiter to set MTQC */
3134 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3135 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3136 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3137 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3138 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3139 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3140 }
3141
3142 return;
3143}
3144
3145/*********************************************************************
3146 *
3147 * Free all transmit rings.
3148 *
3149 **********************************************************************/
3150static void
3151ixgbe_free_transmit_structures(struct adapter *adapter)
3152{
3153 struct tx_ring *txr = adapter->tx_rings;
3154
3155 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3156 IXGBE_TX_LOCK(txr);
3157 ixgbe_free_transmit_buffers(txr);
3158 ixgbe_dma_free(adapter, &txr->txdma);
3159 IXGBE_TX_UNLOCK(txr);
3160 IXGBE_TX_LOCK_DESTROY(txr);
3161 }
3162 kfree(adapter->tx_rings, M_DEVBUF);
3163}
3164
3165/*********************************************************************
3166 *
3167 * Free transmit ring related data structures.
3168 *
3169 **********************************************************************/
3170static void
3171ixgbe_free_transmit_buffers(struct tx_ring *txr)
3172{
3173 struct adapter *adapter = txr->adapter;
3174 struct ixgbe_tx_buf *tx_buffer;
3175 int i;
3176
3177 INIT_DEBUGOUT("free_transmit_ring: begin");
3178
3179 if (txr->tx_buffers == NULL)
3180 return;
3181
3182 tx_buffer = txr->tx_buffers;
3183 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3184 if (tx_buffer->m_head != NULL) {
3185 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3186 BUS_DMASYNC_POSTWRITE);
3187 bus_dmamap_unload(txr->txtag,
3188 tx_buffer->map);
3189 m_freem(tx_buffer->m_head);
3190 tx_buffer->m_head = NULL;
3191 if (tx_buffer->map != NULL) {
3192 bus_dmamap_destroy(txr->txtag,
3193 tx_buffer->map);
3194 tx_buffer->map = NULL;
3195 }
3196 } else if (tx_buffer->map != NULL) {
3197 bus_dmamap_unload(txr->txtag,
3198 tx_buffer->map);
3199 bus_dmamap_destroy(txr->txtag,
3200 tx_buffer->map);
3201 tx_buffer->map = NULL;
3202 }
3203 }
3204#if 0 /* __FreeBSD_version >= 800000 */
3205 if (txr->br != NULL)
3206 buf_ring_free(txr->br, M_DEVBUF);
3207#endif
3208 if (txr->tx_buffers != NULL) {
3209 kfree(txr->tx_buffers, M_DEVBUF);
3210 txr->tx_buffers = NULL;
3211 }
3212 if (txr->txtag != NULL) {
3213 bus_dma_tag_destroy(txr->txtag);
3214 txr->txtag = NULL;
3215 }
3216 return;
3217}
3218
3219/*********************************************************************
3220 *
3221 * Advanced Context Descriptor setup for VLAN or CSUM
3222 *
3223 **********************************************************************/
3224
3225static bool
3226ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3227{
3228 struct adapter *adapter = txr->adapter;
3229 struct ixgbe_adv_tx_context_desc *TXD;
3230 struct ixgbe_tx_buf *tx_buffer;
3231 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3232 struct ether_vlan_header *eh;
3233 struct ip *ip;
3234 struct ip6_hdr *ip6;
3235 int ehdrlen, ip_hlen = 0;
3236 u16 etype;
3237 u8 ipproto = 0;
3238 bool offload = TRUE;
3239 int ctxd = txr->next_avail_desc;
3240#ifdef NET_VLAN
3241 u16 vtag = 0;
3242#endif
3243
3244
3245 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3246 offload = FALSE;
3247
3248 tx_buffer = &txr->tx_buffers[ctxd];
3249 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3250
3251 /*
3252 ** In advanced descriptors the vlan tag must
3253 ** be placed into the descriptor itself.
3254 */
3255#ifdef NET_VLAN
3256 if (mp->m_flags & M_VLANTAG) {
3257 vtag = htole16(mp->m_pkthdr.ether_vtag);
3258 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3259 } else if (offload == FALSE)
3260 return FALSE;
3261#endif
3262
3263 /*
3264 * Determine where frame payload starts.
3265 * Jump over vlan headers if already present,
3266 * helpful for QinQ too.
3267 */
3268 eh = mtod(mp, struct ether_vlan_header *);
3269 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3270 etype = ntohs(eh->evl_proto);
3271 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3272 } else {
3273 etype = ntohs(eh->evl_encap_proto);
3274 ehdrlen = ETHER_HDR_LEN;
3275 }
3276
3277 /* Set the ether header length */
3278 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3279
3280 switch (etype) {
3281 case ETHERTYPE_IP:
3282 ip = (struct ip *)(mp->m_data + ehdrlen);
3283 ip_hlen = ip->ip_hl << 2;
3284 ipproto = ip->ip_p;
3285 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3286 break;
3287 case ETHERTYPE_IPV6:
3288 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3289 ip_hlen = sizeof(struct ip6_hdr);
3290 /* XXX-BZ this will go badly in case of ext hdrs. */
3291 ipproto = ip6->ip6_nxt;
3292 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3293 break;
3294 default:
3295 offload = FALSE;
3296 break;
3297 }
3298
3299 vlan_macip_lens |= ip_hlen;
3300 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3301
3302 switch (ipproto) {
3303 case IPPROTO_TCP:
3304 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3305 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3306 break;
3307
3308 case IPPROTO_UDP:
3309 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3310 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3311 break;
3312
3313#if 0
3314 case IPPROTO_SCTP:
3315 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3316 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3317 break;
3318#endif
3319 default:
3320 offload = FALSE;
3321 break;
3322 }
3323
3324 /* Now copy bits into descriptor */
3325 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3326 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3327 TXD->seqnum_seed = htole32(0);
3328 TXD->mss_l4len_idx = htole32(0);
3329
3330 tx_buffer->m_head = NULL;
3331 tx_buffer->eop_index = -1;
3332
3333 /* We've consumed the first desc, adjust counters */
3334 if (++ctxd == adapter->num_tx_desc)
3335 ctxd = 0;
3336 txr->next_avail_desc = ctxd;
3337 --txr->tx_avail;
3338
3339 return (offload);
3340}
3341
3342/**********************************************************************
3343 *
3344 * Setup work for hardware segmentation offload (TSO) on
3345 * adapters using advanced tx descriptors
3346 *
3347 **********************************************************************/
3348#if 0 /* NET_TSO */
3349static bool
3350ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3351 u32 *olinfo_status)
3352{
3353 struct adapter *adapter = txr->adapter;
3354 struct ixgbe_adv_tx_context_desc *TXD;
3355 struct ixgbe_tx_buf *tx_buffer;
3356#ifdef NET_VLAN
3357 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3358 u16 vtag = 0, eh_type;
3359#else
3360 u16 eh_type;
3361 u32 type_tucmd_mlhl = 0;
3362#endif
3363 u32 mss_l4len_idx = 0, len;
3364 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3365 struct ether_vlan_header *eh;
3366#ifdef INET6
3367 struct ip6_hdr *ip6;
3368#endif
3369#ifdef INET
3370 struct ip *ip;
3371#endif
3372 struct tcphdr *th;
3373
3374
3375 /*
3376 * Determine where frame payload starts.
3377 * Jump over vlan headers if already present
3378 */
3379 eh = mtod(mp, struct ether_vlan_header *);
3380 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3381 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3382 eh_type = eh->evl_proto;
3383 } else {
3384 ehdrlen = ETHER_HDR_LEN;
3385 eh_type = eh->evl_encap_proto;
3386 }
3387
3388 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3389 len = ehdrlen + sizeof(struct tcphdr);
3390 switch (ntohs(eh_type)) {
3391#ifdef INET6
3392 case ETHERTYPE_IPV6:
3393 if (mp->m_len < len + sizeof(struct ip6_hdr))
3394 return FALSE;
3395 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3396 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3397 if (ip6->ip6_nxt != IPPROTO_TCP)
3398 return FALSE;
3399 ip_hlen = sizeof(struct ip6_hdr);
3400 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3401 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3402 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3403 break;
3404#endif
3405#ifdef INET
3406 case ETHERTYPE_IP:
3407 if (mp->m_len < len + sizeof(struct ip))
3408 return FALSE;
3409 ip = (struct ip *)(mp->m_data + ehdrlen);
3410 if (ip->ip_p != IPPROTO_TCP)
3411 return FALSE;
3412 ip->ip_sum = 0;
3413 ip_hlen = ip->ip_hl << 2;
3414 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3415 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3416 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3417 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3418 /* Tell transmit desc to also do IPv4 checksum. */
3419 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3420 break;
3421#endif
3422 default:
3423 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3424 __func__, ntohs(eh_type));
3425 break;
3426 }
3427
3428 ctxd = txr->next_avail_desc;
3429 tx_buffer = &txr->tx_buffers[ctxd];
3430 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3431
3432 tcp_hlen = th->th_off << 2;
3433
3434 /* This is used in the transmit desc in encap */
3435 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3436
3437 /* VLAN MACLEN IPLEN */
3438#ifdef NET_VLAN
3439 if (mp->m_flags & M_VLANTAG) {
3440 vtag = htole16(mp->m_pkthdr.ether_vtag);
3441 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3442 }
3443
3444 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3445 vlan_macip_lens |= ip_hlen;
3446 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3447#endif
3448
3449 /* ADV DTYPE TUCMD */
3450 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3451 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3452 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3453
3454 /* MSS L4LEN IDX */
3455 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3456 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3457 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3458
3459 TXD->seqnum_seed = htole32(0);
3460 tx_buffer->m_head = NULL;
3461 tx_buffer->eop_index = -1;
3462
3463 if (++ctxd == adapter->num_tx_desc)
3464 ctxd = 0;
3465
3466 txr->tx_avail--;
3467 txr->next_avail_desc = ctxd;
3468 return TRUE;
3469}
3470#endif
3471
3472#ifdef IXGBE_FDIR
3473/*
3474** This routine parses packet headers so that Flow
3475** Director can make a hashed filter table entry
3476** allowing traffic flows to be identified and kept
3477** on the same cpu. This would be a performance
3478** hit, but we only do it at IXGBE_FDIR_RATE of
3479** packets.
3480*/
3481static void
3482ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3483{
3484 struct adapter *adapter = txr->adapter;
3485 struct ix_queue *que;
3486 struct ip *ip;
3487 struct tcphdr *th;
3488 struct udphdr *uh;
3489 struct ether_vlan_header *eh;
3490 union ixgbe_atr_hash_dword input = {.dword = 0};
3491 union ixgbe_atr_hash_dword common = {.dword = 0};
3492 int ehdrlen, ip_hlen;
3493 u16 etype;
3494
3495 eh = mtod(mp, struct ether_vlan_header *);
3496 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3497 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3498 etype = eh->evl_proto;
3499 } else {
3500 ehdrlen = ETHER_HDR_LEN;
3501 etype = eh->evl_encap_proto;
3502 }
3503
3504 /* Only handling IPv4 */
3505 if (etype != htons(ETHERTYPE_IP))
3506 return;
3507
3508 ip = (struct ip *)(mp->m_data + ehdrlen);
3509 ip_hlen = ip->ip_hl << 2;
3510
3511 /* check if we're UDP or TCP */
3512 switch (ip->ip_p) {
3513 case IPPROTO_TCP:
3514 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3515 /* src and dst are inverted */
3516 common.port.dst ^= th->th_sport;
3517 common.port.src ^= th->th_dport;
3518 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3519 break;
3520 case IPPROTO_UDP:
3521 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3522 /* src and dst are inverted */
3523 common.port.dst ^= uh->uh_sport;
3524 common.port.src ^= uh->uh_dport;
3525 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3526 break;
3527 default:
3528 return;
3529 }
3530
3531 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3532 if (mp->m_pkthdr.ether_vtag)
3533 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3534 else
3535 common.flex_bytes ^= etype;
3536 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3537
3538 que = &adapter->queues[txr->me];
3539 /*
3540 ** This assumes the Rx queue and Tx
3541 ** queue are bound to the same CPU
3542 */
3543 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3544 input, common, que->msix);
3545}
3546#endif /* IXGBE_FDIR */
3547
3548/**********************************************************************
3549 *
3550 * Examine each tx_buffer in the used queue. If the hardware is done
3551 * processing the packet then free associated resources. The
3552 * tx_buffer is put back on the free queue.
3553 *
3554 **********************************************************************/
3555static bool
3556ixgbe_txeof(struct tx_ring *txr)
3557{
3558 struct adapter *adapter = txr->adapter;
3559 struct ifnet *ifp = adapter->ifp;
3560 u32 first, last, done, processed;
3561 struct ixgbe_tx_buf *tx_buffer;
3562 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3563
3564 KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3565
3566#ifdef DEV_NETMAP
3567 if (ifp->if_capenable & IFCAP_NETMAP) {
3568 struct netmap_adapter *na = NA(ifp);
3569 struct netmap_kring *kring = &na->tx_rings[txr->me];
3570
3571 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3572
3573 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3574 BUS_DMASYNC_POSTREAD);
3575 /*
3576 * In netmap mode, all the work is done in the context
3577 * of the client thread. Interrupt handlers only wake up
3578 * clients, which may be sleeping on individual rings
3579 * or on a global resource for all rings.
3580 * To implement tx interrupt mitigation, we wake up the client
3581 * thread roughly every half ring, even if the NIC interrupts
3582 * more frequently. This is implemented as follows:
3583 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3584 * the slot that should wake up the thread (nkr_num_slots
3585 * means the user thread should not be woken up);
3586 * - the driver ignores tx interrupts unless netmap_mitigate=0
3587 * or the slot has the DD bit set.
3588 *
3589 * When the driver has separate locks, we need to
3590 * release and re-acquire txlock to avoid deadlocks.
3591 * XXX see if we can find a better way.
3592 */
3593 if (!netmap_mitigate ||
3594 (kring->nr_kflags < kring->nkr_num_slots &&
3595 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3596 kring->nr_kflags = kring->nkr_num_slots;
3597 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3598 IXGBE_TX_UNLOCK(txr);
3599 IXGBE_CORE_LOCK(adapter);
3600 selwakeuppri(&na->tx_si, PI_NET);
3601 IXGBE_CORE_UNLOCK(adapter);
3602 IXGBE_TX_LOCK(txr);
3603 }
3604 return FALSE;
3605 }
3606#endif /* DEV_NETMAP */
3607
3608 if (txr->tx_avail == adapter->num_tx_desc) {
3609 txr->queue_status = IXGBE_QUEUE_IDLE;
3610 return FALSE;
3611 }
3612
3613 processed = 0;
3614 first = txr->next_to_clean;
3615 tx_buffer = &txr->tx_buffers[first];
3616 /* For cleanup we just use legacy struct */
3617 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3618 last = tx_buffer->eop_index;
3619 if (last == -1)
3620 return FALSE;
3621 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3622
3623 /*
3624 ** Get the index of the first descriptor
3625 ** BEYOND the EOP and call that 'done'.
3626 ** I do this so the comparison in the
3627 ** inner while loop below can be simple
3628 */
3629 if (++last == adapter->num_tx_desc) last = 0;
3630 done = last;
3631
3632 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3633 BUS_DMASYNC_POSTREAD);
3634 /*
3635 ** Only the EOP descriptor of a packet now has the DD
3636 ** bit set, this is what we look for...
3637 */
3638 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3639 /* We clean the range of the packet */
3640 while (first != done) {
3641 tx_desc->upper.data = 0;
3642 tx_desc->lower.data = 0;
3643 tx_desc->buffer_addr = 0;
3644 ++txr->tx_avail;
3645 ++processed;
3646
3647 if (tx_buffer->m_head) {
3648 txr->bytes +=
3649 tx_buffer->m_head->m_pkthdr.len;
3650 bus_dmamap_sync(txr->txtag,
3651 tx_buffer->map,
3652 BUS_DMASYNC_POSTWRITE);
3653 bus_dmamap_unload(txr->txtag,
3654 tx_buffer->map);
3655 m_freem(tx_buffer->m_head);
3656 tx_buffer->m_head = NULL;
3657 tx_buffer->map = NULL;
3658 }
3659 tx_buffer->eop_index = -1;
3660 txr->watchdog_time = ticks;
3661
3662 if (++first == adapter->num_tx_desc)
3663 first = 0;
3664
3665 tx_buffer = &txr->tx_buffers[first];
3666 tx_desc =
3667 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3668 }
3669 ++txr->packets;
3670 ++ifp->if_opackets;
3671 /* See if there is more work now */
3672 last = tx_buffer->eop_index;
3673 if (last != -1) {
3674 eop_desc =
3675 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3676 /* Get next done point */
3677 if (++last == adapter->num_tx_desc) last = 0;
3678 done = last;
3679 } else
3680 break;
3681 }
3682 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3683 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3684
3685 txr->next_to_clean = first;
3686
3687 /*
3688 ** Watchdog calculation, we know there's
3689 ** work outstanding or the first return
3690 ** would have been taken, so none processed
3691 ** for too long indicates a hang.
3692 */
3693 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3694 txr->queue_status = IXGBE_QUEUE_HUNG;
3695
3696 /* With a minimum free clear the depleted state bit. */
3697 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3698 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3699
3700 if (txr->tx_avail == adapter->num_tx_desc) {
3701 txr->queue_status = IXGBE_QUEUE_IDLE;
3702 return (FALSE);
3703 }
3704
3705 return TRUE;
3706}
3707
3708/*********************************************************************
3709 *
3710 * Refresh mbuf buffers for RX descriptor rings
3711 * - now keeps its own state so discards due to resource
3712 * exhaustion are unnecessary, if an mbuf cannot be obtained
3713 * it just returns, keeping its placeholder, thus it can simply
3714 * be recalled to try again.
3715 *
3716 **********************************************************************/
3717static void
3718ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3719{
3720 struct adapter *adapter = rxr->adapter;
3721 bus_dma_segment_t hseg[1];
3722 bus_dma_segment_t pseg[1];
3723 struct ixgbe_rx_buf *rxbuf;
3724 struct mbuf *mh, *mp;
3725 int i, j, nsegs, error;
3726 bool refreshed = FALSE;
3727
3728 i = j = rxr->next_to_refresh;
3729 /* Control the loop with one beyond */
3730 if (++j == adapter->num_rx_desc)
3731 j = 0;
3732
3733 while (j != limit) {
3734 rxbuf = &rxr->rx_buffers[i];
3735 if (rxr->hdr_split == FALSE)
3736 goto no_split;
3737
3738 if (rxbuf->m_head == NULL) {
3739 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3740 if (mh == NULL)
3741 goto update;
3742 } else
3743 mh = rxbuf->m_head;
3744
3745 mh->m_pkthdr.len = mh->m_len = MHLEN;
3746 mh->m_len = MHLEN;
3747 mh->m_flags |= M_PKTHDR;
3748 /* Get the memory mapping */
3749 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3750 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3751 if (error != 0) {
3752 kprintf("Refresh mbufs: hdr dmamap load"
3753 " failure - %d\n", error);
3754 m_free(mh);
3755 rxbuf->m_head = NULL;
3756 goto update;
3757 }
3758 rxbuf->m_head = mh;
3759 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3760 BUS_DMASYNC_PREREAD);
3761 rxr->rx_base[i].read.hdr_addr =
3762 htole64(hseg[0].ds_addr);
3763
3764no_split:
3765 if (rxbuf->m_pack == NULL) {
3766 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3767 M_PKTHDR, adapter->rx_mbuf_sz);
3768 if (mp == NULL)
3769 goto update;
3770 } else
3771 mp = rxbuf->m_pack;
3772
3773 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3774 /* Get the memory mapping */
3775 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3776 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3777 if (error != 0) {
3778 kprintf("Refresh mbufs: payload dmamap load"
3779 " failure - %d\n", error);
3780 m_free(mp);
3781 rxbuf->m_pack = NULL;
3782 goto update;
3783 }
3784 rxbuf->m_pack = mp;
3785 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3786 BUS_DMASYNC_PREREAD);
3787 rxr->rx_base[i].read.pkt_addr =
3788 htole64(pseg[0].ds_addr);
3789
3790 refreshed = TRUE;
3791 /* Next is precalculated */
3792 i = j;
3793 rxr->next_to_refresh = i;
3794 if (++j == adapter->num_rx_desc)
3795 j = 0;
3796 }
3797update:
3798 if (refreshed) /* Update hardware tail index */
3799 IXGBE_WRITE_REG(&adapter->hw,
3800 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3801 return;
3802}
3803
3804/*********************************************************************
3805 *
3806 * Allocate memory for rx_buffer structures. Since we use one
3807 * rx_buffer per received packet, the maximum number of rx_buffer's
3808 * that we'll need is equal to the number of receive descriptors
3809 * that we've allocated.
3810 *
3811 **********************************************************************/
3812static int
3813ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3814{
3815 struct adapter *adapter = rxr->adapter;
3816 device_t dev = adapter->dev;
3817 struct ixgbe_rx_buf *rxbuf;
3818 int i, bsize, error;
3819
3820 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3821 if (!(rxr->rx_buffers =
3822 (struct ixgbe_rx_buf *) kmalloc(bsize,
3823 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3824 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3825 error = ENOMEM;
3826 goto fail;
3827 }
3828
3829 if ((error = bus_dma_tag_create(NULL, /* parent */
3830 1, 0, /* alignment, bounds */
3831 BUS_SPACE_MAXADDR, /* lowaddr */
3832 BUS_SPACE_MAXADDR, /* highaddr */
3833 NULL, NULL, /* filter, filterarg */
3834 MSIZE, /* maxsize */
3835 1, /* nsegments */
3836 MSIZE, /* maxsegsize */
3837 0, /* flags */
3838 &rxr->htag))) {
3839 device_printf(dev, "Unable to create RX DMA tag\n");
3840 goto fail;
3841 }
3842
3843 if ((error = bus_dma_tag_create(NULL, /* parent */
3844 1, 0, /* alignment, bounds */
3845 BUS_SPACE_MAXADDR, /* lowaddr */
3846 BUS_SPACE_MAXADDR, /* highaddr */
3847 NULL, NULL, /* filter, filterarg */
3848 MJUM16BYTES, /* maxsize */
3849 1, /* nsegments */
3850 MJUM16BYTES, /* maxsegsize */
3851 0, /* flags */
3852 &rxr->ptag))) {
3853 device_printf(dev, "Unable to create RX DMA tag\n");
3854 goto fail;
3855 }
3856
3857 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3858 rxbuf = &rxr->rx_buffers[i];
3859 error = bus_dmamap_create(rxr->htag,
3860 BUS_DMA_NOWAIT, &rxbuf->hmap);
3861 if (error) {
3862 device_printf(dev, "Unable to create RX head map\n");
3863 goto fail;
3864 }
3865 error = bus_dmamap_create(rxr->ptag,
3866 BUS_DMA_NOWAIT, &rxbuf->pmap);
3867 if (error) {
3868 device_printf(dev, "Unable to create RX pkt map\n");
3869 goto fail;
3870 }
3871 }
3872
3873 return (0);
3874
3875fail:
3876 /* Frees all, but can handle partial completion */
3877 ixgbe_free_receive_structures(adapter);
3878 return (error);
3879}
3880
3881/*
3882** Used to detect a descriptor that has
3883** been merged by Hardware RSC.
3884*/
3885static inline u32
3886ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3887{
3888 return (le32toh(rx->wb.lower.lo_dword.data) &
3889 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3890}
3891
3892/*********************************************************************
3893 *
3894 * Initialize Hardware RSC (LRO) feature on 82599
3895 * for an RX ring, this is toggled by the LRO capability
3896 * even though it is transparent to the stack.
3897 *
3898 **********************************************************************/
3899#if 0 /* NET_LRO */
3900static void
3901ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3902{
3903 struct adapter *adapter = rxr->adapter;
3904 struct ixgbe_hw *hw = &adapter->hw;
3905 u32 rscctrl, rdrxctl;
3906
3907 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3908 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3909#ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3910 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3911#endif /* DEV_NETMAP */
3912 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3913 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3914 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3915
3916 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3917 rscctrl |= IXGBE_RSCCTL_RSCEN;
3918 /*
3919 ** Limit the total number of descriptors that
3920 ** can be combined, so it does not exceed 64K
3921 */
3922 if (adapter->rx_mbuf_sz == MCLBYTES)
3923 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3924 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3925 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3926 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3927 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3928 else /* Using 16K cluster */
3929 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3930
3931 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3932
3933 /* Enable TCP header recognition */
3934 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3935 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3936 IXGBE_PSRTYPE_TCPHDR));
3937
3938 /* Disable RSC for ACK packets */
3939 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3940 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3941
3942 rxr->hw_rsc = TRUE;
3943}
3944#endif
3945
3946static void
3947ixgbe_free_receive_ring(struct rx_ring *rxr)
3948{
3949 struct adapter *adapter;
3950 struct ixgbe_rx_buf *rxbuf;
3951 int i;
3952
3953 adapter = rxr->adapter;
3954 for (i = 0; i < adapter->num_rx_desc; i++) {
3955 rxbuf = &rxr->rx_buffers[i];
3956 if (rxbuf->m_head != NULL) {
3957 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3958 BUS_DMASYNC_POSTREAD);
3959 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3960 rxbuf->m_head->m_flags |= M_PKTHDR;
3961 m_freem(rxbuf->m_head);
3962 }
3963 if (rxbuf->m_pack != NULL) {
3964 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3965 BUS_DMASYNC_POSTREAD);
3966 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3967 rxbuf->m_pack->m_flags |= M_PKTHDR;
3968 m_freem(rxbuf->m_pack);
3969 }
3970 rxbuf->m_head = NULL;
3971 rxbuf->m_pack = NULL;
3972 }
3973}
3974
3975
3976/*********************************************************************
3977 *
3978 * Initialize a receive ring and its buffers.
3979 *
3980 **********************************************************************/
3981static int
3982ixgbe_setup_receive_ring(struct rx_ring *rxr)
3983{
3984 struct adapter *adapter;
3985 struct ifnet *ifp;
3986 device_t dev;
3987 struct ixgbe_rx_buf *rxbuf;
3988 bus_dma_segment_t pseg[1], hseg[1];
3989#if 0 /* NET_LRO */
3990 struct lro_ctrl *lro = &rxr->lro;
3991#endif
3992 int rsize, nsegs, error = 0;
3993#ifdef DEV_NETMAP
3994 struct netmap_adapter *na = NA(rxr->adapter->ifp);
3995 struct netmap_slot *slot;
3996#endif /* DEV_NETMAP */
3997
3998 adapter = rxr->adapter;
3999 ifp = adapter->ifp;
4000 dev = adapter->dev;
4001
4002 /* Clear the ring contents */
4003 IXGBE_RX_LOCK(rxr);
4004#ifdef DEV_NETMAP
4005 /* same as in ixgbe_setup_transmit_ring() */
4006 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4007#endif /* DEV_NETMAP */
4008 rsize = roundup2(adapter->num_rx_desc *
4009 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4010 bzero((void *)rxr->rx_base, rsize);
4011
4012 /* Free current RX buffer structs and their mbufs */
4013 ixgbe_free_receive_ring(rxr);
4014
4015 /* Configure header split? */
4016 if (ixgbe_header_split)
4017 rxr->hdr_split = TRUE;
4018
4019 /* Now replenish the mbufs */
4020 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4021 struct mbuf *mh, *mp;
4022
4023 rxbuf = &rxr->rx_buffers[j];
4024#ifdef DEV_NETMAP
4025 /*
4026 * In netmap mode, fill the map and set the buffer
4027 * address in the NIC ring, considering the offset
4028 * between the netmap and NIC rings (see comment in
4029 * ixgbe_setup_transmit_ring() ). No need to allocate
4030 * an mbuf, so end the block with a continue;
4031 */
4032 if (slot) {
4033 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4034 uint64_t paddr;
4035 void *addr;
4036
4037 addr = PNMB(slot + sj, &paddr);
4038 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4039 /* Update descriptor */
4040 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4041 continue;
4042 }
4043#endif /* DEV_NETMAP */
4044 /*
4045 ** Don't allocate mbufs if not
4046 ** doing header split, its wasteful
4047 */
4048 if (rxr->hdr_split == FALSE)
4049 goto skip_head;
4050
4051 /* First the header */
4052 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4053 if (rxbuf->m_head == NULL) {
4054 error = ENOBUFS;
4055 goto fail;
4056 }
4057 m_adj(rxbuf->m_head, ETHER_ALIGN);
4058 mh = rxbuf->m_head;
4059 mh->m_len = mh->m_pkthdr.len = MHLEN;
4060 mh->m_flags |= M_PKTHDR;
4061 /* Get the memory mapping */
4062 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4063 rxbuf->hmap, rxbuf->m_head, hseg, 1,
4064 &nsegs, BUS_DMA_NOWAIT);
4065
4066 if (error != 0) /* Nothing elegant to do here */
4067 goto fail;
4068 bus_dmamap_sync(rxr->htag,
4069 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4070 /* Update descriptor */
4071 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4072
4073skip_head:
4074 /* Now the payload cluster */
4075 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4076 M_PKTHDR, adapter->rx_mbuf_sz);
4077 if (rxbuf->m_pack == NULL) {
4078 error = ENOBUFS;
4079 goto fail;
4080 }
4081 mp = rxbuf->m_pack;
4082 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4083 /* Get the memory mapping */
4084 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4085 rxbuf->pmap, mp, hseg, 1,
4086 &nsegs, BUS_DMA_NOWAIT);
4087 if (error != 0)
4088 goto fail;
4089 bus_dmamap_sync(rxr->ptag,
4090 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4091 /* Update descriptor */
4092 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4093 }
4094
4095
4096 /* Setup our descriptor indices */
4097 rxr->next_to_check = 0;
4098 rxr->next_to_refresh = 0;
4099 rxr->lro_enabled = FALSE;
4100 rxr->rx_split_packets = 0;
4101 rxr->rx_bytes = 0;
4102 rxr->discard = FALSE;
4103 rxr->vtag_strip = FALSE;
4104
4105 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,