ixgbe: Remove the multicast spinlock
[dragonfly.git] / sys / dev / netif / ixgbe / ixgbe.c
CommitLineData
9407f759
FT
1/******************************************************************************
2
3 Copyright (c) 2001-2012, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
ce451236 33/*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
9407f759
FT
34
35#include "opt_inet.h"
36#include "opt_inet6.h"
37
38#include "ixgbe.h"
39
40/*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43int ixgbe_display_debug_stats = 0;
44
45/*********************************************************************
46 * Driver version
47 *********************************************************************/
ce451236 48char ixgbe_driver_version[] = "2.4.8";
9407f759
FT
49
50/*********************************************************************
51 * PCI Device ID Table
52 *
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
56 *
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
59
60static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
61{
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
ce451236 81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
9407f759
FT
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
ce451236 84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
9407f759
FT
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
87 {0, 0, 0, 0, 0}
88};
89
90/*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
93
94static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
96};
97
98/*********************************************************************
99 * Function prototypes
100 *********************************************************************/
101static int ixgbe_probe(device_t);
102static int ixgbe_attach(device_t);
103static int ixgbe_detach(device_t);
104static int ixgbe_shutdown(device_t);
105static void ixgbe_start(struct ifnet *);
106static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107#if 0 /* __FreeBSD_version >= 800000 */
108static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111static void ixgbe_qflush(struct ifnet *);
112#endif
113static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114static void ixgbe_init(void *);
115static void ixgbe_init_locked(struct adapter *);
116static void ixgbe_stop(void *);
117static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118static int ixgbe_media_change(struct ifnet *);
119static void ixgbe_identify_hardware(struct adapter *);
120static int ixgbe_allocate_pci_resources(struct adapter *);
121static int ixgbe_allocate_msix(struct adapter *);
122static int ixgbe_allocate_legacy(struct adapter *);
123static int ixgbe_allocate_queues(struct adapter *);
124#if 0 /* HAVE_MSIX */
125static int ixgbe_setup_msix(struct adapter *);
126#endif
127static void ixgbe_free_pci_resources(struct adapter *);
128static void ixgbe_local_timer(void *);
129static int ixgbe_setup_interface(device_t, struct adapter *);
130static void ixgbe_config_link(struct adapter *);
131
132static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133static int ixgbe_setup_transmit_structures(struct adapter *);
134static void ixgbe_setup_transmit_ring(struct tx_ring *);
135static void ixgbe_initialize_transmit_units(struct adapter *);
136static void ixgbe_free_transmit_structures(struct adapter *);
137static void ixgbe_free_transmit_buffers(struct tx_ring *);
138
139static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140static int ixgbe_setup_receive_structures(struct adapter *);
141static int ixgbe_setup_receive_ring(struct rx_ring *);
142static void ixgbe_initialize_receive_units(struct adapter *);
143static void ixgbe_free_receive_structures(struct adapter *);
144static void ixgbe_free_receive_buffers(struct rx_ring *);
145#if 0 /* NET_LRO */
146static void ixgbe_setup_hw_rsc(struct rx_ring *);
147#endif
148
149static void ixgbe_enable_intr(struct adapter *);
150static void ixgbe_disable_intr(struct adapter *);
151static void ixgbe_update_stats_counters(struct adapter *);
152static bool ixgbe_txeof(struct tx_ring *);
153static bool ixgbe_rxeof(struct ix_queue *, int);
154static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
155static void ixgbe_set_promisc(struct adapter *);
156static void ixgbe_set_multi(struct adapter *);
157static void ixgbe_update_link_status(struct adapter *);
158static void ixgbe_refresh_mbufs(struct rx_ring *, int);
159static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
160static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
161static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
162static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
163static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
164 struct ixgbe_dma_alloc *, int);
165static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
166static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
167 const char *, int *, int);
168static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
169#if 0 /* NET_TSO */
170static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
171#endif
172static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173static void ixgbe_configure_ivars(struct adapter *);
174static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
175
176static void ixgbe_setup_vlan_hw_support(struct adapter *);
177static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
179
180static void ixgbe_add_hw_stats(struct adapter *adapter);
181
182static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
184 struct mbuf *, u32);
185
186/* Support for pluggable optic modules */
187static bool ixgbe_sfp_probe(struct adapter *);
188static void ixgbe_setup_optics(struct adapter *);
189
190/* Legacy (single vector interrupt handler */
191static void ixgbe_legacy_irq(void *);
192
193/* The MSI/X Interrupt handlers */
194static void ixgbe_msix_que(void *);
195static void ixgbe_msix_link(void *);
196
197/* Deferred interrupt tasklets */
198static void ixgbe_handle_que(void *, int);
199static void ixgbe_handle_link(void *, int);
200static void ixgbe_handle_msf(void *, int);
201static void ixgbe_handle_mod(void *, int);
202
203#ifdef IXGBE_FDIR
204static void ixgbe_atr(struct tx_ring *, struct mbuf *);
205static void ixgbe_reinit_fdir(void *, int);
206#endif
207
208/*********************************************************************
209 * FreeBSD Device Interface Entry Points
210 *********************************************************************/
211
212static device_method_t ixgbe_methods[] = {
213 /* Device interface */
214 DEVMETHOD(device_probe, ixgbe_probe),
215 DEVMETHOD(device_attach, ixgbe_attach),
216 DEVMETHOD(device_detach, ixgbe_detach),
217 DEVMETHOD(device_shutdown, ixgbe_shutdown),
218 {0, 0}
219};
220
221static driver_t ixgbe_driver = {
222 "ix", ixgbe_methods, sizeof(struct adapter),
223};
224
225devclass_t ixgbe_devclass;
226DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
227
228MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
229MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230
231/*
232** TUNEABLE PARAMETERS:
233*/
234
235/*
236** AIM: Adaptive Interrupt Moderation
237** which means that the interrupt rate
238** is varied over time based on the
239** traffic for that interrupt vector
240*/
241static int ixgbe_enable_aim = TRUE;
242TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
243
244static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
245TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
246
247/* How many packets rxeof tries to clean at a time */
248static int ixgbe_rx_process_limit = 128;
249TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
250
9407f759
FT
251/*
252** Smart speed setting, default to on
253** this only works as a compile option
254** right now as its during attach, set
255** this to 'ixgbe_smart_speed_off' to
256** disable.
257*/
258static int ixgbe_smart_speed = ixgbe_smart_speed_on;
259
7e665ea1
FT
260static int ixgbe_msi_enable = 1;
261TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
262
9407f759
FT
263/*
264 * MSIX should be the default for best performance,
265 * but this allows it to be forced off for testing.
266 */
267static int ixgbe_enable_msix = 1;
268TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
269
270/*
271 * Header split: this causes the hardware to DMA
272 * the header into a separate mbuf from the payload,
273 * it can be a performance win in some workloads, but
274 * in others it actually hurts, its off by default.
275 */
276static int ixgbe_header_split = FALSE;
277TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
278
279/*
280 * Number of Queues, can be set to 0,
281 * it then autoconfigures based on the
282 * number of cpus with a max of 8. This
283 * can be overriden manually here.
284 */
285static int ixgbe_num_queues = 0;
286TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
287
288/*
289** Number of TX descriptors per ring,
290** setting higher than RX as this seems
291** the better performing choice.
292*/
293static int ixgbe_txd = PERFORM_TXD;
294TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
295
296/* Number of RX descriptors per ring */
297static int ixgbe_rxd = PERFORM_RXD;
298TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
299
300/* Keep running tab on them for sanity check */
301static int ixgbe_total_ports;
302
303#ifdef IXGBE_FDIR
304/*
305** For Flow Director: this is the
306** number of TX packets we sample
307** for the filter pool, this means
308** every 20th packet will be probed.
309**
310** This feature can be disabled by
311** setting this to 0.
312*/
313static int atr_sample_rate = 20;
314/*
315** Flow Director actually 'steals'
316** part of the packet buffer as its
317** filter pool, this variable controls
318** how much it uses:
319** 0 = 64K, 1 = 128K, 2 = 256K
320*/
321static int fdir_pballoc = 1;
322#endif
323
324#ifdef DEV_NETMAP
325/*
326 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
327 * be a reference on how to implement netmap support in a driver.
328 * Additional comments are in ixgbe_netmap.h .
329 *
330 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
331 * that extend the standard driver.
332 */
333#include <dev/netmap/ixgbe_netmap.h>
334#endif /* DEV_NETMAP */
335
336/*********************************************************************
337 * Device identification routine
338 *
339 * ixgbe_probe determines if the driver should be loaded on
340 * adapter based on PCI vendor/device id of the adapter.
341 *
342 * return BUS_PROBE_DEFAULT on success, positive on failure
343 *********************************************************************/
344
345static int
346ixgbe_probe(device_t dev)
347{
348 ixgbe_vendor_info_t *ent;
349
350 u16 pci_vendor_id = 0;
351 u16 pci_device_id = 0;
352 u16 pci_subvendor_id = 0;
353 u16 pci_subdevice_id = 0;
354 char adapter_name[256];
355
356 INIT_DEBUGOUT("ixgbe_probe: begin");
357
358 pci_vendor_id = pci_get_vendor(dev);
359 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
360 return (ENXIO);
361
362 pci_device_id = pci_get_device(dev);
363 pci_subvendor_id = pci_get_subvendor(dev);
364 pci_subdevice_id = pci_get_subdevice(dev);
365
366 ent = ixgbe_vendor_info_array;
367 while (ent->vendor_id != 0) {
368 if ((pci_vendor_id == ent->vendor_id) &&
369 (pci_device_id == ent->device_id) &&
370
371 ((pci_subvendor_id == ent->subvendor_id) ||
372 (ent->subvendor_id == 0)) &&
373
374 ((pci_subdevice_id == ent->subdevice_id) ||
375 (ent->subdevice_id == 0))) {
376 ksprintf(adapter_name, "%s, Version - %s",
377 ixgbe_strings[ent->index],
378 ixgbe_driver_version);
379 device_set_desc_copy(dev, adapter_name);
380 ++ixgbe_total_ports;
381 return (BUS_PROBE_DEFAULT);
382 }
383 ent++;
384 }
385 return (ENXIO);
386}
387
388/*********************************************************************
389 * Device initialization routine
390 *
391 * The attach entry point is called when the driver is being loaded.
392 * This routine identifies the type of hardware, allocates all resources
393 * and initializes the hardware.
394 *
395 * return 0 on success, positive on failure
396 *********************************************************************/
397
398static int
399ixgbe_attach(device_t dev)
400{
401 struct adapter *adapter;
402 struct ixgbe_hw *hw;
403 int error = 0;
404 u16 csum;
405 u32 ctrl_ext;
406
407 INIT_DEBUGOUT("ixgbe_attach: begin");
408
409 if (resource_disabled("ixgbe", device_get_unit(dev))) {
410 device_printf(dev, "Disabled by device hint\n");
411 return (ENXIO);
412 }
413
414 /* Allocate, clear, and link in our adapter structure */
415 adapter = device_get_softc(dev);
416 adapter->dev = adapter->osdep.dev = dev;
417 hw = &adapter->hw;
418
419 /* Core Lock Init*/
420 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
9407f759
FT
421
422 /* SYSCTL APIs */
423
424 sysctl_ctx_init(&adapter->sysctl_ctx);
425 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
426 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
427 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
428 if (adapter->sysctl_tree == NULL) {
429 device_printf(adapter->dev, "can't add sysctl node\n");
430 return (EINVAL);
431 }
432 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
433 SYSCTL_CHILDREN(adapter->sysctl_tree),
434 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
435 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
436
437 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
438 SYSCTL_CHILDREN(adapter->sysctl_tree),
439 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
440 &ixgbe_enable_aim, 1, "Interrupt Moderation");
441
442 /*
443 ** Allow a kind of speed control by forcing the autoneg
444 ** advertised speed list to only a certain value, this
445 ** supports 1G on 82599 devices, and 100Mb on x540.
446 */
447 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
448 SYSCTL_CHILDREN(adapter->sysctl_tree),
449 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
450 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
451
452 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
453 SYSCTL_CHILDREN(adapter->sysctl_tree),
454 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
455 0, ixgbe_set_thermal_test, "I", "Thermal Test");
456
457 /* Set up the timer callout */
9407f759
FT
458 callout_init(&adapter->timer);
459
460 /* Determine hardware revision */
461 ixgbe_identify_hardware(adapter);
462
463 /* Do base PCI setup - map BAR0 */
464 if (ixgbe_allocate_pci_resources(adapter)) {
465 device_printf(dev, "Allocation of PCI resources failed\n");
466 error = ENXIO;
467 goto err_out;
468 }
469
470 /* Do descriptor calc and sanity checks */
471 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
472 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
473 device_printf(dev, "TXD config issue, using default!\n");
474 adapter->num_tx_desc = DEFAULT_TXD;
475 } else
476 adapter->num_tx_desc = ixgbe_txd;
477
478 /*
479 ** With many RX rings it is easy to exceed the
480 ** system mbuf allocation. Tuning nmbclusters
481 ** can alleviate this.
482 */
483 if (nmbclusters > 0 ) {
484 int s;
485 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
486 if (s > nmbclusters) {
487 device_printf(dev, "RX Descriptors exceed "
488 "system mbuf max, using default instead!\n");
489 ixgbe_rxd = DEFAULT_RXD;
490 }
491 }
492
493 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
494 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
495 device_printf(dev, "RXD config issue, using default!\n");
496 adapter->num_rx_desc = DEFAULT_RXD;
497 } else
498 adapter->num_rx_desc = ixgbe_rxd;
499
500 /* Allocate our TX/RX Queues */
501 if (ixgbe_allocate_queues(adapter)) {
502 error = ENOMEM;
503 goto err_out;
504 }
505
506 /* Allocate multicast array memory. */
507 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
508 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
509 if (adapter->mta == NULL) {
510 device_printf(dev, "Can not allocate multicast setup array\n");
511 error = ENOMEM;
512 goto err_late;
513 }
514
515 /* Initialize the shared code */
516 error = ixgbe_init_shared_code(hw);
517 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
518 /*
519 ** No optics in this port, set up
520 ** so the timer routine will probe
521 ** for later insertion.
522 */
523 adapter->sfp_probe = TRUE;
524 error = 0;
525 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
526 device_printf(dev,"Unsupported SFP+ module detected!\n");
527 error = EIO;
528 goto err_late;
529 } else if (error) {
530 device_printf(dev,"Unable to initialize the shared code\n");
531 error = EIO;
532 goto err_late;
533 }
534
535 /* Make sure we have a good EEPROM before we read from it */
536 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
537 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
538 error = EIO;
539 goto err_late;
540 }
541
9407f759 542 error = ixgbe_init_hw(hw);
ce451236
FT
543 switch (error) {
544 case IXGBE_ERR_EEPROM_VERSION:
9407f759
FT
545 device_printf(dev, "This device is a pre-production adapter/"
546 "LOM. Please be aware there may be issues associated "
547 "with your hardware.\n If you are experiencing problems "
548 "please contact your Intel or hardware representative "
549 "who provided you with this hardware.\n");
ce451236
FT
550 break;
551 case IXGBE_ERR_SFP_NOT_SUPPORTED:
9407f759 552 device_printf(dev,"Unsupported SFP+ Module\n");
9407f759
FT
553 error = EIO;
554 device_printf(dev,"Hardware Initialization Failure\n");
555 goto err_late;
ce451236
FT
556 case IXGBE_ERR_SFP_NOT_PRESENT:
557 device_printf(dev,"No SFP+ Module found\n");
558 /* falls thru */
559 default:
560 break;
9407f759
FT
561 }
562
563 /* Detect and set physical type */
564 ixgbe_setup_optics(adapter);
565
566 if ((adapter->msix > 1) && (ixgbe_enable_msix))
567 error = ixgbe_allocate_msix(adapter);
568 else
569 error = ixgbe_allocate_legacy(adapter);
570 if (error)
571 goto err_late;
572
573 /* Setup OS specific network interface */
574 if (ixgbe_setup_interface(dev, adapter) != 0)
575 goto err_late;
576
577 /* Sysctl for limiting the amount of work done in the taskqueue */
578 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
579 "max number of rx packets to process", &adapter->rx_process_limit,
580 ixgbe_rx_process_limit);
581
582 /* Initialize statistics */
583 ixgbe_update_stats_counters(adapter);
584
585 /* Register for VLAN events */
586 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
587 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
588 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
589 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
590
591 /* Print PCIE bus type/speed/width info */
592 ixgbe_get_bus_info(hw);
593 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
594 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
595 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
596 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
597 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
598 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
599 ("Unknown"));
600
601 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
602 (hw->bus.speed == ixgbe_bus_speed_2500)) {
603 device_printf(dev, "PCI-Express bandwidth available"
604 " for this card\n is not sufficient for"
605 " optimal performance.\n");
606 device_printf(dev, "For optimal performance a x8 "
607 "PCIE, or x4 PCIE 2 slot is required.\n");
608 }
609
610 /* let hardware know driver is loaded */
611 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
612 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
613 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
614
615 ixgbe_add_hw_stats(adapter);
616
617#ifdef DEV_NETMAP
618 ixgbe_netmap_attach(adapter);
619#endif /* DEV_NETMAP */
620 INIT_DEBUGOUT("ixgbe_attach: end");
621 return (0);
622err_late:
623 ixgbe_free_transmit_structures(adapter);
624 ixgbe_free_receive_structures(adapter);
625err_out:
626 if (adapter->ifp != NULL)
627 if_free(adapter->ifp);
628 ixgbe_free_pci_resources(adapter);
629 kfree(adapter->mta, M_DEVBUF);
630 return (error);
631
632}
633
634/*********************************************************************
635 * Device removal routine
636 *
637 * The detach entry point is called when the driver is being removed.
638 * This routine stops the adapter and deallocates all the resources
639 * that were allocated for driver operation.
640 *
641 * return 0 on success, positive on failure
642 *********************************************************************/
643
644static int
645ixgbe_detach(device_t dev)
646{
647 struct adapter *adapter = device_get_softc(dev);
648 struct ix_queue *que = adapter->queues;
649 u32 ctrl_ext;
650
651 INIT_DEBUGOUT("ixgbe_detach: begin");
652
653#ifdef NET_VLAN
654 /* Make sure VLANS are not using driver */
655 if (adapter->ifp->if_vlantrunk != NULL) {
656 device_printf(dev,"Vlan in use, detach first\n");
657 return (EBUSY);
658 }
659#endif
660
661 IXGBE_CORE_LOCK(adapter);
662 ixgbe_stop(adapter);
663 IXGBE_CORE_UNLOCK(adapter);
664
665 for (int i = 0; i < adapter->num_queues; i++, que++) {
666 if (que->tq) {
667 taskqueue_drain(que->tq, &que->que_task);
668 taskqueue_free(que->tq);
669 }
670 }
671
672 /* Drain the Link queue */
673 if (adapter->tq) {
674 taskqueue_drain(adapter->tq, &adapter->link_task);
675 taskqueue_drain(adapter->tq, &adapter->mod_task);
676 taskqueue_drain(adapter->tq, &adapter->msf_task);
677#ifdef IXGBE_FDIR
678 taskqueue_drain(adapter->tq, &adapter->fdir_task);
679#endif
680 taskqueue_free(adapter->tq);
681 }
682
683 /* let hardware know driver is unloading */
684 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
685 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
686 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
687
688 /* Unregister VLAN events */
689#ifdef NET_VLAN
690 if (adapter->vlan_attach != NULL)
691 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
692 if (adapter->vlan_detach != NULL)
693 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
694#endif
695
696 ether_ifdetach(adapter->ifp);
697 callout_stop(&adapter->timer);
698 lockuninit(&adapter->core_lock);
699#ifdef DEV_NETMAP
700 netmap_detach(adapter->ifp);
701#endif /* DEV_NETMAP */
702 ixgbe_free_pci_resources(adapter);
703 bus_generic_detach(dev);
704 if_free(adapter->ifp);
705
706 ixgbe_free_transmit_structures(adapter);
707 ixgbe_free_receive_structures(adapter);
708 kfree(adapter->mta, M_DEVBUF);
709 sysctl_ctx_free(&adapter->sysctl_ctx);
710
9407f759
FT
711 IXGBE_CORE_LOCK_DESTROY(adapter);
712 return (0);
713}
714
715/*********************************************************************
716 *
717 * Shutdown entry point
718 *
719 **********************************************************************/
720
721static int
722ixgbe_shutdown(device_t dev)
723{
724 struct adapter *adapter = device_get_softc(dev);
725 IXGBE_CORE_LOCK(adapter);
726 ixgbe_stop(adapter);
727 IXGBE_CORE_UNLOCK(adapter);
728 return (0);
729}
730
731
732/*********************************************************************
733 * Transmit entry point
734 *
735 * ixgbe_start is called by the stack to initiate a transmit.
736 * The driver will remain in this routine as long as there are
737 * packets to transmit and transmit resources are available.
738 * In case resources are not available stack is notified and
739 * the packet is requeued.
740 **********************************************************************/
741
742static void
743ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
744{
745 struct mbuf *m_head;
746 struct adapter *adapter = txr->adapter;
747
748 IXGBE_TX_LOCK_ASSERT(txr);
749
750 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
751 return;
752 if (!adapter->link_active)
753 return;
754
755 while (!ifq_is_empty(&ifp->if_snd)) {
756 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
757 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
758 break;
759 }
760
761 m_head = ifq_dequeue(&ifp->if_snd, NULL);
762 if (m_head == NULL)
763 break;
764
765 if (ixgbe_xmit(txr, &m_head)) {
766#if 0 /* XXX: prepend to an ALTQ queue ? */
767 if (m_head != NULL)
768 IF_PREPEND(&ifp->if_snd, m_head);
769#endif
770 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
771 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
772 break;
773 }
774 /* Send a copy of the frame to the BPF listener */
775 ETHER_BPF_MTAP(ifp, m_head);
776
777 /* Set watchdog on */
778 txr->watchdog_time = ticks;
779 txr->queue_status = IXGBE_QUEUE_WORKING;
780
781 }
782 return;
783}
784
785/*
786 * Legacy TX start - called by the stack, this
787 * always uses the first tx ring, and should
788 * not be used with multiqueue tx enabled.
789 */
790static void
791ixgbe_start(struct ifnet *ifp)
792{
793 struct adapter *adapter = ifp->if_softc;
794 struct tx_ring *txr = adapter->tx_rings;
795
796 if (ifp->if_flags & IFF_RUNNING) {
797 IXGBE_TX_LOCK(txr);
798 ixgbe_start_locked(txr, ifp);
799 IXGBE_TX_UNLOCK(txr);
800 }
801 return;
802}
803
804#if 0 /* __FreeBSD_version >= 800000 */
805/*
806** Multiqueue Transmit driver
807**
808*/
809static int
810ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
811{
812 struct adapter *adapter = ifp->if_softc;
813 struct ix_queue *que;
814 struct tx_ring *txr;
815 int i = 0, err = 0;
816
817 /* Which queue to use */
818 if ((m->m_flags & M_FLOWID) != 0)
819 i = m->m_pkthdr.flowid % adapter->num_queues;
820 else
821 i = curcpu % adapter->num_queues;
822
823 txr = &adapter->tx_rings[i];
824 que = &adapter->queues[i];
825
826 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
827 IXGBE_TX_TRYLOCK(txr)) {
828 err = ixgbe_mq_start_locked(ifp, txr, m);
829 IXGBE_TX_UNLOCK(txr);
830 } else {
831 err = drbr_enqueue(ifp, txr->br, m);
832 taskqueue_enqueue(que->tq, &que->que_task);
833 }
834
835 return (err);
836}
837
838static int
839ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
840{
841 struct adapter *adapter = txr->adapter;
842 struct mbuf *next;
843 int enqueued, err = 0;
844
845 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
846 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
847 adapter->link_active == 0) {
848 if (m != NULL)
849 err = drbr_enqueue(ifp, txr->br, m);
850 return (err);
851 }
852
853 enqueued = 0;
854 if (m == NULL) {
855 next = drbr_dequeue(ifp, txr->br);
856 } else if (drbr_needs_enqueue(ifp, txr->br)) {
857 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
858 return (err);
859 next = drbr_dequeue(ifp, txr->br);
860 } else
861 next = m;
862
863 /* Process the queue */
864 while (next != NULL) {
865 if ((err = ixgbe_xmit(txr, &next)) != 0) {
866 if (next != NULL)
867 err = drbr_enqueue(ifp, txr->br, next);
868 break;
869 }
870 enqueued++;
871 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
872 /* Send a copy of the frame to the BPF listener */
873 ETHER_BPF_MTAP(ifp, next);
874 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
875 break;
876 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
877 ixgbe_txeof(txr);
878 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
879 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
880 break;
881 }
882 next = drbr_dequeue(ifp, txr->br);
883 }
884
885 if (enqueued > 0) {
886 /* Set watchdog on */
887 txr->queue_status |= IXGBE_QUEUE_WORKING;
888 txr->watchdog_time = ticks;
889 }
890
891 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
892 ixgbe_txeof(txr);
893
894 return (err);
895}
896
897/*
898** Flush all ring buffers
899*/
900static void
901ixgbe_qflush(struct ifnet *ifp)
902{
903 struct adapter *adapter = ifp->if_softc;
904 struct tx_ring *txr = adapter->tx_rings;
905 struct mbuf *m;
906
907 for (int i = 0; i < adapter->num_queues; i++, txr++) {
908 IXGBE_TX_LOCK(txr);
909 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
910 m_freem(m);
911 IXGBE_TX_UNLOCK(txr);
912 }
913 if_qflush(ifp);
914}
915#endif /* __FreeBSD_version >= 800000 */
916
917/*********************************************************************
918 * Ioctl entry point
919 *
920 * ixgbe_ioctl is called when the user wants to configure the
921 * interface.
922 *
923 * return 0 on success, positive on failure
924 **********************************************************************/
925
926static int
927ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
928{
929 struct adapter *adapter = ifp->if_softc;
930 struct ifreq *ifr = (struct ifreq *) data;
931#if defined(INET) || defined(INET6)
932 struct ifaddr *ifa = (struct ifaddr *)data;
933 bool avoid_reset = FALSE;
934#endif
935 int error = 0;
936
937 switch (command) {
938
939 case SIOCSIFADDR:
940#ifdef INET
941 if (ifa->ifa_addr->sa_family == AF_INET)
942 avoid_reset = TRUE;
943#endif
944#ifdef INET6
945 if (ifa->ifa_addr->sa_family == AF_INET6)
946 avoid_reset = TRUE;
947#endif
948#if defined(INET) || defined(INET6)
949 /*
950 ** Calling init results in link renegotiation,
951 ** so we avoid doing it when possible.
952 */
953 if (avoid_reset) {
954 ifp->if_flags |= IFF_UP;
955 if (!(ifp->if_flags & IFF_RUNNING))
956 ixgbe_init(adapter);
957 if (!(ifp->if_flags & IFF_NOARP))
958 arp_ifinit(ifp, ifa);
959 } else
960 error = ether_ioctl(ifp, command, data);
961#endif
962 break;
963 case SIOCSIFMTU:
964 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
965 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
966 error = EINVAL;
967 } else {
968 IXGBE_CORE_LOCK(adapter);
969 ifp->if_mtu = ifr->ifr_mtu;
970 adapter->max_frame_size =
971 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
972 ixgbe_init_locked(adapter);
973 IXGBE_CORE_UNLOCK(adapter);
974 }
975 break;
976 case SIOCSIFFLAGS:
977 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
978 IXGBE_CORE_LOCK(adapter);
979 if (ifp->if_flags & IFF_UP) {
980 if ((ifp->if_flags & IFF_RUNNING)) {
981 if ((ifp->if_flags ^ adapter->if_flags) &
982 (IFF_PROMISC | IFF_ALLMULTI)) {
983 ixgbe_set_promisc(adapter);
984 }
985 } else
986 ixgbe_init_locked(adapter);
987 } else
988 if (ifp->if_flags & IFF_RUNNING)
989 ixgbe_stop(adapter);
990 adapter->if_flags = ifp->if_flags;
991 IXGBE_CORE_UNLOCK(adapter);
992 break;
993 case SIOCADDMULTI:
994 case SIOCDELMULTI:
995 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
996 if (ifp->if_flags & IFF_RUNNING) {
997 IXGBE_CORE_LOCK(adapter);
998 ixgbe_disable_intr(adapter);
999 ixgbe_set_multi(adapter);
1000 ixgbe_enable_intr(adapter);
1001 IXGBE_CORE_UNLOCK(adapter);
1002 }
1003 break;
1004 case SIOCSIFMEDIA:
1005 case SIOCGIFMEDIA:
1006 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1007 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1008 break;
1009 case SIOCSIFCAP:
1010 {
1011 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1012 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1013 if (mask & IFCAP_HWCSUM)
1014 ifp->if_capenable ^= IFCAP_HWCSUM;
1015#if 0 /* NET_TSO */
1016 if (mask & IFCAP_TSO4)
1017 ifp->if_capenable ^= IFCAP_TSO4;
1018 if (mask & IFCAP_TSO6)
1019 ifp->if_capenable ^= IFCAP_TSO6;
1020#endif
1021#if 0 /* NET_LRO */
1022 if (mask & IFCAP_LRO)
1023 ifp->if_capenable ^= IFCAP_LRO;
1024#endif
1025 if (mask & IFCAP_VLAN_HWTAGGING)
1026 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1027 if (mask & IFCAP_VLAN_HWFILTER)
1028 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1029#if 0 /* NET_TSO */
1030 if (mask & IFCAP_VLAN_HWTSO)
1031 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1032#endif
1033 if (ifp->if_flags & IFF_RUNNING) {
1034 IXGBE_CORE_LOCK(adapter);
1035 ixgbe_init_locked(adapter);
1036 IXGBE_CORE_UNLOCK(adapter);
1037 }
1038#if 0
1039 VLAN_CAPABILITIES(ifp);
1040#endif
1041 break;
1042 }
1043
1044 default:
1045 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1046 error = ether_ioctl(ifp, command, data);
1047 break;
1048 }
1049
1050 return (error);
1051}
1052
1053/*********************************************************************
1054 * Init entry point
1055 *
1056 * This routine is used in two ways. It is used by the stack as
1057 * init entry point in network interface structure. It is also used
1058 * by the driver as a hw/sw initialization routine to get to a
1059 * consistent state.
1060 *
1061 * return 0 on success, positive on failure
1062 **********************************************************************/
1063#define IXGBE_MHADD_MFS_SHIFT 16
1064
1065static void
1066ixgbe_init_locked(struct adapter *adapter)
1067{
1068 struct ifnet *ifp = adapter->ifp;
1069 device_t dev = adapter->dev;
1070 struct ixgbe_hw *hw = &adapter->hw;
1071 u32 k, txdctl, mhadd, gpie;
1072 u32 rxdctl, rxctrl;
1073
1074 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1075 INIT_DEBUGOUT("ixgbe_init: begin");
1076 hw->adapter_stopped = FALSE;
1077 ixgbe_stop_adapter(hw);
1078 callout_stop(&adapter->timer);
1079
1080 /* reprogram the RAR[0] in case user changed it. */
1081 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1082
1083 /* Get the latest mac address, User can use a LAA */
1084 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1085 IXGBE_ETH_LENGTH_OF_ADDRESS);
1086 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1087 hw->addr_ctrl.rar_used_count = 1;
1088
1089 /* Set the various hardware offload abilities */
1090 ifp->if_hwassist = 0;
1091#if 0 /* NET_TSO */
1092 if (ifp->if_capenable & IFCAP_TSO)
1093 ifp->if_hwassist |= CSUM_TSO;
1094#endif
1095 if (ifp->if_capenable & IFCAP_TXCSUM) {
1096 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1097#if 0
1098 if (hw->mac.type != ixgbe_mac_82598EB)
1099 ifp->if_hwassist |= CSUM_SCTP;
1100#endif
1101 }
1102
1103 /* Prepare transmit descriptors and buffers */
1104 if (ixgbe_setup_transmit_structures(adapter)) {
1105 device_printf(dev,"Could not setup transmit structures\n");
1106 ixgbe_stop(adapter);
1107 return;
1108 }
1109
1110 ixgbe_init_hw(hw);
1111 ixgbe_initialize_transmit_units(adapter);
1112
1113 /* Setup Multicast table */
1114 ixgbe_set_multi(adapter);
1115
1116 /*
1117 ** Determine the correct mbuf pool
1118 ** for doing jumbo/headersplit
1119 */
1120 if (adapter->max_frame_size <= 2048)
1121 adapter->rx_mbuf_sz = MCLBYTES;
1122 else if (adapter->max_frame_size <= 4096)
1123 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1124 else if (adapter->max_frame_size <= 9216)
1125 adapter->rx_mbuf_sz = MJUM9BYTES;
1126 else
1127 adapter->rx_mbuf_sz = MJUM16BYTES;
1128
1129 /* Prepare receive descriptors and buffers */
1130 if (ixgbe_setup_receive_structures(adapter)) {
1131 device_printf(dev,"Could not setup receive structures\n");
1132 ixgbe_stop(adapter);
1133 return;
1134 }
1135
1136 /* Configure RX settings */
1137 ixgbe_initialize_receive_units(adapter);
1138
1139 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1140
1141 /* Enable Fan Failure Interrupt */
1142 gpie |= IXGBE_SDP1_GPIEN;
1143
1144 /* Add for Module detection */
1145 if (hw->mac.type == ixgbe_mac_82599EB)
1146 gpie |= IXGBE_SDP2_GPIEN;
1147
1148 /* Thermal Failure Detection */
1149 if (hw->mac.type == ixgbe_mac_X540)
1150 gpie |= IXGBE_SDP0_GPIEN;
1151
1152 if (adapter->msix > 1) {
1153 /* Enable Enhanced MSIX mode */
1154 gpie |= IXGBE_GPIE_MSIX_MODE;
1155 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1156 IXGBE_GPIE_OCD;
1157 }
1158 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1159
1160 /* Set MTU size */
1161 if (ifp->if_mtu > ETHERMTU) {
1162 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1163 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1164 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1165 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1166 }
1167
1168 /* Now enable all the queues */
1169
1170 for (int i = 0; i < adapter->num_queues; i++) {
1171 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1172 txdctl |= IXGBE_TXDCTL_ENABLE;
1173 /* Set WTHRESH to 8, burst writeback */
1174 txdctl |= (8 << 16);
1175 /*
1176 * When the internal queue falls below PTHRESH (32),
1177 * start prefetching as long as there are at least
1178 * HTHRESH (1) buffers ready. The values are taken
1179 * from the Intel linux driver 3.8.21.
1180 * Prefetching enables tx line rate even with 1 queue.
1181 */
1182 txdctl |= (32 << 0) | (1 << 8);
1183 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1184 }
1185
1186 for (int i = 0; i < adapter->num_queues; i++) {
1187 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1188 if (hw->mac.type == ixgbe_mac_82598EB) {
1189 /*
1190 ** PTHRESH = 21
1191 ** HTHRESH = 4
1192 ** WTHRESH = 8
1193 */
1194 rxdctl &= ~0x3FFFFF;
1195 rxdctl |= 0x080420;
1196 }
1197 rxdctl |= IXGBE_RXDCTL_ENABLE;
1198 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1199 for (k = 0; k < 10; k++) {
1200 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1201 IXGBE_RXDCTL_ENABLE)
1202 break;
1203 else
1204 msec_delay(1);
1205 }
1206 wmb();
1207#ifdef DEV_NETMAP
1208 /*
1209 * In netmap mode, we must preserve the buffers made
1210 * available to userspace before the if_init()
1211 * (this is true by default on the TX side, because
1212 * init makes all buffers available to userspace).
1213 *
1214 * netmap_reset() and the device specific routines
1215 * (e.g. ixgbe_setup_receive_rings()) map these
1216 * buffers at the end of the NIC ring, so here we
1217 * must set the RDT (tail) register to make sure
1218 * they are not overwritten.
1219 *
1220 * In this driver the NIC ring starts at RDH = 0,
1221 * RDT points to the last slot available for reception (?),
1222 * so RDT = num_rx_desc - 1 means the whole ring is available.
1223 */
1224 if (ifp->if_capenable & IFCAP_NETMAP) {
1225 struct netmap_adapter *na = NA(adapter->ifp);
1226 struct netmap_kring *kring = &na->rx_rings[i];
1227 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1228
1229 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1230 } else
1231#endif /* DEV_NETMAP */
1232 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1233 }
1234
1235 /* Set up VLAN support and filter */
1236 ixgbe_setup_vlan_hw_support(adapter);
1237
1238 /* Enable Receive engine */
1239 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1240 if (hw->mac.type == ixgbe_mac_82598EB)
1241 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1242 rxctrl |= IXGBE_RXCTRL_RXEN;
1243 ixgbe_enable_rx_dma(hw, rxctrl);
1244
1245 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1246
1247 /* Set up MSI/X routing */
1248 if (ixgbe_enable_msix) {
1249 ixgbe_configure_ivars(adapter);
1250 /* Set up auto-mask */
1251 if (hw->mac.type == ixgbe_mac_82598EB)
1252 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1253 else {
1254 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1255 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1256 }
1257 } else { /* Simple settings for Legacy/MSI */
1258 ixgbe_set_ivar(adapter, 0, 0, 0);
1259 ixgbe_set_ivar(adapter, 0, 0, 1);
1260 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1261 }
1262
1263#ifdef IXGBE_FDIR
1264 /* Init Flow director */
1265 if (hw->mac.type != ixgbe_mac_82598EB) {
ce451236 1266 u32 hdrm = 32 << fdir_pballoc;
9407f759
FT
1267
1268 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1269 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1270 }
1271#endif
1272
1273 /*
1274 ** Check on any SFP devices that
1275 ** need to be kick-started
1276 */
1277 if (hw->phy.type == ixgbe_phy_none) {
1278 int err = hw->phy.ops.identify(hw);
1279 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1280 device_printf(dev,
1281 "Unsupported SFP+ module type was detected.\n");
1282 return;
1283 }
1284 }
1285
1286 /* Set moderation on the Link interrupt */
1287 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1288
1289 /* Config/Enable Link */
1290 ixgbe_config_link(adapter);
1291
ce451236
FT
1292 /* Hardware Packet Buffer & Flow Control setup */
1293 {
1294 u32 rxpb, frame, size, tmp;
1295
1296 frame = adapter->max_frame_size;
1297
1298 /* Calculate High Water */
1299 if (hw->mac.type == ixgbe_mac_X540)
1300 tmp = IXGBE_DV_X540(frame, frame);
1301 else
1302 tmp = IXGBE_DV(frame, frame);
1303 size = IXGBE_BT2KB(tmp);
1304 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1305 hw->fc.high_water[0] = rxpb - size;
1306
1307 /* Now calculate Low Water */
1308 if (hw->mac.type == ixgbe_mac_X540)
1309 tmp = IXGBE_LOW_DV_X540(frame);
1310 else
1311 tmp = IXGBE_LOW_DV(frame);
1312 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1313
1314 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1315 hw->fc.pause_time = IXGBE_FC_PAUSE;
1316 hw->fc.send_xon = TRUE;
1317 }
1318 /* Initialize the FC settings */
1319 ixgbe_start_hw(hw);
1320
9407f759
FT
1321 /* And now turn on interrupts */
1322 ixgbe_enable_intr(adapter);
1323
1324 /* Now inform the stack we're ready */
1325 ifp->if_flags |= IFF_RUNNING;
1326 ifp->if_flags &= ~IFF_OACTIVE;
1327
1328 return;
1329}
1330
1331static void
1332ixgbe_init(void *arg)
1333{
1334 struct adapter *adapter = arg;
1335
1336 IXGBE_CORE_LOCK(adapter);
1337 ixgbe_init_locked(adapter);
1338 IXGBE_CORE_UNLOCK(adapter);
1339 return;
1340}
1341
1342
1343/*
1344**
1345** MSIX Interrupt Handlers and Tasklets
1346**
1347*/
1348
1349static inline void
1350ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1351{
1352 struct ixgbe_hw *hw = &adapter->hw;
1353 u64 queue = (u64)(1 << vector);
1354 u32 mask;
1355
1356 if (hw->mac.type == ixgbe_mac_82598EB) {
1357 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1358 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1359 } else {
1360 mask = (queue & 0xFFFFFFFF);
1361 if (mask)
1362 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1363 mask = (queue >> 32);
1364 if (mask)
1365 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1366 }
1367}
1368
1369static inline void
1370ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1371{
1372 struct ixgbe_hw *hw = &adapter->hw;
1373 u64 queue = (u64)(1 << vector);
1374 u32 mask;
1375
1376 if (hw->mac.type == ixgbe_mac_82598EB) {
1377 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1379 } else {
1380 mask = (queue & 0xFFFFFFFF);
1381 if (mask)
1382 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1383 mask = (queue >> 32);
1384 if (mask)
1385 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1386 }
1387}
1388
1389static inline void
1390ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1391{
1392 u32 mask;
1393
1394 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1395 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1396 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1397 } else {
1398 mask = (queues & 0xFFFFFFFF);
1399 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1400 mask = (queues >> 32);
1401 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1402 }
1403}
1404
1405
1406static void
1407ixgbe_handle_que(void *context, int pending)
1408{
1409 struct ix_queue *que = context;
1410 struct adapter *adapter = que->adapter;
1411 struct tx_ring *txr = que->txr;
1412 struct ifnet *ifp = adapter->ifp;
1413 bool more;
1414
1415 if (ifp->if_flags & IFF_RUNNING) {
1416 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1417 IXGBE_TX_LOCK(txr);
1418 ixgbe_txeof(txr);
1419#if 0 /*__FreeBSD_version >= 800000*/
1420 if (!drbr_empty(ifp, txr->br))
1421 ixgbe_mq_start_locked(ifp, txr, NULL);
1422#else
1423 if (!ifq_is_empty(&ifp->if_snd))
1424 ixgbe_start_locked(txr, ifp);
1425#endif
1426 IXGBE_TX_UNLOCK(txr);
1427 if (more) {
1428 taskqueue_enqueue(que->tq, &que->que_task);
1429 return;
1430 }
1431 }
1432
1433 /* Reenable this interrupt */
1434 ixgbe_enable_queue(adapter, que->msix);
1435 return;
1436}
1437
1438
1439/*********************************************************************
1440 *
1441 * Legacy Interrupt Service routine
1442 *
1443 **********************************************************************/
1444
1445static void
1446ixgbe_legacy_irq(void *arg)
1447{
1448 struct ix_queue *que = arg;
1449 struct adapter *adapter = que->adapter;
1450 struct ixgbe_hw *hw = &adapter->hw;
1451 struct tx_ring *txr = adapter->tx_rings;
1452 bool more_tx, more_rx;
1453 u32 reg_eicr, loop = MAX_LOOP;
1454
1455
1456 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1457
1458 ++que->irqs;
1459 if (reg_eicr == 0) {
1460 ixgbe_enable_intr(adapter);
1461 return;
1462 }
1463
1464 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1465
1466 IXGBE_TX_LOCK(txr);
1467 do {
1468 more_tx = ixgbe_txeof(txr);
1469 } while (loop-- && more_tx);
1470 IXGBE_TX_UNLOCK(txr);
1471
1472 if (more_rx || more_tx)
1473 taskqueue_enqueue(que->tq, &que->que_task);
1474
1475 /* Check for fan failure */
1476 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1477 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1478 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1479 "REPLACE IMMEDIATELY!!\n");
1480 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1481 }
1482
1483 /* Link status change */
1484 if (reg_eicr & IXGBE_EICR_LSC)
1485 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1486
1487 ixgbe_enable_intr(adapter);
1488 return;
1489}
1490
1491
1492/*********************************************************************
1493 *
1494 * MSIX Queue Interrupt Service routine
1495 *
1496 **********************************************************************/
1497void
1498ixgbe_msix_que(void *arg)
1499{
1500 struct ix_queue *que = arg;
1501 struct adapter *adapter = que->adapter;
1502 struct tx_ring *txr = que->txr;
1503 struct rx_ring *rxr = que->rxr;
1504 bool more_tx, more_rx;
1505 u32 newitr = 0;
1506
1507 ixgbe_disable_queue(adapter, que->msix);
1508 ++que->irqs;
1509
1510 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1511
1512 IXGBE_TX_LOCK(txr);
1513 more_tx = ixgbe_txeof(txr);
1514 /*
1515 ** Make certain that if the stack
1516 ** has anything queued the task gets
1517 ** scheduled to handle it.
1518 */
1519#if 0
1520#if __FreeBSD_version < 800000
1521 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1522#else
1523 if (!drbr_empty(adapter->ifp, txr->br))
1524#endif
1525#endif
1526 if (!ifq_is_empty(&adapter->ifp->if_snd))
1527 more_tx = 1;
1528 IXGBE_TX_UNLOCK(txr);
1529
1530 /* Do AIM now? */
1531
1532 if (ixgbe_enable_aim == FALSE)
1533 goto no_calc;
1534 /*
1535 ** Do Adaptive Interrupt Moderation:
1536 ** - Write out last calculated setting
1537 ** - Calculate based on average size over
1538 ** the last interval.
1539 */
1540 if (que->eitr_setting)
1541 IXGBE_WRITE_REG(&adapter->hw,
1542 IXGBE_EITR(que->msix), que->eitr_setting);
1543
1544 que->eitr_setting = 0;
1545
1546 /* Idle, do nothing */
1547 if ((txr->bytes == 0) && (rxr->bytes == 0))
1548 goto no_calc;
1549
1550 if ((txr->bytes) && (txr->packets))
1551 newitr = txr->bytes/txr->packets;
1552 if ((rxr->bytes) && (rxr->packets))
1553 newitr = max(newitr,
1554 (rxr->bytes / rxr->packets));
1555 newitr += 24; /* account for hardware frame, crc */
1556
1557 /* set an upper boundary */
1558 newitr = min(newitr, 3000);
1559
1560 /* Be nice to the mid range */
1561 if ((newitr > 300) && (newitr < 1200))
1562 newitr = (newitr / 3);
1563 else
1564 newitr = (newitr / 2);
1565
1566 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1567 newitr |= newitr << 16;
1568 else
1569 newitr |= IXGBE_EITR_CNT_WDIS;
1570
1571 /* save for next interrupt */
1572 que->eitr_setting = newitr;
1573
1574 /* Reset state */
1575 txr->bytes = 0;
1576 txr->packets = 0;
1577 rxr->bytes = 0;
1578 rxr->packets = 0;
1579
1580no_calc:
1581 if (more_tx || more_rx)
1582 taskqueue_enqueue(que->tq, &que->que_task);
1583 else /* Reenable this interrupt */
1584 ixgbe_enable_queue(adapter, que->msix);
1585 return;
1586}
1587
1588
1589static void
1590ixgbe_msix_link(void *arg)
1591{
1592 struct adapter *adapter = arg;
1593 struct ixgbe_hw *hw = &adapter->hw;
1594 u32 reg_eicr;
1595
1596 ++adapter->link_irq;
1597
1598 /* First get the cause */
1599 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1600 /* Clear interrupt with write */
1601 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1602
1603 /* Link status change */
1604 if (reg_eicr & IXGBE_EICR_LSC)
1605 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1606
1607 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1608#ifdef IXGBE_FDIR
1609 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1610 /* This is probably overkill :) */
1611 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1612 return;
ce451236
FT
1613 /* Disable the interrupt */
1614 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
9407f759
FT
1615 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1616 } else
1617#endif
1618 if (reg_eicr & IXGBE_EICR_ECC) {
1619 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1620 "Please Reboot!!\n");
1621 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1622 } else
1623
1624 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1625 /* Clear the interrupt */
1626 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1627 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1628 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1629 /* Clear the interrupt */
1630 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1631 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1632 }
1633 }
1634
1635 /* Check for fan failure */
1636 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1637 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1638 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1639 "REPLACE IMMEDIATELY!!\n");
1640 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1641 }
1642
1643 /* Check for over temp condition */
1644 if ((hw->mac.type == ixgbe_mac_X540) &&
1645 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1646 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1647 "PHY IS SHUT DOWN!!\n");
1648 device_printf(adapter->dev, "System shutdown required\n");
1649 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1650 }
1651
1652 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1653 return;
1654}
1655
1656/*********************************************************************
1657 *
1658 * Media Ioctl callback
1659 *
1660 * This routine is called whenever the user queries the status of
1661 * the interface using ifconfig.
1662 *
1663 **********************************************************************/
1664static void
1665ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1666{
1667 struct adapter *adapter = ifp->if_softc;
1668
1669 INIT_DEBUGOUT("ixgbe_media_status: begin");
1670 IXGBE_CORE_LOCK(adapter);
1671 ixgbe_update_link_status(adapter);
1672
1673 ifmr->ifm_status = IFM_AVALID;
1674 ifmr->ifm_active = IFM_ETHER;
1675
1676 if (!adapter->link_active) {
1677 IXGBE_CORE_UNLOCK(adapter);
1678 return;
1679 }
1680
1681 ifmr->ifm_status |= IFM_ACTIVE;
1682
1683 switch (adapter->link_speed) {
1684 case IXGBE_LINK_SPEED_100_FULL:
1685 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1686 break;
1687 case IXGBE_LINK_SPEED_1GB_FULL:
1688 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1689 break;
1690 case IXGBE_LINK_SPEED_10GB_FULL:
1691 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1692 break;
1693 }
1694
1695 IXGBE_CORE_UNLOCK(adapter);
1696
1697 return;
1698}
1699
1700/*********************************************************************
1701 *
1702 * Media Ioctl callback
1703 *
1704 * This routine is called when the user changes speed/duplex using
1705 * media/mediopt option with ifconfig.
1706 *
1707 **********************************************************************/
1708static int
1709ixgbe_media_change(struct ifnet * ifp)
1710{
1711 struct adapter *adapter = ifp->if_softc;
1712 struct ifmedia *ifm = &adapter->media;
1713
1714 INIT_DEBUGOUT("ixgbe_media_change: begin");
1715
1716 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1717 return (EINVAL);
1718
1719 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1720 case IFM_AUTO:
1721 adapter->hw.phy.autoneg_advertised =
1722 IXGBE_LINK_SPEED_100_FULL |
1723 IXGBE_LINK_SPEED_1GB_FULL |
1724 IXGBE_LINK_SPEED_10GB_FULL;
1725 break;
1726 default:
1727 device_printf(adapter->dev, "Only auto media type\n");
1728 return (EINVAL);
1729 }
1730
1731 return (0);
1732}
1733
1734/*********************************************************************
1735 *
1736 * This routine maps the mbufs to tx descriptors, allowing the
1737 * TX engine to transmit the packets.
1738 * - return 0 on success, positive on failure
1739 *
1740 **********************************************************************/
1741
1742static int
1743ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1744{
1745 struct adapter *adapter = txr->adapter;
1746 u32 olinfo_status = 0, cmd_type_len;
1747 u32 paylen = 0;
1748 int i, j, error, nsegs;
1749 int first, last = 0;
1750 struct mbuf *m_head;
1751 bus_dma_segment_t segs[1];
1752 bus_dmamap_t map;
1753 struct ixgbe_tx_buf *txbuf;
1754 union ixgbe_adv_tx_desc *txd = NULL;
1755
1756 m_head = *m_headp;
1757
1758 /* Basic descriptor defines */
1759 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1760 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1761
1762 if (m_head->m_flags & M_VLANTAG)
1763 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1764
1765 /*
1766 * Important to capture the first descriptor
1767 * used because it will contain the index of
1768 * the one we tell the hardware to report back
1769 */
1770 first = txr->next_avail_desc;
1771 txbuf = &txr->tx_buffers[first];
1772 map = txbuf->map;
1773
1774 /*
1775 * Map the packet for DMA.
1776 */
1777 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1778 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1779
1780 if (error == EFBIG) {
1781 struct mbuf *m;
1782
1783 m = m_defrag(*m_headp, MB_DONTWAIT);
1784 if (m == NULL) {
1785 adapter->mbuf_defrag_failed++;
1786 m_freem(*m_headp);
1787 *m_headp = NULL;
1788 return (ENOBUFS);
1789 }
1790 *m_headp = m;
1791
1792 /* Try it again */
1793 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1794 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1795
1796 if (error == ENOMEM) {
1797 adapter->no_tx_dma_setup++;
1798 return (error);
1799 } else if (error != 0) {
1800 adapter->no_tx_dma_setup++;
1801 m_freem(*m_headp);
1802 *m_headp = NULL;
1803 return (error);
1804 }
1805 } else if (error == ENOMEM) {
1806 adapter->no_tx_dma_setup++;
1807 return (error);
1808 } else if (error != 0) {
1809 adapter->no_tx_dma_setup++;
1810 m_freem(*m_headp);
1811 *m_headp = NULL;
1812 return (error);
1813 }
1814
1815 /* Make certain there are enough descriptors */
1816 if (nsegs > txr->tx_avail - 2) {
1817 txr->no_desc_avail++;
1818 error = ENOBUFS;
1819 goto xmit_fail;
1820 }
1821 m_head = *m_headp;
1822
1823 /*
1824 ** Set up the appropriate offload context
1825 ** this becomes the first descriptor of
1826 ** a packet.
1827 */
1828#if 0 /* NET_TSO */
1829 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1830 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1831 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1832 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1833 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1834 ++adapter->tso_tx;
1835 } else
1836 return (ENXIO);
1837 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1838#endif
1839 if (ixgbe_tx_ctx_setup(txr, m_head))
1840 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1841
1842#ifdef IXGBE_IEEE1588
1843 /* This is changing soon to an mtag detection */
1844 if (we detect this mbuf has a TSTAMP mtag)
1845 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1846#endif
1847
1848#ifdef IXGBE_FDIR
1849 /* Do the flow director magic */
1850 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1851 ++txr->atr_count;
1852 if (txr->atr_count >= atr_sample_rate) {
1853 ixgbe_atr(txr, m_head);
1854 txr->atr_count = 0;
1855 }
1856 }
1857#endif
1858 /* Record payload length */
1859 if (paylen == 0)
1860 olinfo_status |= m_head->m_pkthdr.len <<
1861 IXGBE_ADVTXD_PAYLEN_SHIFT;
1862
1863 i = txr->next_avail_desc;
1864 for (j = 0; j < nsegs; j++) {
1865 bus_size_t seglen;
1866 bus_addr_t segaddr;
1867
1868 txbuf = &txr->tx_buffers[i];
1869 txd = &txr->tx_base[i];
1870 seglen = segs[j].ds_len;
1871 segaddr = htole64(segs[j].ds_addr);
1872
1873 txd->read.buffer_addr = segaddr;
1874 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1875 cmd_type_len |seglen);
1876 txd->read.olinfo_status = htole32(olinfo_status);
1877 last = i; /* descriptor that will get completion IRQ */
1878
1879 if (++i == adapter->num_tx_desc)
1880 i = 0;
1881
1882 txbuf->m_head = NULL;
1883 txbuf->eop_index = -1;
1884 }
1885
1886 txd->read.cmd_type_len |=
1887 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1888 txr->tx_avail -= nsegs;
1889 txr->next_avail_desc = i;
1890
1891 txbuf->m_head = m_head;
1892 /* Swap the dma map between the first and last descriptor */
1893 txr->tx_buffers[first].map = txbuf->map;
1894 txbuf->map = map;
1895 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1896
1897 /* Set the index of the descriptor that will be marked done */
1898 txbuf = &txr->tx_buffers[first];
1899 txbuf->eop_index = last;
1900
1901 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1902 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1903 /*
1904 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1905 * hardware that this frame is available to transmit.
1906 */
1907 ++txr->total_packets;
1908 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1909
1910 return (0);
1911
1912xmit_fail:
1913 bus_dmamap_unload(txr->txtag, txbuf->map);
1914 return (error);
1915
1916}
1917
1918static void
1919ixgbe_set_promisc(struct adapter *adapter)
1920{
1921 u_int32_t reg_rctl;
1922 struct ifnet *ifp = adapter->ifp;
1923
1924 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1925 reg_rctl &= (~IXGBE_FCTRL_UPE);
1926 reg_rctl &= (~IXGBE_FCTRL_MPE);
1927 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1928
1929 if (ifp->if_flags & IFF_PROMISC) {
1930 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1931 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1932 } else if (ifp->if_flags & IFF_ALLMULTI) {
1933 reg_rctl |= IXGBE_FCTRL_MPE;
1934 reg_rctl &= ~IXGBE_FCTRL_UPE;
1935 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1936 }
1937 return;
1938}
1939
1940
1941/*********************************************************************
1942 * Multicast Update
1943 *
1944 * This routine is called whenever multicast address list is updated.
1945 *
1946 **********************************************************************/
1947#define IXGBE_RAR_ENTRIES 16
1948
1949static void
1950ixgbe_set_multi(struct adapter *adapter)
1951{
1952 u32 fctrl;
1953 u8 *mta;
1954 u8 *update_ptr;
1955 struct ifmultiaddr *ifma;
1956 int mcnt = 0;
1957 struct ifnet *ifp = adapter->ifp;
1958
1959 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1960
1961 mta = adapter->mta;
1962 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1963 MAX_NUM_MULTICAST_ADDRESSES);
1964
1965 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1966 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1967 if (ifp->if_flags & IFF_PROMISC)
1968 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1969 else if (ifp->if_flags & IFF_ALLMULTI) {
1970 fctrl |= IXGBE_FCTRL_MPE;
1971 fctrl &= ~IXGBE_FCTRL_UPE;
1972 } else
1973 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1974
1975 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1976
9407f759
FT
1977 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1978 if (ifma->ifma_addr->sa_family != AF_LINK)
1979 continue;
1980 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1981 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1982 IXGBE_ETH_LENGTH_OF_ADDRESS);
1983 mcnt++;
1984 }
9407f759
FT
1985
1986 update_ptr = mta;
1987 ixgbe_update_mc_addr_list(&adapter->hw,
1988 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1989
1990 return;
1991}
1992
1993/*
1994 * This is an iterator function now needed by the multicast
1995 * shared code. It simply feeds the shared code routine the
1996 * addresses in the array of ixgbe_set_multi() one by one.
1997 */
1998static u8 *
1999ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2000{
2001 u8 *addr = *update_ptr;
2002 u8 *newptr;
2003 *vmdq = 0;
2004
2005 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2006 *update_ptr = newptr;
2007 return addr;
2008}
2009
2010
2011/*********************************************************************
2012 * Timer routine
2013 *
2014 * This routine checks for link status,updates statistics,
2015 * and runs the watchdog check.
2016 *
2017 **********************************************************************/
2018
2019static void
2020ixgbe_local_timer(void *arg)
2021{
2022 struct adapter *adapter = arg;
2023 device_t dev = adapter->dev;
2024 struct ifnet *ifp = adapter->ifp;
2025 struct ix_queue *que = adapter->queues;
2026 struct tx_ring *txr = adapter->tx_rings;
2027 int hung, busy, paused;
2028
fe609108 2029 IXGBE_CORE_LOCK(adapter);
9407f759
FT
2030 hung = busy = paused = 0;
2031
2032 /* Check for pluggable optics */
2033 if (adapter->sfp_probe)
2034 if (!ixgbe_sfp_probe(adapter))
2035 goto out; /* Nothing to do */
2036
2037 ixgbe_update_link_status(adapter);
2038 ixgbe_update_stats_counters(adapter);
2039
2040 /*
2041 * If the interface has been paused
2042 * then don't do the watchdog check
2043 */
2044 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2045 paused = 1;
2046
2047 /*
2048 ** Check the TX queues status
2049 ** - central locked handling of OACTIVE
2050 ** - watchdog only if all queues show hung
2051 */
2052 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2053 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2054 (paused == 0))
2055 ++hung;
2056 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2057 ++busy;
2058 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2059 taskqueue_enqueue(que->tq, &que->que_task);
2060 }
2061 /* Only truely watchdog if all queues show hung */
2062 if (hung == adapter->num_queues)
2063 goto watchdog;
2064 /* Only turn off the stack flow when ALL are depleted */
2065 if (busy == adapter->num_queues)
2066 ifp->if_flags |= IFF_OACTIVE;
2067 else if ((ifp->if_flags & IFF_OACTIVE) &&
2068 (busy < adapter->num_queues))
2069 ifp->if_flags &= ~IFF_OACTIVE;
2070
2071out:
2072 ixgbe_rearm_queues(adapter, adapter->que_mask);
2073 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
fe609108 2074 IXGBE_CORE_UNLOCK(adapter);
9407f759
FT
2075 return;
2076
2077watchdog:
2078 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2079 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2080 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2081 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2082 device_printf(dev,"TX(%d) desc avail = %d,"
2083 "Next TX to Clean = %d\n",
2084 txr->me, txr->tx_avail, txr->next_to_clean);
2085 adapter->ifp->if_flags &= ~IFF_RUNNING;
2086 adapter->watchdog_events++;
2087 ixgbe_init_locked(adapter);
2088
fe609108 2089 IXGBE_CORE_UNLOCK(adapter);
9407f759
FT
2090}
2091
2092/*
2093** Note: this routine updates the OS on the link state
2094** the real check of the hardware only happens with
2095** a link interrupt.
2096*/
2097static void
2098ixgbe_update_link_status(struct adapter *adapter)
2099{
2100 struct ifnet *ifp = adapter->ifp;
2101 struct tx_ring *txr = adapter->tx_rings;
2102 device_t dev = adapter->dev;
2103
2104
2105 if (adapter->link_up){
2106 if (adapter->link_active == FALSE) {
2107 if (bootverbose)
2108 device_printf(dev,"Link is up %d Gbps %s \n",
2109 ((adapter->link_speed == 128)? 10:1),
2110 "Full Duplex");
2111 adapter->link_active = TRUE;
ce451236
FT
2112 /* Update any Flow Control changes */
2113 ixgbe_fc_enable(&adapter->hw);
9407f759
FT
2114 ifp->if_link_state = LINK_STATE_UP;
2115 if_link_state_change(ifp);
2116 }
2117 } else { /* Link down */
2118 if (adapter->link_active == TRUE) {
2119 if (bootverbose)
2120 device_printf(dev,"Link is Down\n");
2121 ifp->if_link_state = LINK_STATE_DOWN;
2122 if_link_state_change(ifp);
2123 adapter->link_active = FALSE;
2124 for (int i = 0; i < adapter->num_queues;
2125 i++, txr++)
2126 txr->queue_status = IXGBE_QUEUE_IDLE;
2127 }
2128 }
2129
2130 return;
2131}
2132
2133
2134/*********************************************************************
2135 *
2136 * This routine disables all traffic on the adapter by issuing a
2137 * global reset on the MAC and deallocates TX/RX buffers.
2138 *
2139 **********************************************************************/
2140
2141static void
2142ixgbe_stop(void *arg)
2143{
2144 struct ifnet *ifp;
2145 struct adapter *adapter = arg;
2146 struct ixgbe_hw *hw = &adapter->hw;
2147 ifp = adapter->ifp;
2148
2149 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2150
2151 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2152 ixgbe_disable_intr(adapter);
2153 callout_stop(&adapter->timer);
2154
2155 /* Let the stack know...*/
2156 ifp->if_flags &= ~IFF_RUNNING;
2157 ifp->if_flags |= IFF_OACTIVE;
2158
2159 ixgbe_reset_hw(hw);
2160 hw->adapter_stopped = FALSE;
2161 ixgbe_stop_adapter(hw);
2162 /* Turn off the laser */
2163 if (hw->phy.multispeed_fiber)
2164 ixgbe_disable_tx_laser(hw);
2165
2166 /* reprogram the RAR[0] in case user changed it. */
2167 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2168
2169 return;
2170}
2171
2172
2173/*********************************************************************
2174 *
2175 * Determine hardware revision.
2176 *
2177 **********************************************************************/
2178static void
2179ixgbe_identify_hardware(struct adapter *adapter)
2180{
2181 device_t dev = adapter->dev;
2182 struct ixgbe_hw *hw = &adapter->hw;
2183
2184 /* Save off the information about this board */
2185 hw->vendor_id = pci_get_vendor(dev);
2186 hw->device_id = pci_get_device(dev);
2187 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2188 hw->subsystem_vendor_id =
2189 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2190 hw->subsystem_device_id =
2191 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2192
2193 /* We need this here to set the num_segs below */
2194 ixgbe_set_mac_type(hw);
2195
2196 /* Pick up the 82599 and VF settings */
2197 if (hw->mac.type != ixgbe_mac_82598EB) {
2198 hw->phy.smart_speed = ixgbe_smart_speed;
2199 adapter->num_segs = IXGBE_82599_SCATTER;
2200 } else
2201 adapter->num_segs = IXGBE_82598_SCATTER;
2202
2203 return;
2204}
2205
2206/*********************************************************************
2207 *
2208 * Determine optic type
2209 *
2210 **********************************************************************/
2211static void
2212ixgbe_setup_optics(struct adapter *adapter)
2213{
2214 struct ixgbe_hw *hw = &adapter->hw;
2215 int layer;
2216
2217 layer = ixgbe_get_supported_physical_layer(hw);
2218
2219 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2220 adapter->optics = IFM_10G_T;
2221 return;
2222 }
2223
2224 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2225 adapter->optics = IFM_1000_T;
2226 return;
2227 }
2228
2229 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2230 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2231 adapter->optics = IFM_10G_LR;
2232 return;
2233 }
2234
2235 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2236 adapter->optics = IFM_10G_SR;
2237 return;
2238 }
2239
2240 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2241 adapter->optics = IFM_10G_TWINAX;
2242 return;
2243 }
2244
2245 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2246 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2247 adapter->optics = IFM_10G_CX4;
2248 return;
2249 }
2250
2251 /* If we get here just set the default */
2252 adapter->optics = IFM_ETHER | IFM_AUTO;
2253 return;
2254}
2255
2256/*********************************************************************
2257 *
2258 * Setup the Legacy or MSI Interrupt handler
2259 *
2260 **********************************************************************/
2261static int
2262ixgbe_allocate_legacy(struct adapter *adapter)
2263{
2264 device_t dev = adapter->dev;
2265 struct ix_queue *que = adapter->queues;
2266 int error, rid = 0;
7e665ea1 2267 unsigned int intr_flags;
9407f759
FT
2268
2269 /* MSI RID at 1 */
2270 if (adapter->msix == 1)
2271 rid = 1;
2272
7e665ea1
FT
2273 /* Try allocating a MSI interrupt first */
2274 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2275 &rid, &intr_flags);
2276
9407f759
FT
2277 /* We allocate a single interrupt resource */
2278 adapter->res = bus_alloc_resource_any(dev,
7e665ea1 2279 SYS_RES_IRQ, &rid, intr_flags);
9407f759
FT
2280 if (adapter->res == NULL) {
2281 device_printf(dev, "Unable to allocate bus resource: "
2282 "interrupt\n");
2283 return (ENXIO);
2284 }
2285
2286 /*
2287 * Try allocating a fast interrupt and the associated deferred
2288 * processing contexts.
2289 */
2290 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2291 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2292 taskqueue_thread_enqueue, &que->tq);
2293 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2294 device_get_nameunit(adapter->dev));
2295
2296 /* Tasklets for Link, SFP and Multispeed Fiber */
2297 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2298 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2299 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2300#ifdef IXGBE_FDIR
2301 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2302#endif
2303 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2304 taskqueue_thread_enqueue, &adapter->tq);
2305 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2306 device_get_nameunit(adapter->dev));
2307
2308 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2309 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2310 device_printf(dev, "Failed to register fast interrupt "
2311 "handler: %d\n", error);
2312 taskqueue_free(que->tq);
2313 taskqueue_free(adapter->tq);
2314 que->tq = NULL;
2315 adapter->tq = NULL;
2316 return (error);
2317 }
2318 /* For simplicity in the handlers */
2319 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2320
2321 return (0);
2322}
2323
2324
2325/*********************************************************************
2326 *
2327 * Setup MSIX Interrupt resources and handlers
2328 *
2329 **********************************************************************/
2330static int
2331ixgbe_allocate_msix(struct adapter *adapter)
2332{
2333 device_t dev = adapter->dev;
2334 struct ix_queue *que = adapter->queues;
2335 int error, rid, vector = 0;
2336
2337 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2338 rid = vector + 1;
2339 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2340 RF_SHAREABLE | RF_ACTIVE);
2341 if (que->res == NULL) {
2342 device_printf(dev,"Unable to allocate"
2343 " bus resource: que interrupt [%d]\n", vector);
2344 return (ENXIO);
2345 }
2346 /* Set the handler function */
2347 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2348 ixgbe_msix_que, que, &que->tag, &que->serializer);
2349 if (error) {
2350 que->res = NULL;
2351 device_printf(dev, "Failed to register QUE handler");
2352 return (error);
2353 }
2354#if 0 /* __FreeBSD_version >= 800504 */
2355 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2356#endif
2357 que->msix = vector;
2358 adapter->que_mask |= (u64)(1 << que->msix);
2359 /*
2360 ** Bind the msix vector, and thus the
2361 ** ring to the corresponding cpu.
2362 */
2363#if 0 /* XXX */
2364 if (adapter->num_queues > 1)
2365 bus_bind_intr(dev, que->res, i);
2366#endif
2367
2368 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2369 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2370 taskqueue_thread_enqueue, &que->tq);
2371 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2372 device_get_nameunit(adapter->dev));
2373 }
2374
2375 /* and Link */
2376 rid = vector + 1;
2377 adapter->res = bus_alloc_resource_any(dev,
2378 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2379 if (!adapter->res) {
2380 device_printf(dev,"Unable to allocate"
2381 " bus resource: Link interrupt [%d]\n", rid);
2382 return (ENXIO);
2383 }
2384 /* Set the link handler function */
2385 error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2386 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2387 if (error) {
2388 adapter->res = NULL;
2389 device_printf(dev, "Failed to register LINK handler");
2390 return (error);
2391 }
2392#if 0 /* __FreeBSD_version >= 800504 */
2393 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2394#endif
2395 adapter->linkvec = vector;
2396 /* Tasklets for Link, SFP and Multispeed Fiber */
2397 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2398 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2399 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2400#ifdef IXGBE_FDIR
2401 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2402#endif
2403 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2404 taskqueue_thread_enqueue, &adapter->tq);
2405 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2406 device_get_nameunit(adapter->dev));
2407
2408 return (0);
2409}
2410
2411#if 0 /* HAVE_MSIX */
2412/*
2413 * Setup Either MSI/X or MSI
2414 */
2415static int
2416ixgbe_setup_msix(struct adapter *adapter)
2417{
2418 device_t dev = adapter->dev;
2419 int rid, want, queues, msgs;
2420
2421 /* Override by tuneable */
2422 if (ixgbe_enable_msix == 0)
2423 goto msi;
2424
2425 /* First try MSI/X */
2426 rid = PCIR_BAR(MSIX_82598_BAR);
2427 adapter->msix_mem = bus_alloc_resource_any(dev,
2428 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2429 if (!adapter->msix_mem) {
2430 rid += 4; /* 82599 maps in higher BAR */
2431 adapter->msix_mem = bus_alloc_resource_any(dev,
2432 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2433 }
2434 if (!adapter->msix_mem) {
2435 /* May not be enabled */
2436 device_printf(adapter->dev,
2437 "Unable to map MSIX table \n");
2438 goto msi;
2439 }
2440
2441 msgs = pci_msix_count(dev);
2442 if (msgs == 0) { /* system has msix disabled */
2443 bus_release_resource(dev, SYS_RES_MEMORY,
2444 rid, adapter->msix_mem);
2445 adapter->msix_mem = NULL;
2446 goto msi;
2447 }
2448
2449 /* Figure out a reasonable auto config value */
2450 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2451
2452 if (ixgbe_num_queues != 0)
2453 queues = ixgbe_num_queues;
2454 /* Set max queues to 8 when autoconfiguring */
2455 else if ((ixgbe_num_queues == 0) && (queues > 8))
2456 queues = 8;
2457
2458 /*
2459 ** Want one vector (RX/TX pair) per queue
2460 ** plus an additional for Link.
2461 */
2462 want = queues + 1;
2463 if (msgs >= want)
2464 msgs = want;
2465 else {
2466 device_printf(adapter->dev,
2467 "MSIX Configuration Problem, "
2468 "%d vectors but %d queues wanted!\n",
2469 msgs, want);
2470 return (0); /* Will go to Legacy setup */
2471 }
2472 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2473 device_printf(adapter->dev,
2474 "Using MSIX interrupts with %d vectors\n", msgs);
2475 adapter->num_queues = queues;
2476 return (msgs);
2477 }
2478msi:
2479 msgs = pci_msi_count(dev);
2480 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2481 device_printf(adapter->dev,"Using an MSI interrupt\n");
2482 else
2483 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2484 return (msgs);
2485}
2486#endif
2487
2488
2489static int
2490ixgbe_allocate_pci_resources(struct adapter *adapter)
2491{
2492 int rid;
2493 device_t dev = adapter->dev;
2494
2495 rid = PCIR_BAR(0);
2496 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2497 &rid, RF_ACTIVE);
2498
2499 if (!(adapter->pci_mem)) {
2500 device_printf(dev,"Unable to allocate bus resource: memory\n");
2501 return (ENXIO);
2502 }
2503
2504 adapter->osdep.mem_bus_space_tag =
2505 rman_get_bustag(adapter->pci_mem);
2506 adapter->osdep.mem_bus_space_handle =
2507 rman_get_bushandle(adapter->pci_mem);
2508 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2509
2510 /* Legacy defaults */
2511 adapter->num_queues = 1;
2512 adapter->hw.back = &adapter->osdep;
2513
2514 /*
2515 ** Now setup MSI or MSI/X, should
2516 ** return us the number of supported
2517 ** vectors. (Will be 1 for MSI)
2518 */
2519#if 0 /* HAVE_MSIX */
2520 adapter->msix = ixgbe_setup_msix(adapter);
2521#endif
2522 return (0);
2523}
2524
2525static void
2526ixgbe_free_pci_resources(struct adapter * adapter)
2527{
2528 struct ix_queue *que = adapter->queues;
2529 device_t dev = adapter->dev;
2530 int rid, memrid;
2531
2532 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2533 memrid = PCIR_BAR(MSIX_82598_BAR);
2534 else
2535 memrid = PCIR_BAR(MSIX_82599_BAR);
2536
2537 /*
2538 ** There is a slight possibility of a failure mode
2539 ** in attach that will result in entering this function
2540 ** before interrupt resources have been initialized, and
2541 ** in that case we do not want to execute the loops below
2542 ** We can detect this reliably by the state of the adapter
2543 ** res pointer.
2544 */
2545 if (adapter->res == NULL)
2546 goto mem;
2547
2548 /*
2549 ** Release all msix queue resources:
2550 */
2551 for (int i = 0; i < adapter->num_queues; i++, que++) {
2552 rid = que->msix + 1;
2553 if (que->tag != NULL) {
2554 bus_teardown_intr(dev, que->res, que->tag);
2555 que->tag = NULL;
2556 }
2557 if (que->res != NULL)
2558 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2559 }
2560
2561
2562 /* Clean the Legacy or Link interrupt last */
2563 if (adapter->linkvec) /* we are doing MSIX */
2564 rid = adapter->linkvec + 1;
2565 else
2566 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2567
2568 if (adapter->tag != NULL) {
2569 bus_teardown_intr(dev, adapter->res, adapter->tag);
2570 adapter->tag = NULL;
2571 }
2572 if (adapter->res != NULL)
2573 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
7e665ea1
FT
2574 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2575 pci_release_msi(adapter->dev);
9407f759
FT
2576
2577mem:
2578 if (adapter->msix)
2579 pci_release_msi(dev);
2580
2581 if (adapter->msix_mem != NULL)
2582 bus_release_resource(dev, SYS_RES_MEMORY,
2583 memrid, adapter->msix_mem);
2584
2585 if (adapter->pci_mem != NULL)
2586 bus_release_resource(dev, SYS_RES_MEMORY,
2587 PCIR_BAR(0), adapter->pci_mem);
2588
2589 return;
2590}
2591
2592/*********************************************************************
2593 *
2594 * Setup networking device structure and register an interface.
2595 *
2596 **********************************************************************/
2597static int
2598ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2599{
2600 struct ixgbe_hw *hw = &adapter->hw;
2601 struct ifnet *ifp;
2602
2603 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2604
2605 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2606 if (ifp == NULL) {
2607 device_printf(dev, "can not allocate ifnet structure\n");
2608 return (-1);
2609 }
2610 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2611 ifp->if_baudrate = 1000000000;
2612 ifp->if_init = ixgbe_init;
2613 ifp->if_softc = adapter;
2614 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2615 ifp->if_ioctl = ixgbe_ioctl;
2616 ifp->if_start = ixgbe_start;
2617#if 0 /* __FreeBSD_version >= 800000 */
2618 ifp->if_transmit = ixgbe_mq_start;
2619 ifp->if_qflush = ixgbe_qflush;
2620#endif
2621 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2622
2623 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2624
2625 adapter->max_frame_size =
2626 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2627
2628 /*
2629 * Tell the upper layer(s) we support long frames.
2630 */
2631 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2632
2633#if 0 /* NET_TSO */
2634 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2635#endif
2636 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2637 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2638 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2639#if 0 /* NET_TSO */
2640 | IFCAP_VLAN_HWTSO
2641#endif
2642 | IFCAP_VLAN_MTU;
2643 ifp->if_capenable = ifp->if_capabilities;
2644
2645 /* Don't enable LRO by default */
2646#if 0 /* NET_LRO */
2647 ifp->if_capabilities |= IFCAP_LRO;
2648#endif
2649
2650 /*
2651 ** Don't turn this on by default, if vlans are
2652 ** created on another pseudo device (eg. lagg)
2653 ** then vlan events are not passed thru, breaking
2654 ** operation, but with HW FILTER off it works. If
2655 ** using vlans directly on the ixgbe driver you can
2656 ** enable this and get full hardware tag filtering.
2657 */
2658 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2659
2660 /*
2661 * Specify the media types supported by this adapter and register
2662 * callbacks to update media and link information
2663 */
2664 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2665 ixgbe_media_status);
2666 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2667 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2668 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2669 ifmedia_add(&adapter->media,
2670 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2671 ifmedia_add(&adapter->media,
2672 IFM_ETHER | IFM_1000_T, 0, NULL);
2673 }
2674 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2675 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2676
2677 return (0);
2678}
2679
2680static void
2681ixgbe_config_link(struct adapter *adapter)
2682{
2683 struct ixgbe_hw *hw = &adapter->hw;
2684 u32 autoneg, err = 0;
2685 bool sfp, negotiate;
2686
2687 sfp = ixgbe_is_sfp(hw);
2688
2689 if (sfp) {
2690 if (hw->phy.multispeed_fiber) {
2691 hw->mac.ops.setup_sfp(hw);
2692 ixgbe_enable_tx_laser(hw);
2693 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2694 } else
2695 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2696 } else {
2697 if (hw->mac.ops.check_link)
2698 err = ixgbe_check_link(hw, &autoneg,
2699 &adapter->link_up, FALSE);
2700 if (err)
2701 goto out;
2702 autoneg = hw->phy.autoneg_advertised;
2703 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2704 err = hw->mac.ops.get_link_capabilities(hw,
2705 &autoneg, &negotiate);
2706 if (err)
2707 goto out;
2708 if (hw->mac.ops.setup_link)
2709 err = hw->mac.ops.setup_link(hw, autoneg,
2710 negotiate, adapter->link_up);
2711 }
2712out:
2713 return;
2714}
2715
2716/********************************************************************
2717 * Manage DMA'able memory.
2718 *******************************************************************/
2719static void
2720ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2721{
2722 if (error)
2723 return;
2724 *(bus_addr_t *) arg = segs->ds_addr;
2725 return;
2726}
2727
2728static int
2729ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2730 struct ixgbe_dma_alloc *dma, int mapflags)
2731{
2732 device_t dev = adapter->dev;
2733 int r;
2734
2735 r = bus_dma_tag_create(NULL, /* parent */
2736 DBA_ALIGN, 0, /* alignment, bounds */
2737 BUS_SPACE_MAXADDR, /* lowaddr */
2738 BUS_SPACE_MAXADDR, /* highaddr */
2739 NULL, NULL, /* filter, filterarg */
2740 size, /* maxsize */
2741 1, /* nsegments */
2742 size, /* maxsegsize */
2743 BUS_DMA_ALLOCNOW, /* flags */
2744 &dma->dma_tag);
2745 if (r != 0) {
2746 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2747 "error %u\n", r);
2748 goto fail_0;
2749 }
2750 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2751 BUS_DMA_NOWAIT, &dma->dma_map);
2752 if (r != 0) {
2753 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2754 "error %u\n", r);
2755 goto fail_1;
2756 }
2757 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2758 size,
2759 ixgbe_dmamap_cb,
2760 &dma->dma_paddr,
2761 mapflags | BUS_DMA_NOWAIT);
2762 if (r != 0) {
2763 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2764 "error %u\n", r);
2765 goto fail_2;
2766 }
2767 dma->dma_size = size;
2768 return (0);
2769fail_2:
2770 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2771fail_1:
2772 bus_dma_tag_destroy(dma->dma_tag);
2773fail_0:
2774 dma->dma_map = NULL;
2775 dma->dma_tag = NULL;
2776 return (r);
2777}
2778
2779static void
2780ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2781{
2782 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2783 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2784 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2785 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2786 bus_dma_tag_destroy(dma->dma_tag);
2787}
2788
2789
2790/*********************************************************************
2791 *
2792 * Allocate memory for the transmit and receive rings, and then
2793 * the descriptors associated with each, called only once at attach.
2794 *
2795 **********************************************************************/
2796static int
2797ixgbe_allocate_queues(struct adapter *adapter)
2798{
2799 device_t dev = adapter->dev;
2800 struct ix_queue *que;
2801 struct tx_ring *txr;
2802 struct rx_ring *rxr;
2803 int rsize, tsize, error = IXGBE_SUCCESS;
2804 int txconf = 0, rxconf = 0;
2805
2806 /* First allocate the top level queue structs */
2807 if (!(adapter->queues =
2808 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2809 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2810 device_printf(dev, "Unable to allocate queue memory\n");
2811 error = ENOMEM;
2812 goto fail;
2813 }
2814
2815 /* First allocate the TX ring struct memory */
2816 if (!(adapter->tx_rings =
2817 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2818 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2819 device_printf(dev, "Unable to allocate TX ring memory\n");
2820 error = ENOMEM;
2821 goto tx_fail;
2822 }
2823
2824 /* Next allocate the RX */
2825 if (!(adapter->rx_rings =
2826 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2827 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2828 device_printf(dev, "Unable to allocate RX ring memory\n");
2829 error = ENOMEM;
2830 goto rx_fail;
2831 }
2832
2833 /* For the ring itself */
2834 tsize = roundup2(adapter->num_tx_desc *
2835 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2836
2837 /*
2838 * Now set up the TX queues, txconf is needed to handle the
2839 * possibility that things fail midcourse and we need to
2840 * undo memory gracefully
2841 */
2842 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2843 /* Set up some basics */
2844 txr = &adapter->tx_rings[i];
2845 txr->adapter = adapter;
2846 txr->me = i;
2847
2848 /* Initialize the TX side lock */
2849 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2850 device_get_nameunit(dev), txr->me);
2851 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2852
2853 if (ixgbe_dma_malloc(adapter, tsize,
2854 &txr->txdma, BUS_DMA_NOWAIT)) {
2855 device_printf(dev,
2856 "Unable to allocate TX Descriptor memory\n");
2857 error = ENOMEM;
2858 goto err_tx_desc;
2859 }
2860 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2861 bzero((void *)txr->tx_base, tsize);
2862
2863 /* Now allocate transmit buffers for the ring */
2864 if (ixgbe_allocate_transmit_buffers(txr)) {
2865 device_printf(dev,
2866 "Critical Failure setting up transmit buffers\n");
2867 error = ENOMEM;
2868 goto err_tx_desc;
2869 }
2870#if 0 /* __FreeBSD_version >= 800000 */
2871 /* Allocate a buf ring */
2872 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2873 M_WAITOK, &txr->tx_mtx);
2874 if (txr->br == NULL) {
2875 device_printf(dev,
2876 "Critical Failure setting up buf ring\n");
2877 error = ENOMEM;
2878 goto err_tx_desc;
2879 }
2880#endif
2881 }
2882
2883 /*
2884 * Next the RX queues...
2885 */
2886 rsize = roundup2(adapter->num_rx_desc *
2887 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2888 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2889 rxr = &adapter->rx_rings[i];
2890 /* Set up some basics */
2891 rxr->adapter = adapter;
2892 rxr->me = i;
2893
2894 /* Initialize the RX side lock */
2895 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2896 device_get_nameunit(dev), rxr->me);
2897 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2898
2899 if (ixgbe_dma_malloc(adapter, rsize,
2900 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2901 device_printf(dev,
2902 "Unable to allocate RxDescriptor memory\n");
2903 error = ENOMEM;
2904 goto err_rx_desc;
2905 }
2906 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2907 bzero((void *)rxr->rx_base, rsize);
2908
2909 /* Allocate receive buffers for the ring*/
2910 if (ixgbe_allocate_receive_buffers(rxr)) {
2911 device_printf(dev,
2912 "Critical Failure setting up receive buffers\n");
2913 error = ENOMEM;
2914 goto err_rx_desc;
2915 }
2916 }
2917
2918 /*
2919 ** Finally set up the queue holding structs
2920 */
2921 for (int i = 0; i < adapter->num_queues; i++) {
2922 que = &adapter->queues[i];
2923 que->adapter = adapter;
2924 que->txr = &adapter->tx_rings[i];
2925 que->rxr = &adapter->rx_rings[i];
2926 }
2927
2928 return (0);
2929
2930err_rx_desc:
2931 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2932 ixgbe_dma_free(adapter, &rxr->rxdma);
2933err_tx_desc:
2934 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2935 ixgbe_dma_free(adapter, &txr->txdma);
2936 kfree(adapter->rx_rings, M_DEVBUF);
2937rx_fail:
2938 kfree(adapter->tx_rings, M_DEVBUF);
2939tx_fail:
2940 kfree(adapter->queues, M_DEVBUF);
2941fail:
2942 return (error);
2943}
2944
2945/*********************************************************************
2946 *
2947 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2948 * the information needed to transmit a packet on the wire. This is
2949 * called only once at attach, setup is done every reset.
2950 *
2951 **********************************************************************/
2952static int
2953ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2954{
2955 struct adapter *adapter = txr->adapter;
2956 device_t dev = adapter->dev;
2957 struct ixgbe_tx_buf *txbuf;
2958 int error, i;
2959
2960 /*
2961 * Setup DMA descriptor areas.
2962 */
2963 if ((error = bus_dma_tag_create(
2964 NULL, /* parent */
2965 1, 0, /* alignment, bounds */
2966 BUS_SPACE_MAXADDR, /* lowaddr */
2967 BUS_SPACE_MAXADDR, /* highaddr */
2968 NULL, NULL, /* filter, filterarg */
2969 IXGBE_TSO_SIZE, /* maxsize */
2970 adapter->num_segs, /* nsegments */
2971 PAGE_SIZE, /* maxsegsize */
2972 0, /* flags */
2973 &txr->txtag))) {
2974 device_printf(dev,"Unable to allocate TX DMA tag\n");
2975 goto fail;
2976 }
2977
2978 if (!(txr->tx_buffers =
2979 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2980 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2981 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2982 error = ENOMEM;
2983 goto fail;
2984 }
2985
2986 /* Create the descriptor buffer dma maps */
2987 txbuf = txr->tx_buffers;
2988 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2989 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2990 if (error != 0) {
2991 device_printf(dev, "Unable to create TX DMA map\n");
2992 goto fail;
2993 }
2994 }
2995
2996 return 0;
2997fail:
2998 /* We free all, it handles case where we are in the middle */
2999 ixgbe_free_transmit_structures(adapter);
3000 return (error);
3001}
3002
3003/*********************************************************************
3004 *
3005 * Initialize a transmit ring.
3006 *
3007 **********************************************************************/
3008static void
3009ixgbe_setup_transmit_ring(struct tx_ring *txr)
3010{
3011 struct adapter *adapter = txr->adapter;
3012 struct ixgbe_tx_buf *txbuf;
3013 int i;
3014#ifdef DEV_NETMAP
3015 struct netmap_adapter *na = NA(adapter->ifp);
3016 struct netmap_slot *slot;
3017#endif /* DEV_NETMAP */
3018
3019 /* Clear the old ring contents */
3020 IXGBE_TX_LOCK(txr);
3021#ifdef DEV_NETMAP
3022 /*
3023 * (under lock): if in netmap mode, do some consistency
3024 * checks and set slot to entry 0 of the netmap ring.
3025 */
3026 slot = netmap_reset(na, NR_TX, txr->me, 0);
3027#endif /* DEV_NETMAP */
3028 bzero((void *)txr->tx_base,
3029 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3030 /* Reset indices */
3031 txr->next_avail_desc = 0;
3032 txr->next_to_clean = 0;
3033
3034 /* Free any existing tx buffers. */
3035 txbuf = txr->tx_buffers;
3036 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3037 if (txbuf->m_head != NULL) {
3038 bus_dmamap_sync(txr->txtag, txbuf->map,
3039 BUS_DMASYNC_POSTWRITE);
3040 bus_dmamap_unload(txr->txtag, txbuf->map);
3041 m_freem(txbuf->m_head);
3042 txbuf->m_head = NULL;
3043 }
3044#ifdef DEV_NETMAP
3045 /*
3046 * In netmap mode, set the map for the packet buffer.
3047 * NOTE: Some drivers (not this one) also need to set
3048 * the physical buffer address in the NIC ring.
3049 * Slots in the netmap ring (indexed by "si") are
3050 * kring->nkr_hwofs positions "ahead" wrt the
3051 * corresponding slot in the NIC ring. In some drivers
3052 * (not here) nkr_hwofs can be negative. Function
3053 * netmap_idx_n2k() handles wraparounds properly.
3054 */
3055 if (slot) {
3056 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3057 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3058 }
3059#endif /* DEV_NETMAP */
3060 /* Clear the EOP index */
3061 txbuf->eop_index = -1;
3062 }
3063
3064#ifdef IXGBE_FDIR
3065 /* Set the rate at which we sample packets */
3066 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3067 txr->atr_sample = atr_sample_rate;
3068#endif
3069
3070 /* Set number of descriptors available */
3071 txr->tx_avail = adapter->num_tx_desc;
3072
3073 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3074 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3075 IXGBE_TX_UNLOCK(txr);
3076}
3077
3078/*********************************************************************
3079 *
3080 * Initialize all transmit rings.
3081 *
3082 **********************************************************************/
3083static int
3084ixgbe_setup_transmit_structures(struct adapter *adapter)
3085{
3086 struct tx_ring *txr = adapter->tx_rings;
3087
3088 for (int i = 0; i < adapter->num_queues; i++, txr++)
3089 ixgbe_setup_transmit_ring(txr);
3090
3091 return (0);
3092}
3093
3094/*********************************************************************
3095 *
3096 * Enable transmit unit.
3097 *
3098 **********************************************************************/
3099static void
3100ixgbe_initialize_transmit_units(struct adapter *adapter)
3101{
3102 struct tx_ring *txr = adapter->tx_rings;
3103 struct ixgbe_hw *hw = &adapter->hw;
3104
3105 /* Setup the Base and Length of the Tx Descriptor Ring */
3106
3107 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3108 u64 tdba = txr->txdma.dma_paddr;
3109 u32 txctrl;
3110
3111 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3112 (tdba & 0x00000000ffffffffULL));
3113 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3114 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3115 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3116
3117 /* Setup the HW Tx Head and Tail descriptor pointers */
3118 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3119 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3120
3121 /* Setup Transmit Descriptor Cmd Settings */
3122 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3123 txr->queue_status = IXGBE_QUEUE_IDLE;
3124
3125 /* Disable Head Writeback */
3126 switch (hw->mac.type) {
3127 case ixgbe_mac_82598EB:
3128 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3129 break;
3130 case ixgbe_mac_82599EB:
3131 case ixgbe_mac_X540:
3132 default:
3133 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3134 break;
3135 }
ce451236 3136 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
9407f759
FT
3137 switch (hw->mac.type) {
3138 case ixgbe_mac_82598EB:
3139 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3140 break;
3141 case ixgbe_mac_82599EB:
3142 case ixgbe_mac_X540:
3143 default:
3144 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3145 break;
3146 }
3147
3148 }
3149
3150 if (hw->mac.type != ixgbe_mac_82598EB) {
3151 u32 dmatxctl, rttdcs;
3152 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3153 dmatxctl |= IXGBE_DMATXCTL_TE;
3154 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3155 /* Disable arbiter to set MTQC */
3156 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3157 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3158 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3159 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3160 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3161 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3162 }
3163
3164 return;
3165}
3166
3167/*********************************************************************
3168 *
3169 * Free all transmit rings.
3170 *
3171 **********************************************************************/
3172static void
3173ixgbe_free_transmit_structures(struct adapter *adapter)
3174{
3175 struct tx_ring *txr = adapter->tx_rings;
3176
3177 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3178 IXGBE_TX_LOCK(txr);
3179 ixgbe_free_transmit_buffers(txr);
3180 ixgbe_dma_free(adapter, &txr->txdma);
3181 IXGBE_TX_UNLOCK(txr);
3182 IXGBE_TX_LOCK_DESTROY(txr);
3183 }
3184 kfree(adapter->tx_rings, M_DEVBUF);
3185}
3186
3187/*********************************************************************
3188 *
3189 * Free transmit ring related data structures.
3190 *
3191 **********************************************************************/
3192static void
3193ixgbe_free_transmit_buffers(struct tx_ring *txr)
3194{
3195 struct adapter *adapter = txr->adapter;
3196 struct ixgbe_tx_buf *tx_buffer;
3197 int i;
3198
3199 INIT_DEBUGOUT("free_transmit_ring: begin");
3200
3201 if (txr->tx_buffers == NULL)
3202 return;
3203
3204 tx_buffer = txr->tx_buffers;
3205 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3206 if (tx_buffer->m_head != NULL) {
3207 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3208 BUS_DMASYNC_POSTWRITE);
3209 bus_dmamap_unload(txr->txtag,
3210 tx_buffer->map);
3211 m_freem(tx_buffer->m_head);
3212 tx_buffer->m_head = NULL;
3213 if (tx_buffer->map != NULL) {
3214 bus_dmamap_destroy(txr->txtag,
3215 tx_buffer->map);
3216 tx_buffer->map = NULL;
3217 }
3218 } else if (tx_buffer->map != NULL) {
3219 bus_dmamap_unload(txr->txtag,
3220 tx_buffer->map);
3221 bus_dmamap_destroy(txr->txtag,
3222 tx_buffer->map);
3223 tx_buffer->map = NULL;
3224 }
3225 }
3226#if 0 /* __FreeBSD_version >= 800000 */
3227 if (txr->br != NULL)
3228 buf_ring_free(txr->br, M_DEVBUF);
3229#endif
3230 if (txr->tx_buffers != NULL) {
3231 kfree(txr->tx_buffers, M_DEVBUF);
3232 txr->tx_buffers = NULL;
3233 }
3234 if (txr->txtag != NULL) {
3235 bus_dma_tag_destroy(txr->txtag);
3236 txr->txtag = NULL;
3237 }
3238 return;
3239}
3240
3241/*********************************************************************
3242 *
3243 * Advanced Context Descriptor setup for VLAN or CSUM
3244 *
3245 **********************************************************************/
3246
3247static bool
3248ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3249{
3250 struct adapter *adapter = txr->adapter;
3251 struct ixgbe_adv_tx_context_desc *TXD;
3252 struct ixgbe_tx_buf *tx_buffer;
3253 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3254 struct ether_vlan_header *eh;
3255 struct ip *ip;
3256 struct ip6_hdr *ip6;
3257 int ehdrlen, ip_hlen = 0;
3258 u16 etype;
3259 u8 ipproto = 0;
3260 bool offload = TRUE;
3261 int ctxd = txr->next_avail_desc;
3262#ifdef NET_VLAN
3263 u16 vtag = 0;
3264#endif
3265
3266
3267 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3268 offload = FALSE;
3269
3270 tx_buffer = &txr->tx_buffers[ctxd];
3271 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3272
3273 /*
3274 ** In advanced descriptors the vlan tag must
3275 ** be placed into the descriptor itself.
3276 */
3277#ifdef NET_VLAN
3278 if (mp->m_flags & M_VLANTAG) {
3279 vtag = htole16(mp->m_pkthdr.ether_vtag);
3280 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3281 } else if (offload == FALSE)
3282 return FALSE;
3283#endif
3284
3285 /*
3286 * Determine where frame payload starts.
3287 * Jump over vlan headers if already present,
3288 * helpful for QinQ too.
3289 */
3290 eh = mtod(mp, struct ether_vlan_header *);
3291 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3292 etype = ntohs(eh->evl_proto);
3293 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3294 } else {
3295 etype = ntohs(eh->evl_encap_proto);
3296 ehdrlen = ETHER_HDR_LEN;
3297 }
3298
3299 /* Set the ether header length */
3300 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3301
3302 switch (etype) {
3303 case ETHERTYPE_IP:
3304 ip = (struct ip *)(mp->m_data + ehdrlen);
3305 ip_hlen = ip->ip_hl << 2;
3306 ipproto = ip->ip_p;
3307 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3308 break;
3309 case ETHERTYPE_IPV6:
3310 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3311 ip_hlen = sizeof(struct ip6_hdr);
3312 /* XXX-BZ this will go badly in case of ext hdrs. */
3313 ipproto = ip6->ip6_nxt;
3314 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3315 break;
3316 default:
3317 offload = FALSE;
3318 break;
3319 }
3320
3321 vlan_macip_lens |= ip_hlen;
3322 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3323
3324 switch (ipproto) {
3325 case IPPROTO_TCP:
3326 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3327 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3328 break;
3329
3330 case IPPROTO_UDP:
3331 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3332 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3333 break;
3334
3335#if 0
3336 case IPPROTO_SCTP:
3337 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3338 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3339 break;
3340#endif
3341 default:
3342 offload = FALSE;
3343 break;
3344 }
3345
3346 /* Now copy bits into descriptor */
3347 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3348 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3349 TXD->seqnum_seed = htole32(0);
3350 TXD->mss_l4len_idx = htole32(0);
3351
3352 tx_buffer->m_head = NULL;
3353 tx_buffer->eop_index = -1;
3354
3355 /* We've consumed the first desc, adjust counters */
3356 if (++ctxd == adapter->num_tx_desc)
3357 ctxd = 0;
3358 txr->next_avail_desc = ctxd;
3359 --txr->tx_avail;
3360
3361 return (offload);
3362}
3363
3364/**********************************************************************
3365 *
3366 * Setup work for hardware segmentation offload (TSO) on
3367 * adapters using advanced tx descriptors
3368 *
3369 **********************************************************************/
3370#if 0 /* NET_TSO */
3371static bool
3372ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3373 u32 *olinfo_status)
3374{
3375 struct adapter *adapter = txr->adapter;
3376 struct ixgbe_adv_tx_context_desc *TXD;
3377 struct ixgbe_tx_buf *tx_buffer;
3378#ifdef NET_VLAN
3379 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3380 u16 vtag = 0, eh_type;
3381#else
3382 u16 eh_type;
3383 u32 type_tucmd_mlhl = 0;
3384#endif
3385 u32 mss_l4len_idx = 0, len;
3386 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3387 struct ether_vlan_header *eh;
3388#ifdef INET6
3389 struct ip6_hdr *ip6;
3390#endif
3391#ifdef INET
3392 struct ip *ip;
3393#endif
3394 struct tcphdr *th;
3395
3396
3397 /*
3398 * Determine where frame payload starts.
3399 * Jump over vlan headers if already present
3400 */
3401 eh = mtod(mp, struct ether_vlan_header *);
3402 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3403 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3404 eh_type = eh->evl_proto;
3405 } else {
3406 ehdrlen = ETHER_HDR_LEN;
3407 eh_type = eh->evl_encap_proto;
3408 }
3409
3410 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3411 len = ehdrlen + sizeof(struct tcphdr);
3412 switch (ntohs(eh_type)) {
3413#ifdef INET6
3414 case ETHERTYPE_IPV6:
3415 if (mp->m_len < len + sizeof(struct ip6_hdr))
3416 return FALSE;
3417 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3418 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3419 if (ip6->ip6_nxt != IPPROTO_TCP)
3420 return FALSE;
3421 ip_hlen = sizeof(struct ip6_hdr);
3422 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3423 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3424 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3425 break;
3426#endif
3427#ifdef INET
3428 case ETHERTYPE_IP:
3429 if (mp->m_len < len + sizeof(struct ip))
3430 return FALSE;
3431 ip = (struct ip *)(mp->m_data + ehdrlen);
3432 if (ip->ip_p != IPPROTO_TCP)
3433 return FALSE;
3434 ip->ip_sum = 0;
3435 ip_hlen = ip->ip_hl << 2;
3436 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3437 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3438 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3439 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3440 /* Tell transmit desc to also do IPv4 checksum. */
3441 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3442 break;
3443#endif
3444 default:
3445 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3446 __func__, ntohs(eh_type));
3447 break;
3448 }
3449
3450 ctxd = txr->next_avail_desc;
3451 tx_buffer = &txr->tx_buffers[ctxd];
3452 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3453
3454 tcp_hlen = th->th_off << 2;
3455
3456 /* This is used in the transmit desc in encap */
3457 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3458
3459 /* VLAN MACLEN IPLEN */
3460#ifdef NET_VLAN
3461 if (mp->m_flags & M_VLANTAG) {
3462 vtag = htole16(mp->m_pkthdr.ether_vtag);
3463 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3464 }
3465
3466 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3467 vlan_macip_lens |= ip_hlen;
3468 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3469#endif
3470
3471 /* ADV DTYPE TUCMD */
3472 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3473 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3474 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3475
3476 /* MSS L4LEN IDX */
3477 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3478 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3479 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3480
3481 TXD->seqnum_seed = htole32(0);
3482 tx_buffer->m_head = NULL;
3483 tx_buffer->eop_index = -1;
3484
3485 if (++ctxd == adapter->num_tx_desc)
3486 ctxd = 0;
3487
3488 txr->tx_avail--;
3489 txr->next_avail_desc = ctxd;
3490 return TRUE;
3491}
3492#endif
3493
3494#ifdef IXGBE_FDIR
3495/*
3496** This routine parses packet headers so that Flow
3497** Director can make a hashed filter table entry
3498** allowing traffic flows to be identified and kept
3499** on the same cpu. This would be a performance
3500** hit, but we only do it at IXGBE_FDIR_RATE of
3501** packets.
3502*/
3503static void
3504ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3505{
3506 struct adapter *adapter = txr->adapter;
3507 struct ix_queue *que;
3508 struct ip *ip;
3509 struct tcphdr *th;
3510 struct udphdr *uh;
3511 struct ether_vlan_header *eh;
3512 union ixgbe_atr_hash_dword input = {.dword = 0};
3513 union ixgbe_atr_hash_dword common = {.dword = 0};
3514 int ehdrlen, ip_hlen;
3515 u16 etype;
3516
3517 eh = mtod(mp, struct ether_vlan_header *);
3518 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3519 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3520 etype = eh->evl_proto;
3521 } else {
3522 ehdrlen = ETHER_HDR_LEN;
3523 etype = eh->evl_encap_proto;
3524 }
3525
3526 /* Only handling IPv4 */
3527 if (etype != htons(ETHERTYPE_IP))
3528 return;
3529
3530 ip = (struct ip *)(mp->m_data + ehdrlen);
3531 ip_hlen = ip->ip_hl << 2;
3532
3533 /* check if we're UDP or TCP */
3534 switch (ip->ip_p) {
3535 case IPPROTO_TCP:
3536 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3537 /* src and dst are inverted */
3538 common.port.dst ^= th->th_sport;
3539 common.port.src ^= th->th_dport;
3540 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3541 break;
3542 case IPPROTO_UDP:
3543 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3544 /* src and dst are inverted */
3545 common.port.dst ^= uh->uh_sport;
3546 common.port.src ^= uh->uh_dport;
3547 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3548 break;
3549 default:
3550 return;
3551 }
3552
3553 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3554 if (mp->m_pkthdr.ether_vtag)
3555 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3556 else
3557 common.flex_bytes ^= etype;
3558 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3559
3560 que = &adapter->queues[txr->me];
3561 /*
3562 ** This assumes the Rx queue and Tx
3563 ** queue are bound to the same CPU
3564 */
3565 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3566 input, common, que->msix);
3567}
3568#endif /* IXGBE_FDIR */
3569
3570/**********************************************************************
3571 *
3572 * Examine each tx_buffer in the used queue. If the hardware is done
3573 * processing the packet then free associated resources. The
3574 * tx_buffer is put back on the free queue.
3575 *
3576 **********************************************************************/
3577static bool
3578ixgbe_txeof(struct tx_ring *txr)
3579{
3580 struct adapter *adapter = txr->adapter;
3581 struct ifnet *ifp = adapter->ifp;
3582 u32 first, last, done, processed;
3583 struct ixgbe_tx_buf *tx_buffer;
3584 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3585
3586 KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3587
3588#ifdef DEV_NETMAP
3589 if (ifp->if_capenable & IFCAP_NETMAP) {
3590 struct netmap_adapter *na = NA(ifp);
3591 struct netmap_kring *kring = &na->tx_rings[txr->me];
3592
3593 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3594
3595 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3596 BUS_DMASYNC_POSTREAD);
3597 /*
3598 * In netmap mode, all the work is done in the context
3599 * of the client thread. Interrupt handlers only wake up
3600 * clients, which may be sleeping on individual rings
3601 * or on a global resource for all rings.
3602 * To implement tx interrupt mitigation, we wake up the client
3603 * thread roughly every half ring, even if the NIC interrupts
3604 * more frequently. This is implemented as follows:
3605 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3606 * the slot that should wake up the thread (nkr_num_slots
3607 * means the user thread should not be woken up);
3608 * - the driver ignores tx interrupts unless netmap_mitigate=0
3609 * or the slot has the DD bit set.
3610 *
3611 * When the driver has separate locks, we need to
3612 * release and re-acquire txlock to avoid deadlocks.
3613 * XXX see if we can find a better way.
3614 */
3615 if (!netmap_mitigate ||
3616 (kring->nr_kflags < kring->nkr_num_slots &&
3617 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3618 kring->nr_kflags = kring->nkr_num_slots;
3619 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3620 IXGBE_TX_UNLOCK(txr);
3621 IXGBE_CORE_LOCK(adapter);
3622 selwakeuppri(&na->tx_si, PI_NET);
3623 IXGBE_CORE_UNLOCK(adapter);
3624 IXGBE_TX_LOCK(txr);
3625 }
3626 return FALSE;
3627 }
3628#endif /* DEV_NETMAP */
3629
3630 if (txr->tx_avail == adapter->num_tx_desc) {
3631 txr->queue_status = IXGBE_QUEUE_IDLE;
3632 return FALSE;
3633 }
3634
3635 processed = 0;
3636 first = txr->next_to_clean;
3637 tx_buffer = &txr->tx_buffers[first];
3638 /* For cleanup we just use legacy struct */
3639 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3640 last = tx_buffer->eop_index;
3641 if (last == -1)
3642 return FALSE;
3643 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3644
3645 /*
3646 ** Get the index of the first descriptor
3647 ** BEYOND the EOP and call that 'done'.
3648 ** I do this so the comparison in the
3649 ** inner while loop below can be simple
3650 */
3651 if (++last == adapter->num_tx_desc) last = 0;
3652 done = last;
3653
3654 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3655 BUS_DMASYNC_POSTREAD);
3656 /*
3657 ** Only the EOP descriptor of a packet now has the DD
3658 ** bit set, this is what we look for...
3659 */
3660 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3661 /* We clean the range of the packet */
3662 while (first != done) {
3663 tx_desc->upper.data = 0;
3664 tx_desc->lower.data = 0;
3665 tx_desc->buffer_addr = 0;
3666 ++txr->tx_avail;
3667 ++processed;
3668
3669 if (tx_buffer->m_head) {
3670 txr->bytes +=
3671 tx_buffer->m_head->m_pkthdr.len;
3672 bus_dmamap_sync(txr->txtag,
3673 tx_buffer->map,
3674 BUS_DMASYNC_POSTWRITE);
3675 bus_dmamap_unload(txr->txtag,
3676 tx_buffer->map);
3677 m_freem(tx_buffer->m_head);
3678 tx_buffer->m_head = NULL;
3679 tx_buffer->map = NULL;
3680 }
3681 tx_buffer->eop_index = -1;
3682 txr->watchdog_time = ticks;
3683
3684 if (++first == adapter->num_tx_desc)
3685 first = 0;
3686
3687 tx_buffer = &txr->tx_buffers[first];
3688 tx_desc =
3689 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3690 }
3691 ++txr->packets;
3692 ++ifp->if_opackets;
3693 /* See if there is more work now */
3694 last = tx_buffer->eop_index;
3695 if (last != -1) {
3696 eop_desc =
3697 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3698 /* Get next done point */
3699 if (++last == adapter->num_tx_desc) last = 0;
3700 done = last;
3701 } else
3702 break;
3703 }
3704 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3705 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3706
3707 txr->next_to_clean = first;
3708
3709 /*
3710 ** Watchdog calculation, we know there's
3711 ** work outstanding or the first return
3712 ** would have been taken, so none processed
3713 ** for too long indicates a hang.
3714 */
3715 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3716 txr->queue_status = IXGBE_QUEUE_HUNG;
3717
3718 /* With a minimum free clear the depleted state bit. */
3719 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3720 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3721
3722 if (txr->tx_avail == adapter->num_tx_desc) {
3723 txr->queue_status = IXGBE_QUEUE_IDLE;
3724 return (FALSE);
3725 }
3726
3727 return TRUE;
3728}
3729
3730/*********************************************************************
3731 *
3732 * Refresh mbuf buffers for RX descriptor rings
3733 * - now keeps its own state so discards due to resource
3734 * exhaustion are unnecessary, if an mbuf cannot be obtained
3735 * it just returns, keeping its placeholder, thus it can simply
3736 * be recalled to try again.
3737 *
3738 **********************************************************************/
3739static void
3740ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3741{
3742 struct adapter *adapter = rxr->adapter;
3743 bus_dma_segment_t hseg[1];
3744 bus_dma_segment_t pseg[1];
3745 struct ixgbe_rx_buf *rxbuf;
3746 struct mbuf *mh, *mp;
3747 int i, j, nsegs, error;
3748 bool refreshed = FALSE;
3749
3750 i = j = rxr->next_to_refresh;
3751 /* Control the loop with one beyond */
3752 if (++j == adapter->num_rx_desc)
3753 j = 0;
3754
3755 while (j != limit) {
3756 rxbuf = &rxr->rx_buffers[i];
3757 if (rxr->hdr_split == FALSE)
3758 goto no_split;
3759
3760 if (rxbuf->m_head == NULL) {
3761 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3762 if (mh == NULL)
3763 goto update;
3764 } else
3765 mh = rxbuf->m_head;
3766
3767 mh->m_pkthdr.len = mh->m_len = MHLEN;
3768 mh->m_len = MHLEN;
3769 mh->m_flags |= M_PKTHDR;
3770 /* Get the memory mapping */
3771 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3772 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3773 if (error != 0) {
3774 kprintf("Refresh mbufs: hdr dmamap load"
3775 " failure - %d\n", error);
3776 m_free(mh);
3777 rxbuf->m_head = NULL;
3778 goto update;
3779 }
3780 rxbuf->m_head = mh;
3781 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3782 BUS_DMASYNC_PREREAD);
3783 rxr->rx_base[i].read.hdr_addr =
3784 htole64(hseg[0].ds_addr);
3785
3786no_split:
3787 if (rxbuf->m_pack == NULL) {
3788 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3789 M_PKTHDR, adapter->rx_mbuf_sz);
3790 if (mp == NULL)
3791 goto update;
3792 } else
3793 mp = rxbuf->m_pack;
3794
3795 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3796 /* Get the memory mapping */
3797 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3798 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3799 if (error != 0) {
3800 kprintf("Refresh mbufs: payload dmamap load"
3801 " failure - %d\n", error);
3802 m_free(mp);
3803 rxbuf->m_pack = NULL;
3804 goto update;
3805 }
3806 rxbuf->m_pack = mp;
3807 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3808 BUS_DMASYNC_PREREAD);
3809 rxr->rx_base[i].read.pkt_addr =
3810 htole64(pseg[0].ds_addr);
3811
3812 refreshed = TRUE;
3813 /* Next is precalculated */
3814 i = j;
3815 rxr->next_to_refresh = i;
3816 if (++j == adapter->num_rx_desc)
3817 j = 0;
3818 }
3819update:
3820 if (refreshed) /* Update hardware tail index */
3821 IXGBE_WRITE_REG(&adapter->hw,
3822 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3823 return;
3824}
3825
3826/*********************************************************************
3827 *
3828 * Allocate memory for rx_buffer structures. Since we use one
3829 * rx_buffer per received packet, the maximum number of rx_buffer's
3830 * that we'll need is equal to the number of receive descriptors
3831 * that we've allocated.
3832 *
3833 **********************************************************************/
3834static int
3835ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3836{
3837 struct adapter *adapter = rxr->adapter;
3838 device_t dev = adapter->dev;
3839 struct ixgbe_rx_buf *rxbuf;
3840 int i, bsize, error;
3841
3842 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3843 if (!(rxr->rx_buffers =
3844 (struct ixgbe_rx_buf *) kmalloc(bsize,
3845 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3846 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3847 error = ENOMEM;
3848 goto fail;
3849 }
3850
3851 if ((error = bus_dma_tag_create(NULL, /* parent */
3852 1, 0, /* alignment, bounds */
3853 BUS_SPACE_MAXADDR, /* lowaddr */
3854 BUS_SPACE_MAXADDR, /* highaddr */
3855 NULL, NULL, /* filter, filterarg */
3856 MSIZE, /* maxsize */
3857 1, /* nsegments */
3858 MSIZE, /* maxsegsize */
3859 0, /* flags */
3860 &rxr->htag))) {
3861 device_printf(dev, "Unable to create RX DMA tag\n");
3862 goto fail;
3863 }
3864
3865 if ((error = bus_dma_tag_create(NULL, /* parent */
3866 1, 0, /* alignment, bounds */
3867 BUS_SPACE_MAXADDR, /* lowaddr */
3868 BUS_SPACE_MAXADDR, /* highaddr */
3869 NULL, NULL, /* filter, filterarg */
3870 MJUM16BYTES, /* maxsize */
3871 1, /* nsegments */
3872 MJUM16BYTES, /* maxsegsize */
3873 0, /* flags */
3874 &rxr->ptag))) {
3875 device_printf(dev, "Unable to create RX DMA tag\n");
3876 goto fail;
3877 }
3878
3879 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3880 rxbuf = &rxr->rx_buffers[i];
3881 error = bus_dmamap_create(rxr->htag,
3882 BUS_DMA_NOWAIT, &rxbuf->hmap);
3883 if (error) {
3884 device_printf(dev, "Unable to create RX head map\n");
3885 goto fail;
3886 }
3887 error = bus_dmamap_create(rxr->ptag,
3888 BUS_DMA_NOWAIT, &rxbuf->pmap);
3889 if (error) {
3890 device_printf(dev, "Unable to create RX pkt map\n");
3891 goto fail;
3892 }
3893 }
3894
3895 return (0);
3896
3897fail:
3898 /* Frees all, but can handle partial completion */
3899 ixgbe_free_receive_structures(adapter);
3900 return (error);
3901}
3902
3903/*
3904** Used to detect a descriptor that has
3905** been merged by Hardware RSC.
3906*/
3907static inline u32
3908ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3909{
3910 return (le32toh(rx->wb.lower.lo_dword.data) &
3911 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3912}
3913
3914/*********************************************************************
3915 *
3916 * Initialize Hardware RSC (LRO) feature on 82599
3917 * for an RX ring, this is toggled by the LRO capability
3918 * even though it is transparent to the stack.
3919 *
3920 **********************************************************************/
3921#if 0 /* NET_LRO */
3922static void
3923ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3924{
3925 struct adapter *adapter = rxr->adapter;
3926 struct ixgbe_hw *hw = &adapter->hw;
3927 u32 rscctrl, rdrxctl;
3928
3929 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3930 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3931#ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3932 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3933#endif /* DEV_NETMAP */
3934 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3935 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3936 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3937
3938 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3939 rscctrl |= IXGBE_RSCCTL_RSCEN;
3940 /*
3941 ** Limit the total number of descriptors that
3942 ** can be combined, so it does not exceed 64K
3943 */
3944 if (adapter->rx_mbuf_sz == MCLBYTES)
3945 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3946 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3947 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3948 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3949 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3950 else /* Using 16K cluster */
3951 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3952
3953 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3954
3955 /* Enable TCP header recognition */
3956 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3957 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3958 IXGBE_PSRTYPE_TCPHDR));
3959
3960 /* Disable RSC for ACK packets */
3961 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3962 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3963
3964 rxr->hw_rsc = TRUE;
3965}
3966#endif
3967
3968static void
3969ixgbe_free_receive_ring(struct rx_ring *rxr)
3970{
3971 struct adapter *adapter;
3972 struct ixgbe_rx_buf *rxbuf;
3973 int i;
3974
3975 adapter = rxr->adapter;
3976 for (i = 0; i < adapter->num_rx_desc; i++) {
3977 rxbuf = &rxr->rx_buffers[i];
3978 if (rxbuf->m_head != NULL) {
3979 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3980 BUS_DMASYNC_POSTREAD);
3981 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3982 rxbuf->m_head->m_flags |= M_PKTHDR;
3983 m_freem(rxbuf->m_head);
3984 }
3985 if (rxbuf->m_pack != NULL) {
3986 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3987 BUS_DMASYNC_POSTREAD);
3988 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3989 rxbuf->m_pack->m_flags |= M_PKTHDR;
3990 m_freem(rxbuf->m_pack);
3991 }
3992 rxbuf->m_head = NULL;
3993 rxbuf->m_pack = NULL;
3994 }
3995}
3996
3997
3998/*********************************************************************
3999 *
4000 * Initialize a receive ring and its buffers.
4001 *
4002 **********************************************************************/
4003static int
4004ixgbe_setup_receive_ring(struct rx_ring *rxr)
4005{
4006 struct adapter *adapter;
4007 struct ifnet *ifp;
4008 device_t dev;
4009 struct ixgbe_rx_buf *rxbuf;
4010 bus_dma_segment_t pseg[1], hseg[1];
4011#if 0 /* NET_LRO */
4012 struct lro_ctrl *lro = &rxr->lro;
4013#endif
4014 int rsize, nsegs, error = 0;
4015#ifdef DEV_NETMAP
4016 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4017 struct netmap_slot *slot;
4018#endif /* DEV_NETMAP */
4019
4020 adapter = rxr->adapter;
4021 ifp = adapter->ifp;
4022 dev = adapter->dev;
4023
4024 /* Clear the ring contents */
4025 IXGBE_RX_LOCK(rxr);
4026#ifdef DEV_NETMAP
4027 /* same as in ixgbe_setup_transmit_ring() */
4028 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4029#endif /* DEV_NETMAP */
4030 rsize = roundup2(adapter->num_rx_desc *
4031 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4032 bzero((void *)rxr->rx_base, rsize);
4033
4034 /* Free current RX buffer structs and their mbufs */
4035 ixgbe_free_receive_ring(rxr);
4036
4037 /* Configure header split? */
4038 if (ixgbe_header_split)
4039 rxr->hdr_split = TRUE;
4040
4041 /* Now replenish the mbufs */
4042 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4043 struct mbuf *mh, *mp;
4044
4045 rxbuf = &rxr->rx_buffers[j];
4046#ifdef DEV_NETMAP
4047 /*
4048 * In netmap mode, fill the map and set the buffer
4049 * address in the NIC ring, considering the offset
4050 * between the netmap and NIC rings (see comment in
4051 * ixgbe_setup_transmit_ring() ). No need to allocate
4052 * an mbuf, so end the block with a continue;
4053 */
4054 if (slot) {
4055 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4056 uint64_t paddr;
4057 void *addr;
4058
4059 addr = PNMB(slot + sj, &paddr);
4060 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4061 /* Update descriptor */
4062 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4063 continue;
4064 }
4065#endif /* DEV_NETMAP */
4066 /*
4067 ** Don't allocate mbufs if not
4068 ** doing header split, its wasteful
4069 */
4070 if (rxr->hdr_split == FALSE)
4071 goto skip_head;
4072
4073 /* First the header */
4074 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4075 if (rxbuf->m_head == NULL) {
4076 error = ENOBUFS;
4077 goto fail;
4078 }
4079 m_adj(rxbuf->m_head, ETHER_ALIGN);
4080 mh = rxbuf->m_head;
4081 mh->m_len = mh->m_pkthdr.len = MHLEN;
4082 mh->m_flags |= M_PKTHDR;
4083 /* Get the memory mapping */
4084 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4085 rxbuf->hmap, rxbuf->m_head, hseg, 1,
4086 &nsegs, BUS_DMA_NOWAIT);
4087
4088 if (error != 0) /* Nothing elegant to do here */
4089 goto fail;
4090 bus_dmamap_sync(rxr->htag,
4091 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4092 /* Update descriptor */
4093 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4094
4095skip_head:
4096 /* Now the payload cluster */
4097 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4098 M_PKTHDR, adapter->rx_mbuf_sz);
4099 if (rxbuf->m_pack == NULL) {
4100 error = ENOBUFS;
4101 goto fail;
4102 }
4103 mp = rxbuf->m_pack;
4104 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4105 /* Get the memory mapping */
4106 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4107 rxbuf->pmap, mp, hseg, 1,
4108 &nsegs, BUS_DMA_NOWAIT);
4109 if (error != 0)
4110 goto fail;
4111 bus_dmamap_sync(rxr->ptag,
4112 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4113 /* Update descriptor */
4114 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);