ixgbe(4): remove a lockinit() call
[dragonfly.git] / sys / dev / netif / ixgbe / ixgbe.c
CommitLineData
9407f759
FT
1/******************************************************************************
2
3 Copyright (c) 2001-2012, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
ce451236 33/*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
9407f759
FT
34
35#include "opt_inet.h"
36#include "opt_inet6.h"
37
38#include "ixgbe.h"
39
40/*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43int ixgbe_display_debug_stats = 0;
44
45/*********************************************************************
46 * Driver version
47 *********************************************************************/
ce451236 48char ixgbe_driver_version[] = "2.4.8";
9407f759
FT
49
50/*********************************************************************
51 * PCI Device ID Table
52 *
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
56 *
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
59
60static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
61{
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
ce451236 81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
9407f759
FT
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
ce451236 84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
9407f759
FT
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
87 {0, 0, 0, 0, 0}
88};
89
90/*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
93
94static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
96};
97
98/*********************************************************************
99 * Function prototypes
100 *********************************************************************/
101static int ixgbe_probe(device_t);
102static int ixgbe_attach(device_t);
103static int ixgbe_detach(device_t);
104static int ixgbe_shutdown(device_t);
105static void ixgbe_start(struct ifnet *);
106static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107#if 0 /* __FreeBSD_version >= 800000 */
108static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111static void ixgbe_qflush(struct ifnet *);
112#endif
113static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114static void ixgbe_init(void *);
115static void ixgbe_init_locked(struct adapter *);
116static void ixgbe_stop(void *);
117static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118static int ixgbe_media_change(struct ifnet *);
119static void ixgbe_identify_hardware(struct adapter *);
120static int ixgbe_allocate_pci_resources(struct adapter *);
121static int ixgbe_allocate_msix(struct adapter *);
122static int ixgbe_allocate_legacy(struct adapter *);
123static int ixgbe_allocate_queues(struct adapter *);
124#if 0 /* HAVE_MSIX */
125static int ixgbe_setup_msix(struct adapter *);
126#endif
127static void ixgbe_free_pci_resources(struct adapter *);
128static void ixgbe_local_timer(void *);
129static int ixgbe_setup_interface(device_t, struct adapter *);
130static void ixgbe_config_link(struct adapter *);
131
132static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133static int ixgbe_setup_transmit_structures(struct adapter *);
134static void ixgbe_setup_transmit_ring(struct tx_ring *);
135static void ixgbe_initialize_transmit_units(struct adapter *);
136static void ixgbe_free_transmit_structures(struct adapter *);
137static void ixgbe_free_transmit_buffers(struct tx_ring *);
138
139static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140static int ixgbe_setup_receive_structures(struct adapter *);
141static int ixgbe_setup_receive_ring(struct rx_ring *);
142static void ixgbe_initialize_receive_units(struct adapter *);
143static void ixgbe_free_receive_structures(struct adapter *);
144static void ixgbe_free_receive_buffers(struct rx_ring *);
145#if 0 /* NET_LRO */
146static void ixgbe_setup_hw_rsc(struct rx_ring *);
147#endif
148
149static void ixgbe_enable_intr(struct adapter *);
150static void ixgbe_disable_intr(struct adapter *);
151static void ixgbe_update_stats_counters(struct adapter *);
152static bool ixgbe_txeof(struct tx_ring *);
153static bool ixgbe_rxeof(struct ix_queue *, int);
154static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
155static void ixgbe_set_promisc(struct adapter *);
156static void ixgbe_set_multi(struct adapter *);
157static void ixgbe_update_link_status(struct adapter *);
158static void ixgbe_refresh_mbufs(struct rx_ring *, int);
159static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
160static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
161static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
162static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
163static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
164 struct ixgbe_dma_alloc *, int);
165static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
166static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
167 const char *, int *, int);
168static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
169#if 0 /* NET_TSO */
170static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
171#endif
172static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173static void ixgbe_configure_ivars(struct adapter *);
174static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
175
176static void ixgbe_setup_vlan_hw_support(struct adapter *);
177static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
179
180static void ixgbe_add_hw_stats(struct adapter *adapter);
181
182static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
184 struct mbuf *, u32);
185
186/* Support for pluggable optic modules */
187static bool ixgbe_sfp_probe(struct adapter *);
188static void ixgbe_setup_optics(struct adapter *);
189
190/* Legacy (single vector interrupt handler */
191static void ixgbe_legacy_irq(void *);
192
193/* The MSI/X Interrupt handlers */
194static void ixgbe_msix_que(void *);
195static void ixgbe_msix_link(void *);
196
197/* Deferred interrupt tasklets */
198static void ixgbe_handle_que(void *, int);
199static void ixgbe_handle_link(void *, int);
200static void ixgbe_handle_msf(void *, int);
201static void ixgbe_handle_mod(void *, int);
202
203#ifdef IXGBE_FDIR
204static void ixgbe_atr(struct tx_ring *, struct mbuf *);
205static void ixgbe_reinit_fdir(void *, int);
206#endif
207
208/*********************************************************************
209 * FreeBSD Device Interface Entry Points
210 *********************************************************************/
211
212static device_method_t ixgbe_methods[] = {
213 /* Device interface */
214 DEVMETHOD(device_probe, ixgbe_probe),
215 DEVMETHOD(device_attach, ixgbe_attach),
216 DEVMETHOD(device_detach, ixgbe_detach),
217 DEVMETHOD(device_shutdown, ixgbe_shutdown),
218 {0, 0}
219};
220
221static driver_t ixgbe_driver = {
222 "ix", ixgbe_methods, sizeof(struct adapter),
223};
224
225devclass_t ixgbe_devclass;
226DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
227
228MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
229MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230
231/*
232** TUNEABLE PARAMETERS:
233*/
234
235/*
236** AIM: Adaptive Interrupt Moderation
237** which means that the interrupt rate
238** is varied over time based on the
239** traffic for that interrupt vector
240*/
241static int ixgbe_enable_aim = TRUE;
242TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
243
244static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
245TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
246
247/* How many packets rxeof tries to clean at a time */
248static int ixgbe_rx_process_limit = 128;
249TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
250
9407f759
FT
251/*
252** Smart speed setting, default to on
253** this only works as a compile option
254** right now as its during attach, set
255** this to 'ixgbe_smart_speed_off' to
256** disable.
257*/
258static int ixgbe_smart_speed = ixgbe_smart_speed_on;
259
7e665ea1
FT
260static int ixgbe_msi_enable = 1;
261TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
262
9407f759
FT
263/*
264 * MSIX should be the default for best performance,
265 * but this allows it to be forced off for testing.
266 */
267static int ixgbe_enable_msix = 1;
268TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
269
270/*
271 * Header split: this causes the hardware to DMA
272 * the header into a separate mbuf from the payload,
273 * it can be a performance win in some workloads, but
274 * in others it actually hurts, its off by default.
275 */
276static int ixgbe_header_split = FALSE;
277TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
278
279/*
280 * Number of Queues, can be set to 0,
281 * it then autoconfigures based on the
282 * number of cpus with a max of 8. This
283 * can be overriden manually here.
284 */
285static int ixgbe_num_queues = 0;
286TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
287
288/*
289** Number of TX descriptors per ring,
290** setting higher than RX as this seems
291** the better performing choice.
292*/
293static int ixgbe_txd = PERFORM_TXD;
294TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
295
296/* Number of RX descriptors per ring */
297static int ixgbe_rxd = PERFORM_RXD;
298TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
299
300/* Keep running tab on them for sanity check */
301static int ixgbe_total_ports;
302
303#ifdef IXGBE_FDIR
304/*
305** For Flow Director: this is the
306** number of TX packets we sample
307** for the filter pool, this means
308** every 20th packet will be probed.
309**
310** This feature can be disabled by
311** setting this to 0.
312*/
313static int atr_sample_rate = 20;
314/*
315** Flow Director actually 'steals'
316** part of the packet buffer as its
317** filter pool, this variable controls
318** how much it uses:
319** 0 = 64K, 1 = 128K, 2 = 256K
320*/
321static int fdir_pballoc = 1;
322#endif
323
324#ifdef DEV_NETMAP
325/*
326 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
327 * be a reference on how to implement netmap support in a driver.
328 * Additional comments are in ixgbe_netmap.h .
329 *
330 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
331 * that extend the standard driver.
332 */
333#include <dev/netmap/ixgbe_netmap.h>
334#endif /* DEV_NETMAP */
335
336/*********************************************************************
337 * Device identification routine
338 *
339 * ixgbe_probe determines if the driver should be loaded on
340 * adapter based on PCI vendor/device id of the adapter.
341 *
342 * return BUS_PROBE_DEFAULT on success, positive on failure
343 *********************************************************************/
344
345static int
346ixgbe_probe(device_t dev)
347{
348 ixgbe_vendor_info_t *ent;
349
350 u16 pci_vendor_id = 0;
351 u16 pci_device_id = 0;
352 u16 pci_subvendor_id = 0;
353 u16 pci_subdevice_id = 0;
354 char adapter_name[256];
355
356 INIT_DEBUGOUT("ixgbe_probe: begin");
357
358 pci_vendor_id = pci_get_vendor(dev);
359 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
360 return (ENXIO);
361
362 pci_device_id = pci_get_device(dev);
363 pci_subvendor_id = pci_get_subvendor(dev);
364 pci_subdevice_id = pci_get_subdevice(dev);
365
366 ent = ixgbe_vendor_info_array;
367 while (ent->vendor_id != 0) {
368 if ((pci_vendor_id == ent->vendor_id) &&
369 (pci_device_id == ent->device_id) &&
370
371 ((pci_subvendor_id == ent->subvendor_id) ||
372 (ent->subvendor_id == 0)) &&
373
374 ((pci_subdevice_id == ent->subdevice_id) ||
375 (ent->subdevice_id == 0))) {
376 ksprintf(adapter_name, "%s, Version - %s",
377 ixgbe_strings[ent->index],
378 ixgbe_driver_version);
379 device_set_desc_copy(dev, adapter_name);
380 ++ixgbe_total_ports;
381 return (BUS_PROBE_DEFAULT);
382 }
383 ent++;
384 }
385 return (ENXIO);
386}
387
388/*********************************************************************
389 * Device initialization routine
390 *
391 * The attach entry point is called when the driver is being loaded.
392 * This routine identifies the type of hardware, allocates all resources
393 * and initializes the hardware.
394 *
395 * return 0 on success, positive on failure
396 *********************************************************************/
397
398static int
399ixgbe_attach(device_t dev)
400{
401 struct adapter *adapter;
402 struct ixgbe_hw *hw;
403 int error = 0;
404 u16 csum;
405 u32 ctrl_ext;
406
407 INIT_DEBUGOUT("ixgbe_attach: begin");
408
409 if (resource_disabled("ixgbe", device_get_unit(dev))) {
410 device_printf(dev, "Disabled by device hint\n");
411 return (ENXIO);
412 }
413
414 /* Allocate, clear, and link in our adapter structure */
415 adapter = device_get_softc(dev);
416 adapter->dev = adapter->osdep.dev = dev;
417 hw = &adapter->hw;
418
419 /* Core Lock Init*/
420 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
421 spin_init(&adapter->mcast_spin);
422
423 /* SYSCTL APIs */
424
425 sysctl_ctx_init(&adapter->sysctl_ctx);
426 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
427 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
428 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
429 if (adapter->sysctl_tree == NULL) {
430 device_printf(adapter->dev, "can't add sysctl node\n");
431 return (EINVAL);
432 }
433 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
434 SYSCTL_CHILDREN(adapter->sysctl_tree),
435 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
436 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
437
438 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
439 SYSCTL_CHILDREN(adapter->sysctl_tree),
440 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
441 &ixgbe_enable_aim, 1, "Interrupt Moderation");
442
443 /*
444 ** Allow a kind of speed control by forcing the autoneg
445 ** advertised speed list to only a certain value, this
446 ** supports 1G on 82599 devices, and 100Mb on x540.
447 */
448 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
449 SYSCTL_CHILDREN(adapter->sysctl_tree),
450 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
451 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
452
453 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
454 SYSCTL_CHILDREN(adapter->sysctl_tree),
455 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
456 0, ixgbe_set_thermal_test, "I", "Thermal Test");
457
458 /* Set up the timer callout */
9407f759
FT
459 callout_init(&adapter->timer);
460
461 /* Determine hardware revision */
462 ixgbe_identify_hardware(adapter);
463
464 /* Do base PCI setup - map BAR0 */
465 if (ixgbe_allocate_pci_resources(adapter)) {
466 device_printf(dev, "Allocation of PCI resources failed\n");
467 error = ENXIO;
468 goto err_out;
469 }
470
471 /* Do descriptor calc and sanity checks */
472 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
473 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
474 device_printf(dev, "TXD config issue, using default!\n");
475 adapter->num_tx_desc = DEFAULT_TXD;
476 } else
477 adapter->num_tx_desc = ixgbe_txd;
478
479 /*
480 ** With many RX rings it is easy to exceed the
481 ** system mbuf allocation. Tuning nmbclusters
482 ** can alleviate this.
483 */
484 if (nmbclusters > 0 ) {
485 int s;
486 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
487 if (s > nmbclusters) {
488 device_printf(dev, "RX Descriptors exceed "
489 "system mbuf max, using default instead!\n");
490 ixgbe_rxd = DEFAULT_RXD;
491 }
492 }
493
494 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
495 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
496 device_printf(dev, "RXD config issue, using default!\n");
497 adapter->num_rx_desc = DEFAULT_RXD;
498 } else
499 adapter->num_rx_desc = ixgbe_rxd;
500
501 /* Allocate our TX/RX Queues */
502 if (ixgbe_allocate_queues(adapter)) {
503 error = ENOMEM;
504 goto err_out;
505 }
506
507 /* Allocate multicast array memory. */
508 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
509 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
510 if (adapter->mta == NULL) {
511 device_printf(dev, "Can not allocate multicast setup array\n");
512 error = ENOMEM;
513 goto err_late;
514 }
515
516 /* Initialize the shared code */
517 error = ixgbe_init_shared_code(hw);
518 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
519 /*
520 ** No optics in this port, set up
521 ** so the timer routine will probe
522 ** for later insertion.
523 */
524 adapter->sfp_probe = TRUE;
525 error = 0;
526 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
527 device_printf(dev,"Unsupported SFP+ module detected!\n");
528 error = EIO;
529 goto err_late;
530 } else if (error) {
531 device_printf(dev,"Unable to initialize the shared code\n");
532 error = EIO;
533 goto err_late;
534 }
535
536 /* Make sure we have a good EEPROM before we read from it */
537 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
538 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
539 error = EIO;
540 goto err_late;
541 }
542
9407f759 543 error = ixgbe_init_hw(hw);
ce451236
FT
544 switch (error) {
545 case IXGBE_ERR_EEPROM_VERSION:
9407f759
FT
546 device_printf(dev, "This device is a pre-production adapter/"
547 "LOM. Please be aware there may be issues associated "
548 "with your hardware.\n If you are experiencing problems "
549 "please contact your Intel or hardware representative "
550 "who provided you with this hardware.\n");
ce451236
FT
551 break;
552 case IXGBE_ERR_SFP_NOT_SUPPORTED:
9407f759 553 device_printf(dev,"Unsupported SFP+ Module\n");
9407f759
FT
554 error = EIO;
555 device_printf(dev,"Hardware Initialization Failure\n");
556 goto err_late;
ce451236
FT
557 case IXGBE_ERR_SFP_NOT_PRESENT:
558 device_printf(dev,"No SFP+ Module found\n");
559 /* falls thru */
560 default:
561 break;
9407f759
FT
562 }
563
564 /* Detect and set physical type */
565 ixgbe_setup_optics(adapter);
566
567 if ((adapter->msix > 1) && (ixgbe_enable_msix))
568 error = ixgbe_allocate_msix(adapter);
569 else
570 error = ixgbe_allocate_legacy(adapter);
571 if (error)
572 goto err_late;
573
574 /* Setup OS specific network interface */
575 if (ixgbe_setup_interface(dev, adapter) != 0)
576 goto err_late;
577
578 /* Sysctl for limiting the amount of work done in the taskqueue */
579 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
580 "max number of rx packets to process", &adapter->rx_process_limit,
581 ixgbe_rx_process_limit);
582
583 /* Initialize statistics */
584 ixgbe_update_stats_counters(adapter);
585
586 /* Register for VLAN events */
587 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
588 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
589 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
590 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
591
592 /* Print PCIE bus type/speed/width info */
593 ixgbe_get_bus_info(hw);
594 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
595 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
596 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
597 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
598 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
599 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
600 ("Unknown"));
601
602 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
603 (hw->bus.speed == ixgbe_bus_speed_2500)) {
604 device_printf(dev, "PCI-Express bandwidth available"
605 " for this card\n is not sufficient for"
606 " optimal performance.\n");
607 device_printf(dev, "For optimal performance a x8 "
608 "PCIE, or x4 PCIE 2 slot is required.\n");
609 }
610
611 /* let hardware know driver is loaded */
612 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
613 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
614 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
615
616 ixgbe_add_hw_stats(adapter);
617
618#ifdef DEV_NETMAP
619 ixgbe_netmap_attach(adapter);
620#endif /* DEV_NETMAP */
621 INIT_DEBUGOUT("ixgbe_attach: end");
622 return (0);
623err_late:
624 ixgbe_free_transmit_structures(adapter);
625 ixgbe_free_receive_structures(adapter);
626err_out:
627 if (adapter->ifp != NULL)
628 if_free(adapter->ifp);
629 ixgbe_free_pci_resources(adapter);
630 kfree(adapter->mta, M_DEVBUF);
631 return (error);
632
633}
634
635/*********************************************************************
636 * Device removal routine
637 *
638 * The detach entry point is called when the driver is being removed.
639 * This routine stops the adapter and deallocates all the resources
640 * that were allocated for driver operation.
641 *
642 * return 0 on success, positive on failure
643 *********************************************************************/
644
645static int
646ixgbe_detach(device_t dev)
647{
648 struct adapter *adapter = device_get_softc(dev);
649 struct ix_queue *que = adapter->queues;
650 u32 ctrl_ext;
651
652 INIT_DEBUGOUT("ixgbe_detach: begin");
653
654#ifdef NET_VLAN
655 /* Make sure VLANS are not using driver */
656 if (adapter->ifp->if_vlantrunk != NULL) {
657 device_printf(dev,"Vlan in use, detach first\n");
658 return (EBUSY);
659 }
660#endif
661
662 IXGBE_CORE_LOCK(adapter);
663 ixgbe_stop(adapter);
664 IXGBE_CORE_UNLOCK(adapter);
665
666 for (int i = 0; i < adapter->num_queues; i++, que++) {
667 if (que->tq) {
668 taskqueue_drain(que->tq, &que->que_task);
669 taskqueue_free(que->tq);
670 }
671 }
672
673 /* Drain the Link queue */
674 if (adapter->tq) {
675 taskqueue_drain(adapter->tq, &adapter->link_task);
676 taskqueue_drain(adapter->tq, &adapter->mod_task);
677 taskqueue_drain(adapter->tq, &adapter->msf_task);
678#ifdef IXGBE_FDIR
679 taskqueue_drain(adapter->tq, &adapter->fdir_task);
680#endif
681 taskqueue_free(adapter->tq);
682 }
683
684 /* let hardware know driver is unloading */
685 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
686 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
687 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
688
689 /* Unregister VLAN events */
690#ifdef NET_VLAN
691 if (adapter->vlan_attach != NULL)
692 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
693 if (adapter->vlan_detach != NULL)
694 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
695#endif
696
697 ether_ifdetach(adapter->ifp);
698 callout_stop(&adapter->timer);
699 lockuninit(&adapter->core_lock);
700#ifdef DEV_NETMAP
701 netmap_detach(adapter->ifp);
702#endif /* DEV_NETMAP */
703 ixgbe_free_pci_resources(adapter);
704 bus_generic_detach(dev);
705 if_free(adapter->ifp);
706
707 ixgbe_free_transmit_structures(adapter);
708 ixgbe_free_receive_structures(adapter);
709 kfree(adapter->mta, M_DEVBUF);
710 sysctl_ctx_free(&adapter->sysctl_ctx);
711
712 spin_uninit(&adapter->mcast_spin);
713 IXGBE_CORE_LOCK_DESTROY(adapter);
714 return (0);
715}
716
717/*********************************************************************
718 *
719 * Shutdown entry point
720 *
721 **********************************************************************/
722
723static int
724ixgbe_shutdown(device_t dev)
725{
726 struct adapter *adapter = device_get_softc(dev);
727 IXGBE_CORE_LOCK(adapter);
728 ixgbe_stop(adapter);
729 IXGBE_CORE_UNLOCK(adapter);
730 return (0);
731}
732
733
734/*********************************************************************
735 * Transmit entry point
736 *
737 * ixgbe_start is called by the stack to initiate a transmit.
738 * The driver will remain in this routine as long as there are
739 * packets to transmit and transmit resources are available.
740 * In case resources are not available stack is notified and
741 * the packet is requeued.
742 **********************************************************************/
743
744static void
745ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
746{
747 struct mbuf *m_head;
748 struct adapter *adapter = txr->adapter;
749
750 IXGBE_TX_LOCK_ASSERT(txr);
751
752 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
753 return;
754 if (!adapter->link_active)
755 return;
756
757 while (!ifq_is_empty(&ifp->if_snd)) {
758 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
759 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
760 break;
761 }
762
763 m_head = ifq_dequeue(&ifp->if_snd, NULL);
764 if (m_head == NULL)
765 break;
766
767 if (ixgbe_xmit(txr, &m_head)) {
768#if 0 /* XXX: prepend to an ALTQ queue ? */
769 if (m_head != NULL)
770 IF_PREPEND(&ifp->if_snd, m_head);
771#endif
772 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
773 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
774 break;
775 }
776 /* Send a copy of the frame to the BPF listener */
777 ETHER_BPF_MTAP(ifp, m_head);
778
779 /* Set watchdog on */
780 txr->watchdog_time = ticks;
781 txr->queue_status = IXGBE_QUEUE_WORKING;
782
783 }
784 return;
785}
786
787/*
788 * Legacy TX start - called by the stack, this
789 * always uses the first tx ring, and should
790 * not be used with multiqueue tx enabled.
791 */
792static void
793ixgbe_start(struct ifnet *ifp)
794{
795 struct adapter *adapter = ifp->if_softc;
796 struct tx_ring *txr = adapter->tx_rings;
797
798 if (ifp->if_flags & IFF_RUNNING) {
799 IXGBE_TX_LOCK(txr);
800 ixgbe_start_locked(txr, ifp);
801 IXGBE_TX_UNLOCK(txr);
802 }
803 return;
804}
805
806#if 0 /* __FreeBSD_version >= 800000 */
807/*
808** Multiqueue Transmit driver
809**
810*/
811static int
812ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
813{
814 struct adapter *adapter = ifp->if_softc;
815 struct ix_queue *que;
816 struct tx_ring *txr;
817 int i = 0, err = 0;
818
819 /* Which queue to use */
820 if ((m->m_flags & M_FLOWID) != 0)
821 i = m->m_pkthdr.flowid % adapter->num_queues;
822 else
823 i = curcpu % adapter->num_queues;
824
825 txr = &adapter->tx_rings[i];
826 que = &adapter->queues[i];
827
828 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
829 IXGBE_TX_TRYLOCK(txr)) {
830 err = ixgbe_mq_start_locked(ifp, txr, m);
831 IXGBE_TX_UNLOCK(txr);
832 } else {
833 err = drbr_enqueue(ifp, txr->br, m);
834 taskqueue_enqueue(que->tq, &que->que_task);
835 }
836
837 return (err);
838}
839
840static int
841ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
842{
843 struct adapter *adapter = txr->adapter;
844 struct mbuf *next;
845 int enqueued, err = 0;
846
847 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
848 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
849 adapter->link_active == 0) {
850 if (m != NULL)
851 err = drbr_enqueue(ifp, txr->br, m);
852 return (err);
853 }
854
855 enqueued = 0;
856 if (m == NULL) {
857 next = drbr_dequeue(ifp, txr->br);
858 } else if (drbr_needs_enqueue(ifp, txr->br)) {
859 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
860 return (err);
861 next = drbr_dequeue(ifp, txr->br);
862 } else
863 next = m;
864
865 /* Process the queue */
866 while (next != NULL) {
867 if ((err = ixgbe_xmit(txr, &next)) != 0) {
868 if (next != NULL)
869 err = drbr_enqueue(ifp, txr->br, next);
870 break;
871 }
872 enqueued++;
873 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
874 /* Send a copy of the frame to the BPF listener */
875 ETHER_BPF_MTAP(ifp, next);
876 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
877 break;
878 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
879 ixgbe_txeof(txr);
880 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
881 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
882 break;
883 }
884 next = drbr_dequeue(ifp, txr->br);
885 }
886
887 if (enqueued > 0) {
888 /* Set watchdog on */
889 txr->queue_status |= IXGBE_QUEUE_WORKING;
890 txr->watchdog_time = ticks;
891 }
892
893 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
894 ixgbe_txeof(txr);
895
896 return (err);
897}
898
899/*
900** Flush all ring buffers
901*/
902static void
903ixgbe_qflush(struct ifnet *ifp)
904{
905 struct adapter *adapter = ifp->if_softc;
906 struct tx_ring *txr = adapter->tx_rings;
907 struct mbuf *m;
908
909 for (int i = 0; i < adapter->num_queues; i++, txr++) {
910 IXGBE_TX_LOCK(txr);
911 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
912 m_freem(m);
913 IXGBE_TX_UNLOCK(txr);
914 }
915 if_qflush(ifp);
916}
917#endif /* __FreeBSD_version >= 800000 */
918
919/*********************************************************************
920 * Ioctl entry point
921 *
922 * ixgbe_ioctl is called when the user wants to configure the
923 * interface.
924 *
925 * return 0 on success, positive on failure
926 **********************************************************************/
927
928static int
929ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
930{
931 struct adapter *adapter = ifp->if_softc;
932 struct ifreq *ifr = (struct ifreq *) data;
933#if defined(INET) || defined(INET6)
934 struct ifaddr *ifa = (struct ifaddr *)data;
935 bool avoid_reset = FALSE;
936#endif
937 int error = 0;
938
939 switch (command) {
940
941 case SIOCSIFADDR:
942#ifdef INET
943 if (ifa->ifa_addr->sa_family == AF_INET)
944 avoid_reset = TRUE;
945#endif
946#ifdef INET6
947 if (ifa->ifa_addr->sa_family == AF_INET6)
948 avoid_reset = TRUE;
949#endif
950#if defined(INET) || defined(INET6)
951 /*
952 ** Calling init results in link renegotiation,
953 ** so we avoid doing it when possible.
954 */
955 if (avoid_reset) {
956 ifp->if_flags |= IFF_UP;
957 if (!(ifp->if_flags & IFF_RUNNING))
958 ixgbe_init(adapter);
959 if (!(ifp->if_flags & IFF_NOARP))
960 arp_ifinit(ifp, ifa);
961 } else
962 error = ether_ioctl(ifp, command, data);
963#endif
964 break;
965 case SIOCSIFMTU:
966 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
967 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
968 error = EINVAL;
969 } else {
970 IXGBE_CORE_LOCK(adapter);
971 ifp->if_mtu = ifr->ifr_mtu;
972 adapter->max_frame_size =
973 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
974 ixgbe_init_locked(adapter);
975 IXGBE_CORE_UNLOCK(adapter);
976 }
977 break;
978 case SIOCSIFFLAGS:
979 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
980 IXGBE_CORE_LOCK(adapter);
981 if (ifp->if_flags & IFF_UP) {
982 if ((ifp->if_flags & IFF_RUNNING)) {
983 if ((ifp->if_flags ^ adapter->if_flags) &
984 (IFF_PROMISC | IFF_ALLMULTI)) {
985 ixgbe_set_promisc(adapter);
986 }
987 } else
988 ixgbe_init_locked(adapter);
989 } else
990 if (ifp->if_flags & IFF_RUNNING)
991 ixgbe_stop(adapter);
992 adapter->if_flags = ifp->if_flags;
993 IXGBE_CORE_UNLOCK(adapter);
994 break;
995 case SIOCADDMULTI:
996 case SIOCDELMULTI:
997 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
998 if (ifp->if_flags & IFF_RUNNING) {
999 IXGBE_CORE_LOCK(adapter);
1000 ixgbe_disable_intr(adapter);
1001 ixgbe_set_multi(adapter);
1002 ixgbe_enable_intr(adapter);
1003 IXGBE_CORE_UNLOCK(adapter);
1004 }
1005 break;
1006 case SIOCSIFMEDIA:
1007 case SIOCGIFMEDIA:
1008 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1009 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1010 break;
1011 case SIOCSIFCAP:
1012 {
1013 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1014 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1015 if (mask & IFCAP_HWCSUM)
1016 ifp->if_capenable ^= IFCAP_HWCSUM;
1017#if 0 /* NET_TSO */
1018 if (mask & IFCAP_TSO4)
1019 ifp->if_capenable ^= IFCAP_TSO4;
1020 if (mask & IFCAP_TSO6)
1021 ifp->if_capenable ^= IFCAP_TSO6;
1022#endif
1023#if 0 /* NET_LRO */
1024 if (mask & IFCAP_LRO)
1025 ifp->if_capenable ^= IFCAP_LRO;
1026#endif
1027 if (mask & IFCAP_VLAN_HWTAGGING)
1028 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1029 if (mask & IFCAP_VLAN_HWFILTER)
1030 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1031#if 0 /* NET_TSO */
1032 if (mask & IFCAP_VLAN_HWTSO)
1033 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1034#endif
1035 if (ifp->if_flags & IFF_RUNNING) {
1036 IXGBE_CORE_LOCK(adapter);
1037 ixgbe_init_locked(adapter);
1038 IXGBE_CORE_UNLOCK(adapter);
1039 }
1040#if 0
1041 VLAN_CAPABILITIES(ifp);
1042#endif
1043 break;
1044 }
1045
1046 default:
1047 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1048 error = ether_ioctl(ifp, command, data);
1049 break;
1050 }
1051
1052 return (error);
1053}
1054
1055/*********************************************************************
1056 * Init entry point
1057 *
1058 * This routine is used in two ways. It is used by the stack as
1059 * init entry point in network interface structure. It is also used
1060 * by the driver as a hw/sw initialization routine to get to a
1061 * consistent state.
1062 *
1063 * return 0 on success, positive on failure
1064 **********************************************************************/
1065#define IXGBE_MHADD_MFS_SHIFT 16
1066
1067static void
1068ixgbe_init_locked(struct adapter *adapter)
1069{
1070 struct ifnet *ifp = adapter->ifp;
1071 device_t dev = adapter->dev;
1072 struct ixgbe_hw *hw = &adapter->hw;
1073 u32 k, txdctl, mhadd, gpie;
1074 u32 rxdctl, rxctrl;
1075
1076 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1077 INIT_DEBUGOUT("ixgbe_init: begin");
1078 hw->adapter_stopped = FALSE;
1079 ixgbe_stop_adapter(hw);
1080 callout_stop(&adapter->timer);
1081
1082 /* reprogram the RAR[0] in case user changed it. */
1083 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1084
1085 /* Get the latest mac address, User can use a LAA */
1086 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1087 IXGBE_ETH_LENGTH_OF_ADDRESS);
1088 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1089 hw->addr_ctrl.rar_used_count = 1;
1090
1091 /* Set the various hardware offload abilities */
1092 ifp->if_hwassist = 0;
1093#if 0 /* NET_TSO */
1094 if (ifp->if_capenable & IFCAP_TSO)
1095 ifp->if_hwassist |= CSUM_TSO;
1096#endif
1097 if (ifp->if_capenable & IFCAP_TXCSUM) {
1098 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1099#if 0
1100 if (hw->mac.type != ixgbe_mac_82598EB)
1101 ifp->if_hwassist |= CSUM_SCTP;
1102#endif
1103 }
1104
1105 /* Prepare transmit descriptors and buffers */
1106 if (ixgbe_setup_transmit_structures(adapter)) {
1107 device_printf(dev,"Could not setup transmit structures\n");
1108 ixgbe_stop(adapter);
1109 return;
1110 }
1111
1112 ixgbe_init_hw(hw);
1113 ixgbe_initialize_transmit_units(adapter);
1114
1115 /* Setup Multicast table */
1116 ixgbe_set_multi(adapter);
1117
1118 /*
1119 ** Determine the correct mbuf pool
1120 ** for doing jumbo/headersplit
1121 */
1122 if (adapter->max_frame_size <= 2048)
1123 adapter->rx_mbuf_sz = MCLBYTES;
1124 else if (adapter->max_frame_size <= 4096)
1125 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1126 else if (adapter->max_frame_size <= 9216)
1127 adapter->rx_mbuf_sz = MJUM9BYTES;
1128 else
1129 adapter->rx_mbuf_sz = MJUM16BYTES;
1130
1131 /* Prepare receive descriptors and buffers */
1132 if (ixgbe_setup_receive_structures(adapter)) {
1133 device_printf(dev,"Could not setup receive structures\n");
1134 ixgbe_stop(adapter);
1135 return;
1136 }
1137
1138 /* Configure RX settings */
1139 ixgbe_initialize_receive_units(adapter);
1140
1141 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1142
1143 /* Enable Fan Failure Interrupt */
1144 gpie |= IXGBE_SDP1_GPIEN;
1145
1146 /* Add for Module detection */
1147 if (hw->mac.type == ixgbe_mac_82599EB)
1148 gpie |= IXGBE_SDP2_GPIEN;
1149
1150 /* Thermal Failure Detection */
1151 if (hw->mac.type == ixgbe_mac_X540)
1152 gpie |= IXGBE_SDP0_GPIEN;
1153
1154 if (adapter->msix > 1) {
1155 /* Enable Enhanced MSIX mode */
1156 gpie |= IXGBE_GPIE_MSIX_MODE;
1157 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1158 IXGBE_GPIE_OCD;
1159 }
1160 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1161
1162 /* Set MTU size */
1163 if (ifp->if_mtu > ETHERMTU) {
1164 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1165 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1166 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1167 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1168 }
1169
1170 /* Now enable all the queues */
1171
1172 for (int i = 0; i < adapter->num_queues; i++) {
1173 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1174 txdctl |= IXGBE_TXDCTL_ENABLE;
1175 /* Set WTHRESH to 8, burst writeback */
1176 txdctl |= (8 << 16);
1177 /*
1178 * When the internal queue falls below PTHRESH (32),
1179 * start prefetching as long as there are at least
1180 * HTHRESH (1) buffers ready. The values are taken
1181 * from the Intel linux driver 3.8.21.
1182 * Prefetching enables tx line rate even with 1 queue.
1183 */
1184 txdctl |= (32 << 0) | (1 << 8);
1185 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1186 }
1187
1188 for (int i = 0; i < adapter->num_queues; i++) {
1189 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1190 if (hw->mac.type == ixgbe_mac_82598EB) {
1191 /*
1192 ** PTHRESH = 21
1193 ** HTHRESH = 4
1194 ** WTHRESH = 8
1195 */
1196 rxdctl &= ~0x3FFFFF;
1197 rxdctl |= 0x080420;
1198 }
1199 rxdctl |= IXGBE_RXDCTL_ENABLE;
1200 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1201 for (k = 0; k < 10; k++) {
1202 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1203 IXGBE_RXDCTL_ENABLE)
1204 break;
1205 else
1206 msec_delay(1);
1207 }
1208 wmb();
1209#ifdef DEV_NETMAP
1210 /*
1211 * In netmap mode, we must preserve the buffers made
1212 * available to userspace before the if_init()
1213 * (this is true by default on the TX side, because
1214 * init makes all buffers available to userspace).
1215 *
1216 * netmap_reset() and the device specific routines
1217 * (e.g. ixgbe_setup_receive_rings()) map these
1218 * buffers at the end of the NIC ring, so here we
1219 * must set the RDT (tail) register to make sure
1220 * they are not overwritten.
1221 *
1222 * In this driver the NIC ring starts at RDH = 0,
1223 * RDT points to the last slot available for reception (?),
1224 * so RDT = num_rx_desc - 1 means the whole ring is available.
1225 */
1226 if (ifp->if_capenable & IFCAP_NETMAP) {
1227 struct netmap_adapter *na = NA(adapter->ifp);
1228 struct netmap_kring *kring = &na->rx_rings[i];
1229 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1230
1231 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1232 } else
1233#endif /* DEV_NETMAP */
1234 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1235 }
1236
1237 /* Set up VLAN support and filter */
1238 ixgbe_setup_vlan_hw_support(adapter);
1239
1240 /* Enable Receive engine */
1241 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1242 if (hw->mac.type == ixgbe_mac_82598EB)
1243 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1244 rxctrl |= IXGBE_RXCTRL_RXEN;
1245 ixgbe_enable_rx_dma(hw, rxctrl);
1246
1247 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1248
1249 /* Set up MSI/X routing */
1250 if (ixgbe_enable_msix) {
1251 ixgbe_configure_ivars(adapter);
1252 /* Set up auto-mask */
1253 if (hw->mac.type == ixgbe_mac_82598EB)
1254 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1255 else {
1256 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1257 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1258 }
1259 } else { /* Simple settings for Legacy/MSI */
1260 ixgbe_set_ivar(adapter, 0, 0, 0);
1261 ixgbe_set_ivar(adapter, 0, 0, 1);
1262 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1263 }
1264
1265#ifdef IXGBE_FDIR
1266 /* Init Flow director */
1267 if (hw->mac.type != ixgbe_mac_82598EB) {
ce451236 1268 u32 hdrm = 32 << fdir_pballoc;
9407f759
FT
1269
1270 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1271 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1272 }
1273#endif
1274
1275 /*
1276 ** Check on any SFP devices that
1277 ** need to be kick-started
1278 */
1279 if (hw->phy.type == ixgbe_phy_none) {
1280 int err = hw->phy.ops.identify(hw);
1281 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1282 device_printf(dev,
1283 "Unsupported SFP+ module type was detected.\n");
1284 return;
1285 }
1286 }
1287
1288 /* Set moderation on the Link interrupt */
1289 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1290
1291 /* Config/Enable Link */
1292 ixgbe_config_link(adapter);
1293
ce451236
FT
1294 /* Hardware Packet Buffer & Flow Control setup */
1295 {
1296 u32 rxpb, frame, size, tmp;
1297
1298 frame = adapter->max_frame_size;
1299
1300 /* Calculate High Water */
1301 if (hw->mac.type == ixgbe_mac_X540)
1302 tmp = IXGBE_DV_X540(frame, frame);
1303 else
1304 tmp = IXGBE_DV(frame, frame);
1305 size = IXGBE_BT2KB(tmp);
1306 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1307 hw->fc.high_water[0] = rxpb - size;
1308
1309 /* Now calculate Low Water */
1310 if (hw->mac.type == ixgbe_mac_X540)
1311 tmp = IXGBE_LOW_DV_X540(frame);
1312 else
1313 tmp = IXGBE_LOW_DV(frame);
1314 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1315
1316 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1317 hw->fc.pause_time = IXGBE_FC_PAUSE;
1318 hw->fc.send_xon = TRUE;
1319 }
1320 /* Initialize the FC settings */
1321 ixgbe_start_hw(hw);
1322
9407f759
FT
1323 /* And now turn on interrupts */
1324 ixgbe_enable_intr(adapter);
1325
1326 /* Now inform the stack we're ready */
1327 ifp->if_flags |= IFF_RUNNING;
1328 ifp->if_flags &= ~IFF_OACTIVE;
1329
1330 return;
1331}
1332
1333static void
1334ixgbe_init(void *arg)
1335{
1336 struct adapter *adapter = arg;
1337
1338 IXGBE_CORE_LOCK(adapter);
1339 ixgbe_init_locked(adapter);
1340 IXGBE_CORE_UNLOCK(adapter);
1341 return;
1342}
1343
1344
1345/*
1346**
1347** MSIX Interrupt Handlers and Tasklets
1348**
1349*/
1350
1351static inline void
1352ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1353{
1354 struct ixgbe_hw *hw = &adapter->hw;
1355 u64 queue = (u64)(1 << vector);
1356 u32 mask;
1357
1358 if (hw->mac.type == ixgbe_mac_82598EB) {
1359 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1360 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1361 } else {
1362 mask = (queue & 0xFFFFFFFF);
1363 if (mask)
1364 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1365 mask = (queue >> 32);
1366 if (mask)
1367 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1368 }
1369}
1370
1371static inline void
1372ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1373{
1374 struct ixgbe_hw *hw = &adapter->hw;
1375 u64 queue = (u64)(1 << vector);
1376 u32 mask;
1377
1378 if (hw->mac.type == ixgbe_mac_82598EB) {
1379 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1380 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1381 } else {
1382 mask = (queue & 0xFFFFFFFF);
1383 if (mask)
1384 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1385 mask = (queue >> 32);
1386 if (mask)
1387 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1388 }
1389}
1390
1391static inline void
1392ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1393{
1394 u32 mask;
1395
1396 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1397 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1398 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1399 } else {
1400 mask = (queues & 0xFFFFFFFF);
1401 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1402 mask = (queues >> 32);
1403 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1404 }
1405}
1406
1407
1408static void
1409ixgbe_handle_que(void *context, int pending)
1410{
1411 struct ix_queue *que = context;
1412 struct adapter *adapter = que->adapter;
1413 struct tx_ring *txr = que->txr;
1414 struct ifnet *ifp = adapter->ifp;
1415 bool more;
1416
1417 if (ifp->if_flags & IFF_RUNNING) {
1418 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1419 IXGBE_TX_LOCK(txr);
1420 ixgbe_txeof(txr);
1421#if 0 /*__FreeBSD_version >= 800000*/
1422 if (!drbr_empty(ifp, txr->br))
1423 ixgbe_mq_start_locked(ifp, txr, NULL);
1424#else
1425 if (!ifq_is_empty(&ifp->if_snd))
1426 ixgbe_start_locked(txr, ifp);
1427#endif
1428 IXGBE_TX_UNLOCK(txr);
1429 if (more) {
1430 taskqueue_enqueue(que->tq, &que->que_task);
1431 return;
1432 }
1433 }
1434
1435 /* Reenable this interrupt */
1436 ixgbe_enable_queue(adapter, que->msix);
1437 return;
1438}
1439
1440
1441/*********************************************************************
1442 *
1443 * Legacy Interrupt Service routine
1444 *
1445 **********************************************************************/
1446
1447static void
1448ixgbe_legacy_irq(void *arg)
1449{
1450 struct ix_queue *que = arg;
1451 struct adapter *adapter = que->adapter;
1452 struct ixgbe_hw *hw = &adapter->hw;
1453 struct tx_ring *txr = adapter->tx_rings;
1454 bool more_tx, more_rx;
1455 u32 reg_eicr, loop = MAX_LOOP;
1456
1457
1458 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1459
1460 ++que->irqs;
1461 if (reg_eicr == 0) {
1462 ixgbe_enable_intr(adapter);
1463 return;
1464 }
1465
1466 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1467
1468 IXGBE_TX_LOCK(txr);
1469 do {
1470 more_tx = ixgbe_txeof(txr);
1471 } while (loop-- && more_tx);
1472 IXGBE_TX_UNLOCK(txr);
1473
1474 if (more_rx || more_tx)
1475 taskqueue_enqueue(que->tq, &que->que_task);
1476
1477 /* Check for fan failure */
1478 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1479 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1480 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1481 "REPLACE IMMEDIATELY!!\n");
1482 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1483 }
1484
1485 /* Link status change */
1486 if (reg_eicr & IXGBE_EICR_LSC)
1487 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1488
1489 ixgbe_enable_intr(adapter);
1490 return;
1491}
1492
1493
1494/*********************************************************************
1495 *
1496 * MSIX Queue Interrupt Service routine
1497 *
1498 **********************************************************************/
1499void
1500ixgbe_msix_que(void *arg)
1501{
1502 struct ix_queue *que = arg;
1503 struct adapter *adapter = que->adapter;
1504 struct tx_ring *txr = que->txr;
1505 struct rx_ring *rxr = que->rxr;
1506 bool more_tx, more_rx;
1507 u32 newitr = 0;
1508
1509 ixgbe_disable_queue(adapter, que->msix);
1510 ++que->irqs;
1511
1512 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1513
1514 IXGBE_TX_LOCK(txr);
1515 more_tx = ixgbe_txeof(txr);
1516 /*
1517 ** Make certain that if the stack
1518 ** has anything queued the task gets
1519 ** scheduled to handle it.
1520 */
1521#if 0
1522#if __FreeBSD_version < 800000
1523 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1524#else
1525 if (!drbr_empty(adapter->ifp, txr->br))
1526#endif
1527#endif
1528 if (!ifq_is_empty(&adapter->ifp->if_snd))
1529 more_tx = 1;
1530 IXGBE_TX_UNLOCK(txr);
1531
1532 /* Do AIM now? */
1533
1534 if (ixgbe_enable_aim == FALSE)
1535 goto no_calc;
1536 /*
1537 ** Do Adaptive Interrupt Moderation:
1538 ** - Write out last calculated setting
1539 ** - Calculate based on average size over
1540 ** the last interval.
1541 */
1542 if (que->eitr_setting)
1543 IXGBE_WRITE_REG(&adapter->hw,
1544 IXGBE_EITR(que->msix), que->eitr_setting);
1545
1546 que->eitr_setting = 0;
1547
1548 /* Idle, do nothing */
1549 if ((txr->bytes == 0) && (rxr->bytes == 0))
1550 goto no_calc;
1551
1552 if ((txr->bytes) && (txr->packets))
1553 newitr = txr->bytes/txr->packets;
1554 if ((rxr->bytes) && (rxr->packets))
1555 newitr = max(newitr,
1556 (rxr->bytes / rxr->packets));
1557 newitr += 24; /* account for hardware frame, crc */
1558
1559 /* set an upper boundary */
1560 newitr = min(newitr, 3000);
1561
1562 /* Be nice to the mid range */
1563 if ((newitr > 300) && (newitr < 1200))
1564 newitr = (newitr / 3);
1565 else
1566 newitr = (newitr / 2);
1567
1568 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1569 newitr |= newitr << 16;
1570 else
1571 newitr |= IXGBE_EITR_CNT_WDIS;
1572
1573 /* save for next interrupt */
1574 que->eitr_setting = newitr;
1575
1576 /* Reset state */
1577 txr->bytes = 0;
1578 txr->packets = 0;
1579 rxr->bytes = 0;
1580 rxr->packets = 0;
1581
1582no_calc:
1583 if (more_tx || more_rx)
1584 taskqueue_enqueue(que->tq, &que->que_task);
1585 else /* Reenable this interrupt */
1586 ixgbe_enable_queue(adapter, que->msix);
1587 return;
1588}
1589
1590
1591static void
1592ixgbe_msix_link(void *arg)
1593{
1594 struct adapter *adapter = arg;
1595 struct ixgbe_hw *hw = &adapter->hw;
1596 u32 reg_eicr;
1597
1598 ++adapter->link_irq;
1599
1600 /* First get the cause */
1601 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1602 /* Clear interrupt with write */
1603 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1604
1605 /* Link status change */
1606 if (reg_eicr & IXGBE_EICR_LSC)
1607 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1608
1609 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1610#ifdef IXGBE_FDIR
1611 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1612 /* This is probably overkill :) */
1613 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1614 return;
ce451236
FT
1615 /* Disable the interrupt */
1616 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
9407f759
FT
1617 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1618 } else
1619#endif
1620 if (reg_eicr & IXGBE_EICR_ECC) {
1621 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1622 "Please Reboot!!\n");
1623 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1624 } else
1625
1626 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1627 /* Clear the interrupt */
1628 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1629 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1630 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1631 /* Clear the interrupt */
1632 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1633 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1634 }
1635 }
1636
1637 /* Check for fan failure */
1638 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1639 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1640 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1641 "REPLACE IMMEDIATELY!!\n");
1642 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1643 }
1644
1645 /* Check for over temp condition */
1646 if ((hw->mac.type == ixgbe_mac_X540) &&
1647 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1648 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1649 "PHY IS SHUT DOWN!!\n");
1650 device_printf(adapter->dev, "System shutdown required\n");
1651 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1652 }
1653
1654 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1655 return;
1656}
1657
1658/*********************************************************************
1659 *
1660 * Media Ioctl callback
1661 *
1662 * This routine is called whenever the user queries the status of
1663 * the interface using ifconfig.
1664 *
1665 **********************************************************************/
1666static void
1667ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1668{
1669 struct adapter *adapter = ifp->if_softc;
1670
1671 INIT_DEBUGOUT("ixgbe_media_status: begin");
1672 IXGBE_CORE_LOCK(adapter);
1673 ixgbe_update_link_status(adapter);
1674
1675 ifmr->ifm_status = IFM_AVALID;
1676 ifmr->ifm_active = IFM_ETHER;
1677
1678 if (!adapter->link_active) {
1679 IXGBE_CORE_UNLOCK(adapter);
1680 return;
1681 }
1682
1683 ifmr->ifm_status |= IFM_ACTIVE;
1684
1685 switch (adapter->link_speed) {
1686 case IXGBE_LINK_SPEED_100_FULL:
1687 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1688 break;
1689 case IXGBE_LINK_SPEED_1GB_FULL:
1690 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1691 break;
1692 case IXGBE_LINK_SPEED_10GB_FULL:
1693 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1694 break;
1695 }
1696
1697 IXGBE_CORE_UNLOCK(adapter);
1698
1699 return;
1700}
1701
1702/*********************************************************************
1703 *
1704 * Media Ioctl callback
1705 *
1706 * This routine is called when the user changes speed/duplex using
1707 * media/mediopt option with ifconfig.
1708 *
1709 **********************************************************************/
1710static int
1711ixgbe_media_change(struct ifnet * ifp)
1712{
1713 struct adapter *adapter = ifp->if_softc;
1714 struct ifmedia *ifm = &adapter->media;
1715
1716 INIT_DEBUGOUT("ixgbe_media_change: begin");
1717
1718 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1719 return (EINVAL);
1720
1721 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1722 case IFM_AUTO:
1723 adapter->hw.phy.autoneg_advertised =
1724 IXGBE_LINK_SPEED_100_FULL |
1725 IXGBE_LINK_SPEED_1GB_FULL |
1726 IXGBE_LINK_SPEED_10GB_FULL;
1727 break;
1728 default:
1729 device_printf(adapter->dev, "Only auto media type\n");
1730 return (EINVAL);
1731 }
1732
1733 return (0);
1734}
1735
1736/*********************************************************************
1737 *
1738 * This routine maps the mbufs to tx descriptors, allowing the
1739 * TX engine to transmit the packets.
1740 * - return 0 on success, positive on failure
1741 *
1742 **********************************************************************/
1743
1744static int
1745ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1746{
1747 struct adapter *adapter = txr->adapter;
1748 u32 olinfo_status = 0, cmd_type_len;
1749 u32 paylen = 0;
1750 int i, j, error, nsegs;
1751 int first, last = 0;
1752 struct mbuf *m_head;
1753 bus_dma_segment_t segs[1];
1754 bus_dmamap_t map;
1755 struct ixgbe_tx_buf *txbuf;
1756 union ixgbe_adv_tx_desc *txd = NULL;
1757
1758 m_head = *m_headp;
1759
1760 /* Basic descriptor defines */
1761 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1762 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1763
1764 if (m_head->m_flags & M_VLANTAG)
1765 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1766
1767 /*
1768 * Important to capture the first descriptor
1769 * used because it will contain the index of
1770 * the one we tell the hardware to report back
1771 */
1772 first = txr->next_avail_desc;
1773 txbuf = &txr->tx_buffers[first];
1774 map = txbuf->map;
1775
1776 /*
1777 * Map the packet for DMA.
1778 */
1779 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1780 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1781
1782 if (error == EFBIG) {
1783 struct mbuf *m;
1784
1785 m = m_defrag(*m_headp, MB_DONTWAIT);
1786 if (m == NULL) {
1787 adapter->mbuf_defrag_failed++;
1788 m_freem(*m_headp);
1789 *m_headp = NULL;
1790 return (ENOBUFS);
1791 }
1792 *m_headp = m;
1793
1794 /* Try it again */
1795 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1796 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1797
1798 if (error == ENOMEM) {
1799 adapter->no_tx_dma_setup++;
1800 return (error);
1801 } else if (error != 0) {
1802 adapter->no_tx_dma_setup++;
1803 m_freem(*m_headp);
1804 *m_headp = NULL;
1805 return (error);
1806 }
1807 } else if (error == ENOMEM) {
1808 adapter->no_tx_dma_setup++;
1809 return (error);
1810 } else if (error != 0) {
1811 adapter->no_tx_dma_setup++;
1812 m_freem(*m_headp);
1813 *m_headp = NULL;
1814 return (error);
1815 }
1816
1817 /* Make certain there are enough descriptors */
1818 if (nsegs > txr->tx_avail - 2) {
1819 txr->no_desc_avail++;
1820 error = ENOBUFS;
1821 goto xmit_fail;
1822 }
1823 m_head = *m_headp;
1824
1825 /*
1826 ** Set up the appropriate offload context
1827 ** this becomes the first descriptor of
1828 ** a packet.
1829 */
1830#if 0 /* NET_TSO */
1831 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1832 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1833 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1834 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1835 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1836 ++adapter->tso_tx;
1837 } else
1838 return (ENXIO);
1839 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1840#endif
1841 if (ixgbe_tx_ctx_setup(txr, m_head))
1842 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1843
1844#ifdef IXGBE_IEEE1588
1845 /* This is changing soon to an mtag detection */
1846 if (we detect this mbuf has a TSTAMP mtag)
1847 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1848#endif
1849
1850#ifdef IXGBE_FDIR
1851 /* Do the flow director magic */
1852 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1853 ++txr->atr_count;
1854 if (txr->atr_count >= atr_sample_rate) {
1855 ixgbe_atr(txr, m_head);
1856 txr->atr_count = 0;
1857 }
1858 }
1859#endif
1860 /* Record payload length */
1861 if (paylen == 0)
1862 olinfo_status |= m_head->m_pkthdr.len <<
1863 IXGBE_ADVTXD_PAYLEN_SHIFT;
1864
1865 i = txr->next_avail_desc;
1866 for (j = 0; j < nsegs; j++) {
1867 bus_size_t seglen;
1868 bus_addr_t segaddr;
1869
1870 txbuf = &txr->tx_buffers[i];
1871 txd = &txr->tx_base[i];
1872 seglen = segs[j].ds_len;
1873 segaddr = htole64(segs[j].ds_addr);
1874
1875 txd->read.buffer_addr = segaddr;
1876 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1877 cmd_type_len |seglen);
1878 txd->read.olinfo_status = htole32(olinfo_status);
1879 last = i; /* descriptor that will get completion IRQ */
1880
1881 if (++i == adapter->num_tx_desc)
1882 i = 0;
1883
1884 txbuf->m_head = NULL;
1885 txbuf->eop_index = -1;
1886 }
1887
1888 txd->read.cmd_type_len |=
1889 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1890 txr->tx_avail -= nsegs;
1891 txr->next_avail_desc = i;
1892
1893 txbuf->m_head = m_head;
1894 /* Swap the dma map between the first and last descriptor */
1895 txr->tx_buffers[first].map = txbuf->map;
1896 txbuf->map = map;
1897 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1898
1899 /* Set the index of the descriptor that will be marked done */
1900 txbuf = &txr->tx_buffers[first];
1901 txbuf->eop_index = last;
1902
1903 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1904 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1905 /*
1906 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1907 * hardware that this frame is available to transmit.
1908 */
1909 ++txr->total_packets;
1910 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1911
1912 return (0);
1913
1914xmit_fail:
1915 bus_dmamap_unload(txr->txtag, txbuf->map);
1916 return (error);
1917
1918}
1919
1920static void
1921ixgbe_set_promisc(struct adapter *adapter)
1922{
1923 u_int32_t reg_rctl;
1924 struct ifnet *ifp = adapter->ifp;
1925
1926 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1927 reg_rctl &= (~IXGBE_FCTRL_UPE);
1928 reg_rctl &= (~IXGBE_FCTRL_MPE);
1929 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1930
1931 if (ifp->if_flags & IFF_PROMISC) {
1932 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1933 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1934 } else if (ifp->if_flags & IFF_ALLMULTI) {
1935 reg_rctl |= IXGBE_FCTRL_MPE;
1936 reg_rctl &= ~IXGBE_FCTRL_UPE;
1937 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1938 }
1939 return;
1940}
1941
1942
1943/*********************************************************************
1944 * Multicast Update
1945 *
1946 * This routine is called whenever multicast address list is updated.
1947 *
1948 **********************************************************************/
1949#define IXGBE_RAR_ENTRIES 16
1950
1951static void
1952ixgbe_set_multi(struct adapter *adapter)
1953{
1954 u32 fctrl;
1955 u8 *mta;
1956 u8 *update_ptr;
1957 struct ifmultiaddr *ifma;
1958 int mcnt = 0;
1959 struct ifnet *ifp = adapter->ifp;
1960
1961 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1962
1963 mta = adapter->mta;
1964 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1965 MAX_NUM_MULTICAST_ADDRESSES);
1966
1967 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1968 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1969 if (ifp->if_flags & IFF_PROMISC)
1970 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1971 else if (ifp->if_flags & IFF_ALLMULTI) {
1972 fctrl |= IXGBE_FCTRL_MPE;
1973 fctrl &= ~IXGBE_FCTRL_UPE;
1974 } else
1975 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1976
1977 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1978
1979 spin_lock(&adapter->mcast_spin);
1980 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1981 if (ifma->ifma_addr->sa_family != AF_LINK)
1982 continue;
1983 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1984 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1985 IXGBE_ETH_LENGTH_OF_ADDRESS);
1986 mcnt++;
1987 }
1988 spin_unlock(&adapter->mcast_spin);
1989
1990 update_ptr = mta;
1991 ixgbe_update_mc_addr_list(&adapter->hw,
1992 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1993
1994 return;
1995}
1996
1997/*
1998 * This is an iterator function now needed by the multicast
1999 * shared code. It simply feeds the shared code routine the
2000 * addresses in the array of ixgbe_set_multi() one by one.
2001 */
2002static u8 *
2003ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2004{
2005 u8 *addr = *update_ptr;
2006 u8 *newptr;
2007 *vmdq = 0;
2008
2009 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2010 *update_ptr = newptr;
2011 return addr;
2012}
2013
2014
2015/*********************************************************************
2016 * Timer routine
2017 *
2018 * This routine checks for link status,updates statistics,
2019 * and runs the watchdog check.
2020 *
2021 **********************************************************************/
2022
2023static void
2024ixgbe_local_timer(void *arg)
2025{
2026 struct adapter *adapter = arg;
2027 device_t dev = adapter->dev;
2028 struct ifnet *ifp = adapter->ifp;
2029 struct ix_queue *que = adapter->queues;
2030 struct tx_ring *txr = adapter->tx_rings;
2031 int hung, busy, paused;
2032
2033 lockmgr(&adapter->core_lock, LK_EXCLUSIVE);
2034 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2035 hung = busy = paused = 0;
2036
2037 /* Check for pluggable optics */
2038 if (adapter->sfp_probe)
2039 if (!ixgbe_sfp_probe(adapter))
2040 goto out; /* Nothing to do */
2041
2042 ixgbe_update_link_status(adapter);
2043 ixgbe_update_stats_counters(adapter);
2044
2045 /*
2046 * If the interface has been paused
2047 * then don't do the watchdog check
2048 */
2049 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2050 paused = 1;
2051
2052 /*
2053 ** Check the TX queues status
2054 ** - central locked handling of OACTIVE
2055 ** - watchdog only if all queues show hung
2056 */
2057 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2058 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2059 (paused == 0))
2060 ++hung;
2061 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2062 ++busy;
2063 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2064 taskqueue_enqueue(que->tq, &que->que_task);
2065 }
2066 /* Only truely watchdog if all queues show hung */
2067 if (hung == adapter->num_queues)
2068 goto watchdog;
2069 /* Only turn off the stack flow when ALL are depleted */
2070 if (busy == adapter->num_queues)
2071 ifp->if_flags |= IFF_OACTIVE;
2072 else if ((ifp->if_flags & IFF_OACTIVE) &&
2073 (busy < adapter->num_queues))
2074 ifp->if_flags &= ~IFF_OACTIVE;
2075
2076out:
2077 ixgbe_rearm_queues(adapter, adapter->que_mask);
2078 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2079 lockmgr(&adapter->core_lock, LK_RELEASE);
2080 return;
2081
2082watchdog:
2083 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2084 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2085 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2086 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2087 device_printf(dev,"TX(%d) desc avail = %d,"
2088 "Next TX to Clean = %d\n",
2089 txr->me, txr->tx_avail, txr->next_to_clean);
2090 adapter->ifp->if_flags &= ~IFF_RUNNING;
2091 adapter->watchdog_events++;
2092 ixgbe_init_locked(adapter);
2093
2094 lockmgr(&adapter->core_lock, LK_RELEASE);
2095}
2096
2097/*
2098** Note: this routine updates the OS on the link state
2099** the real check of the hardware only happens with
2100** a link interrupt.
2101*/
2102static void
2103ixgbe_update_link_status(struct adapter *adapter)
2104{
2105 struct ifnet *ifp = adapter->ifp;
2106 struct tx_ring *txr = adapter->tx_rings;
2107 device_t dev = adapter->dev;
2108
2109
2110 if (adapter->link_up){
2111 if (adapter->link_active == FALSE) {
2112 if (bootverbose)
2113 device_printf(dev,"Link is up %d Gbps %s \n",
2114 ((adapter->link_speed == 128)? 10:1),
2115 "Full Duplex");
2116 adapter->link_active = TRUE;
ce451236
FT
2117 /* Update any Flow Control changes */
2118 ixgbe_fc_enable(&adapter->hw);
9407f759
FT
2119 ifp->if_link_state = LINK_STATE_UP;
2120 if_link_state_change(ifp);
2121 }
2122 } else { /* Link down */
2123 if (adapter->link_active == TRUE) {
2124 if (bootverbose)
2125 device_printf(dev,"Link is Down\n");
2126 ifp->if_link_state = LINK_STATE_DOWN;
2127 if_link_state_change(ifp);
2128 adapter->link_active = FALSE;
2129 for (int i = 0; i < adapter->num_queues;
2130 i++, txr++)
2131 txr->queue_status = IXGBE_QUEUE_IDLE;
2132 }
2133 }
2134
2135 return;
2136}
2137
2138
2139/*********************************************************************
2140 *
2141 * This routine disables all traffic on the adapter by issuing a
2142 * global reset on the MAC and deallocates TX/RX buffers.
2143 *
2144 **********************************************************************/
2145
2146static void
2147ixgbe_stop(void *arg)
2148{
2149 struct ifnet *ifp;
2150 struct adapter *adapter = arg;
2151 struct ixgbe_hw *hw = &adapter->hw;
2152 ifp = adapter->ifp;
2153
2154 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2155
2156 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2157 ixgbe_disable_intr(adapter);
2158 callout_stop(&adapter->timer);
2159
2160 /* Let the stack know...*/
2161 ifp->if_flags &= ~IFF_RUNNING;
2162 ifp->if_flags |= IFF_OACTIVE;
2163
2164 ixgbe_reset_hw(hw);
2165 hw->adapter_stopped = FALSE;
2166 ixgbe_stop_adapter(hw);
2167 /* Turn off the laser */
2168 if (hw->phy.multispeed_fiber)
2169 ixgbe_disable_tx_laser(hw);
2170
2171 /* reprogram the RAR[0] in case user changed it. */
2172 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2173
2174 return;
2175}
2176
2177
2178/*********************************************************************
2179 *
2180 * Determine hardware revision.
2181 *
2182 **********************************************************************/
2183static void
2184ixgbe_identify_hardware(struct adapter *adapter)
2185{
2186 device_t dev = adapter->dev;
2187 struct ixgbe_hw *hw = &adapter->hw;
2188
2189 /* Save off the information about this board */
2190 hw->vendor_id = pci_get_vendor(dev);
2191 hw->device_id = pci_get_device(dev);
2192 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2193 hw->subsystem_vendor_id =
2194 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2195 hw->subsystem_device_id =
2196 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2197
2198 /* We need this here to set the num_segs below */
2199 ixgbe_set_mac_type(hw);
2200
2201 /* Pick up the 82599 and VF settings */
2202 if (hw->mac.type != ixgbe_mac_82598EB) {
2203 hw->phy.smart_speed = ixgbe_smart_speed;
2204 adapter->num_segs = IXGBE_82599_SCATTER;
2205 } else
2206 adapter->num_segs = IXGBE_82598_SCATTER;
2207
2208 return;
2209}
2210
2211/*********************************************************************
2212 *
2213 * Determine optic type
2214 *
2215 **********************************************************************/
2216static void
2217ixgbe_setup_optics(struct adapter *adapter)
2218{
2219 struct ixgbe_hw *hw = &adapter->hw;
2220 int layer;
2221
2222 layer = ixgbe_get_supported_physical_layer(hw);
2223
2224 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2225 adapter->optics = IFM_10G_T;
2226 return;
2227 }
2228
2229 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2230 adapter->optics = IFM_1000_T;
2231 return;
2232 }
2233
2234 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2235 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2236 adapter->optics = IFM_10G_LR;
2237 return;
2238 }
2239
2240 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2241 adapter->optics = IFM_10G_SR;
2242 return;
2243 }
2244
2245 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2246 adapter->optics = IFM_10G_TWINAX;
2247 return;
2248 }
2249
2250 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2251 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2252 adapter->optics = IFM_10G_CX4;
2253 return;
2254 }
2255
2256 /* If we get here just set the default */
2257 adapter->optics = IFM_ETHER | IFM_AUTO;
2258 return;
2259}
2260
2261/*********************************************************************
2262 *
2263 * Setup the Legacy or MSI Interrupt handler
2264 *
2265 **********************************************************************/
2266static int
2267ixgbe_allocate_legacy(struct adapter *adapter)
2268{
2269 device_t dev = adapter->dev;
2270 struct ix_queue *que = adapter->queues;
2271 int error, rid = 0;
7e665ea1 2272 unsigned int intr_flags;
9407f759
FT
2273
2274 /* MSI RID at 1 */
2275 if (adapter->msix == 1)
2276 rid = 1;
2277
7e665ea1
FT
2278 /* Try allocating a MSI interrupt first */
2279 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2280 &rid, &intr_flags);
2281
9407f759
FT
2282 /* We allocate a single interrupt resource */
2283 adapter->res = bus_alloc_resource_any(dev,
7e665ea1 2284 SYS_RES_IRQ, &rid, intr_flags);
9407f759
FT
2285 if (adapter->res == NULL) {
2286 device_printf(dev, "Unable to allocate bus resource: "
2287 "interrupt\n");
2288 return (ENXIO);
2289 }
2290
2291 /*
2292 * Try allocating a fast interrupt and the associated deferred
2293 * processing contexts.
2294 */
2295 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2296 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2297 taskqueue_thread_enqueue, &que->tq);
2298 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2299 device_get_nameunit(adapter->dev));
2300
2301 /* Tasklets for Link, SFP and Multispeed Fiber */
2302 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2303 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2304 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2305#ifdef IXGBE_FDIR
2306 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2307#endif
2308 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2309 taskqueue_thread_enqueue, &adapter->tq);
2310 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2311 device_get_nameunit(adapter->dev));
2312
2313 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2314 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2315 device_printf(dev, "Failed to register fast interrupt "
2316 "handler: %d\n", error);
2317 taskqueue_free(que->tq);
2318 taskqueue_free(adapter->tq);
2319 que->tq = NULL;
2320 adapter->tq = NULL;
2321 return (error);
2322 }
2323 /* For simplicity in the handlers */
2324 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2325
2326 return (0);
2327}
2328
2329
2330/*********************************************************************
2331 *
2332 * Setup MSIX Interrupt resources and handlers
2333 *
2334 **********************************************************************/
2335static int
2336ixgbe_allocate_msix(struct adapter *adapter)
2337{
2338 device_t dev = adapter->dev;
2339 struct ix_queue *que = adapter->queues;
2340 int error, rid, vector = 0;
2341
2342 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2343 rid = vector + 1;
2344 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2345 RF_SHAREABLE | RF_ACTIVE);
2346 if (que->res == NULL) {
2347 device_printf(dev,"Unable to allocate"
2348 " bus resource: que interrupt [%d]\n", vector);
2349 return (ENXIO);
2350 }
2351 /* Set the handler function */
2352 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2353 ixgbe_msix_que, que, &que->tag, &que->serializer);
2354 if (error) {
2355 que->res = NULL;
2356 device_printf(dev, "Failed to register QUE handler");
2357 return (error);
2358 }
2359#if 0 /* __FreeBSD_version >= 800504 */
2360 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2361#endif
2362 que->msix = vector;
2363 adapter->que_mask |= (u64)(1 << que->msix);
2364 /*
2365 ** Bind the msix vector, and thus the
2366 ** ring to the corresponding cpu.
2367 */
2368#if 0 /* XXX */
2369 if (adapter->num_queues > 1)
2370 bus_bind_intr(dev, que->res, i);
2371#endif
2372
2373 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2374 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2375 taskqueue_thread_enqueue, &que->tq);
2376 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2377 device_get_nameunit(adapter->dev));
2378 }
2379
2380 /* and Link */
2381 rid = vector + 1;
2382 adapter->res = bus_alloc_resource_any(dev,
2383 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2384 if (!adapter->res) {
2385 device_printf(dev,"Unable to allocate"
2386 " bus resource: Link interrupt [%d]\n", rid);
2387 return (ENXIO);
2388 }
2389 /* Set the link handler function */
2390 error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2391 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2392 if (error) {
2393 adapter->res = NULL;
2394 device_printf(dev, "Failed to register LINK handler");
2395 return (error);
2396 }
2397#if 0 /* __FreeBSD_version >= 800504 */
2398 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2399#endif
2400 adapter->linkvec = vector;
2401 /* Tasklets for Link, SFP and Multispeed Fiber */
2402 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2403 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2404 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2405#ifdef IXGBE_FDIR
2406 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2407#endif
2408 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2409 taskqueue_thread_enqueue, &adapter->tq);
2410 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2411 device_get_nameunit(adapter->dev));
2412
2413 return (0);
2414}
2415
2416#if 0 /* HAVE_MSIX */
2417/*
2418 * Setup Either MSI/X or MSI
2419 */
2420static int
2421ixgbe_setup_msix(struct adapter *adapter)
2422{
2423 device_t dev = adapter->dev;
2424 int rid, want, queues, msgs;
2425
2426 /* Override by tuneable */
2427 if (ixgbe_enable_msix == 0)
2428 goto msi;
2429
2430 /* First try MSI/X */
2431 rid = PCIR_BAR(MSIX_82598_BAR);
2432 adapter->msix_mem = bus_alloc_resource_any(dev,
2433 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2434 if (!adapter->msix_mem) {
2435 rid += 4; /* 82599 maps in higher BAR */
2436 adapter->msix_mem = bus_alloc_resource_any(dev,
2437 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2438 }
2439 if (!adapter->msix_mem) {
2440 /* May not be enabled */
2441 device_printf(adapter->dev,
2442 "Unable to map MSIX table \n");
2443 goto msi;
2444 }
2445
2446 msgs = pci_msix_count(dev);
2447 if (msgs == 0) { /* system has msix disabled */
2448 bus_release_resource(dev, SYS_RES_MEMORY,
2449 rid, adapter->msix_mem);
2450 adapter->msix_mem = NULL;
2451 goto msi;
2452 }
2453
2454 /* Figure out a reasonable auto config value */
2455 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2456
2457 if (ixgbe_num_queues != 0)
2458 queues = ixgbe_num_queues;
2459 /* Set max queues to 8 when autoconfiguring */
2460 else if ((ixgbe_num_queues == 0) && (queues > 8))
2461 queues = 8;
2462
2463 /*
2464 ** Want one vector (RX/TX pair) per queue
2465 ** plus an additional for Link.
2466 */
2467 want = queues + 1;
2468 if (msgs >= want)
2469 msgs = want;
2470 else {
2471 device_printf(adapter->dev,
2472 "MSIX Configuration Problem, "
2473 "%d vectors but %d queues wanted!\n",
2474 msgs, want);
2475 return (0); /* Will go to Legacy setup */
2476 }
2477 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2478 device_printf(adapter->dev,
2479 "Using MSIX interrupts with %d vectors\n", msgs);
2480 adapter->num_queues = queues;
2481 return (msgs);
2482 }
2483msi:
2484 msgs = pci_msi_count(dev);
2485 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2486 device_printf(adapter->dev,"Using an MSI interrupt\n");
2487 else
2488 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2489 return (msgs);
2490}
2491#endif
2492
2493
2494static int
2495ixgbe_allocate_pci_resources(struct adapter *adapter)
2496{
2497 int rid;
2498 device_t dev = adapter->dev;
2499
2500 rid = PCIR_BAR(0);
2501 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2502 &rid, RF_ACTIVE);
2503
2504 if (!(adapter->pci_mem)) {
2505 device_printf(dev,"Unable to allocate bus resource: memory\n");
2506 return (ENXIO);
2507 }
2508
2509 adapter->osdep.mem_bus_space_tag =
2510 rman_get_bustag(adapter->pci_mem);
2511 adapter->osdep.mem_bus_space_handle =
2512 rman_get_bushandle(adapter->pci_mem);
2513 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2514
2515 /* Legacy defaults */
2516 adapter->num_queues = 1;
2517 adapter->hw.back = &adapter->osdep;
2518
2519 /*
2520 ** Now setup MSI or MSI/X, should
2521 ** return us the number of supported
2522 ** vectors. (Will be 1 for MSI)
2523 */
2524#if 0 /* HAVE_MSIX */
2525 adapter->msix = ixgbe_setup_msix(adapter);
2526#endif
2527 return (0);
2528}
2529
2530static void
2531ixgbe_free_pci_resources(struct adapter * adapter)
2532{
2533 struct ix_queue *que = adapter->queues;
2534 device_t dev = adapter->dev;
2535 int rid, memrid;
2536
2537 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2538 memrid = PCIR_BAR(MSIX_82598_BAR);
2539 else
2540 memrid = PCIR_BAR(MSIX_82599_BAR);
2541
2542 /*
2543 ** There is a slight possibility of a failure mode
2544 ** in attach that will result in entering this function
2545 ** before interrupt resources have been initialized, and
2546 ** in that case we do not want to execute the loops below
2547 ** We can detect this reliably by the state of the adapter
2548 ** res pointer.
2549 */
2550 if (adapter->res == NULL)
2551 goto mem;
2552
2553 /*
2554 ** Release all msix queue resources:
2555 */
2556 for (int i = 0; i < adapter->num_queues; i++, que++) {
2557 rid = que->msix + 1;
2558 if (que->tag != NULL) {
2559 bus_teardown_intr(dev, que->res, que->tag);
2560 que->tag = NULL;
2561 }
2562 if (que->res != NULL)
2563 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2564 }
2565
2566
2567 /* Clean the Legacy or Link interrupt last */
2568 if (adapter->linkvec) /* we are doing MSIX */
2569 rid = adapter->linkvec + 1;
2570 else
2571 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2572
2573 if (adapter->tag != NULL) {
2574 bus_teardown_intr(dev, adapter->res, adapter->tag);
2575 adapter->tag = NULL;
2576 }
2577 if (adapter->res != NULL)
2578 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
7e665ea1
FT
2579 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2580 pci_release_msi(adapter->dev);
9407f759
FT
2581
2582mem:
2583 if (adapter->msix)
2584 pci_release_msi(dev);
2585
2586 if (adapter->msix_mem != NULL)
2587 bus_release_resource(dev, SYS_RES_MEMORY,
2588 memrid, adapter->msix_mem);
2589
2590 if (adapter->pci_mem != NULL)
2591 bus_release_resource(dev, SYS_RES_MEMORY,
2592 PCIR_BAR(0), adapter->pci_mem);
2593
2594 return;
2595}
2596
2597/*********************************************************************
2598 *
2599 * Setup networking device structure and register an interface.
2600 *
2601 **********************************************************************/
2602static int
2603ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2604{
2605 struct ixgbe_hw *hw = &adapter->hw;
2606 struct ifnet *ifp;
2607
2608 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2609
2610 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2611 if (ifp == NULL) {
2612 device_printf(dev, "can not allocate ifnet structure\n");
2613 return (-1);
2614 }
2615 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2616 ifp->if_baudrate = 1000000000;
2617 ifp->if_init = ixgbe_init;
2618 ifp->if_softc = adapter;
2619 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2620 ifp->if_ioctl = ixgbe_ioctl;
2621 ifp->if_start = ixgbe_start;
2622#if 0 /* __FreeBSD_version >= 800000 */
2623 ifp->if_transmit = ixgbe_mq_start;
2624 ifp->if_qflush = ixgbe_qflush;
2625#endif
2626 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2627
2628 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2629
2630 adapter->max_frame_size =
2631 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2632
2633 /*
2634 * Tell the upper layer(s) we support long frames.
2635 */
2636 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2637
2638#if 0 /* NET_TSO */
2639 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2640#endif
2641 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2642 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2643 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2644#if 0 /* NET_TSO */
2645 | IFCAP_VLAN_HWTSO
2646#endif
2647 | IFCAP_VLAN_MTU;
2648 ifp->if_capenable = ifp->if_capabilities;
2649
2650 /* Don't enable LRO by default */
2651#if 0 /* NET_LRO */
2652 ifp->if_capabilities |= IFCAP_LRO;
2653#endif
2654
2655 /*
2656 ** Don't turn this on by default, if vlans are
2657 ** created on another pseudo device (eg. lagg)
2658 ** then vlan events are not passed thru, breaking
2659 ** operation, but with HW FILTER off it works. If
2660 ** using vlans directly on the ixgbe driver you can
2661 ** enable this and get full hardware tag filtering.
2662 */
2663 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2664
2665 /*
2666 * Specify the media types supported by this adapter and register
2667 * callbacks to update media and link information
2668 */
2669 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2670 ixgbe_media_status);
2671 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2672 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2673 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2674 ifmedia_add(&adapter->media,
2675 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2676 ifmedia_add(&adapter->media,
2677 IFM_ETHER | IFM_1000_T, 0, NULL);
2678 }
2679 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2680 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2681
2682 return (0);
2683}
2684
2685static void
2686ixgbe_config_link(struct adapter *adapter)
2687{
2688 struct ixgbe_hw *hw = &adapter->hw;
2689 u32 autoneg, err = 0;
2690 bool sfp, negotiate;
2691
2692 sfp = ixgbe_is_sfp(hw);
2693
2694 if (sfp) {
2695 if (hw->phy.multispeed_fiber) {
2696 hw->mac.ops.setup_sfp(hw);
2697 ixgbe_enable_tx_laser(hw);
2698 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2699 } else
2700 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2701 } else {
2702 if (hw->mac.ops.check_link)
2703 err = ixgbe_check_link(hw, &autoneg,
2704 &adapter->link_up, FALSE);
2705 if (err)
2706 goto out;
2707 autoneg = hw->phy.autoneg_advertised;
2708 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2709 err = hw->mac.ops.get_link_capabilities(hw,
2710 &autoneg, &negotiate);
2711 if (err)
2712 goto out;
2713 if (hw->mac.ops.setup_link)
2714 err = hw->mac.ops.setup_link(hw, autoneg,
2715 negotiate, adapter->link_up);
2716 }
2717out:
2718 return;
2719}
2720
2721/********************************************************************
2722 * Manage DMA'able memory.
2723 *******************************************************************/
2724static void
2725ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2726{
2727 if (error)
2728 return;
2729 *(bus_addr_t *) arg = segs->ds_addr;
2730 return;
2731}
2732
2733static int
2734ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2735 struct ixgbe_dma_alloc *dma, int mapflags)
2736{
2737 device_t dev = adapter->dev;
2738 int r;
2739
2740 r = bus_dma_tag_create(NULL, /* parent */
2741 DBA_ALIGN, 0, /* alignment, bounds */
2742 BUS_SPACE_MAXADDR, /* lowaddr */
2743 BUS_SPACE_MAXADDR, /* highaddr */
2744 NULL, NULL, /* filter, filterarg */
2745 size, /* maxsize */
2746 1, /* nsegments */
2747 size, /* maxsegsize */
2748 BUS_DMA_ALLOCNOW, /* flags */
2749 &dma->dma_tag);
2750 if (r != 0) {
2751 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2752 "error %u\n", r);
2753 goto fail_0;
2754 }
2755 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2756 BUS_DMA_NOWAIT, &dma->dma_map);
2757 if (r != 0) {
2758 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2759 "error %u\n", r);
2760 goto fail_1;
2761 }
2762 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2763 size,
2764 ixgbe_dmamap_cb,
2765 &dma->dma_paddr,
2766 mapflags | BUS_DMA_NOWAIT);
2767 if (r != 0) {
2768 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2769 "error %u\n", r);
2770 goto fail_2;
2771 }
2772 dma->dma_size = size;
2773 return (0);
2774fail_2:
2775 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2776fail_1:
2777 bus_dma_tag_destroy(dma->dma_tag);
2778fail_0:
2779 dma->dma_map = NULL;
2780 dma->dma_tag = NULL;
2781 return (r);
2782}
2783
2784static void
2785ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2786{
2787 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2788 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2789 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2790 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2791 bus_dma_tag_destroy(dma->dma_tag);
2792}
2793
2794
2795/*********************************************************************
2796 *
2797 * Allocate memory for the transmit and receive rings, and then
2798 * the descriptors associated with each, called only once at attach.
2799 *
2800 **********************************************************************/
2801static int
2802ixgbe_allocate_queues(struct adapter *adapter)
2803{
2804 device_t dev = adapter->dev;
2805 struct ix_queue *que;
2806 struct tx_ring *txr;
2807 struct rx_ring *rxr;
2808 int rsize, tsize, error = IXGBE_SUCCESS;
2809 int txconf = 0, rxconf = 0;
2810
2811 /* First allocate the top level queue structs */
2812 if (!(adapter->queues =
2813 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2814 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2815 device_printf(dev, "Unable to allocate queue memory\n");
2816 error = ENOMEM;
2817 goto fail;
2818 }
2819
2820 /* First allocate the TX ring struct memory */
2821 if (!(adapter->tx_rings =
2822 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2823 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2824 device_printf(dev, "Unable to allocate TX ring memory\n");
2825 error = ENOMEM;
2826 goto tx_fail;
2827 }
2828
2829 /* Next allocate the RX */
2830 if (!(adapter->rx_rings =
2831 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2832 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2833 device_printf(dev, "Unable to allocate RX ring memory\n");
2834 error = ENOMEM;
2835 goto rx_fail;
2836 }
2837
2838 /* For the ring itself */
2839 tsize = roundup2(adapter->num_tx_desc *
2840 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2841
2842 /*
2843 * Now set up the TX queues, txconf is needed to handle the
2844 * possibility that things fail midcourse and we need to
2845 * undo memory gracefully
2846 */
2847 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2848 /* Set up some basics */
2849 txr = &adapter->tx_rings[i];
2850 txr->adapter = adapter;
2851 txr->me = i;
2852
2853 /* Initialize the TX side lock */
2854 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2855 device_get_nameunit(dev), txr->me);
2856 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2857
2858 if (ixgbe_dma_malloc(adapter, tsize,
2859 &txr->txdma, BUS_DMA_NOWAIT)) {
2860 device_printf(dev,
2861 "Unable to allocate TX Descriptor memory\n");
2862 error = ENOMEM;
2863 goto err_tx_desc;
2864 }
2865 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2866 bzero((void *)txr->tx_base, tsize);
2867
2868 /* Now allocate transmit buffers for the ring */
2869 if (ixgbe_allocate_transmit_buffers(txr)) {
2870 device_printf(dev,
2871 "Critical Failure setting up transmit buffers\n");
2872 error = ENOMEM;
2873 goto err_tx_desc;
2874 }
2875#if 0 /* __FreeBSD_version >= 800000 */
2876 /* Allocate a buf ring */
2877 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2878 M_WAITOK, &txr->tx_mtx);
2879 if (txr->br == NULL) {
2880 device_printf(dev,
2881 "Critical Failure setting up buf ring\n");
2882 error = ENOMEM;
2883 goto err_tx_desc;
2884 }
2885#endif
2886 }
2887
2888 /*
2889 * Next the RX queues...
2890 */
2891 rsize = roundup2(adapter->num_rx_desc *
2892 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2893 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2894 rxr = &adapter->rx_rings[i];
2895 /* Set up some basics */
2896 rxr->adapter = adapter;
2897 rxr->me = i;
2898
2899 /* Initialize the RX side lock */
2900 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2901 device_get_nameunit(dev), rxr->me);
2902 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2903
2904 if (ixgbe_dma_malloc(adapter, rsize,
2905 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2906 device_printf(dev,
2907 "Unable to allocate RxDescriptor memory\n");
2908 error = ENOMEM;
2909 goto err_rx_desc;
2910 }
2911 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2912 bzero((void *)rxr->rx_base, rsize);
2913
2914 /* Allocate receive buffers for the ring*/
2915 if (ixgbe_allocate_receive_buffers(rxr)) {
2916 device_printf(dev,
2917 "Critical Failure setting up receive buffers\n");
2918 error = ENOMEM;
2919 goto err_rx_desc;
2920 }
2921 }
2922
2923 /*
2924 ** Finally set up the queue holding structs
2925 */
2926 for (int i = 0; i < adapter->num_queues; i++) {
2927 que = &adapter->queues[i];
2928 que->adapter = adapter;
2929 que->txr = &adapter->tx_rings[i];
2930 que->rxr = &adapter->rx_rings[i];
2931 }
2932
2933 return (0);
2934
2935err_rx_desc:
2936 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2937 ixgbe_dma_free(adapter, &rxr->rxdma);
2938err_tx_desc:
2939 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2940 ixgbe_dma_free(adapter, &txr->txdma);
2941 kfree(adapter->rx_rings, M_DEVBUF);
2942rx_fail:
2943 kfree(adapter->tx_rings, M_DEVBUF);
2944tx_fail:
2945 kfree(adapter->queues, M_DEVBUF);
2946fail:
2947 return (error);
2948}
2949
2950/*********************************************************************
2951 *
2952 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2953 * the information needed to transmit a packet on the wire. This is
2954 * called only once at attach, setup is done every reset.
2955 *
2956 **********************************************************************/
2957static int
2958ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2959{
2960 struct adapter *adapter = txr->adapter;
2961 device_t dev = adapter->dev;
2962 struct ixgbe_tx_buf *txbuf;
2963 int error, i;
2964
2965 /*
2966 * Setup DMA descriptor areas.
2967 */
2968 if ((error = bus_dma_tag_create(
2969 NULL, /* parent */
2970 1, 0, /* alignment, bounds */
2971 BUS_SPACE_MAXADDR, /* lowaddr */
2972 BUS_SPACE_MAXADDR, /* highaddr */
2973 NULL, NULL, /* filter, filterarg */
2974 IXGBE_TSO_SIZE, /* maxsize */
2975 adapter->num_segs, /* nsegments */
2976 PAGE_SIZE, /* maxsegsize */
2977 0, /* flags */
2978 &txr->txtag))) {
2979 device_printf(dev,"Unable to allocate TX DMA tag\n");
2980 goto fail;
2981 }
2982
2983 if (!(txr->tx_buffers =
2984 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2985 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2986 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2987 error = ENOMEM;
2988 goto fail;
2989 }
2990
2991 /* Create the descriptor buffer dma maps */
2992 txbuf = txr->tx_buffers;
2993 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2994 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2995 if (error != 0) {
2996 device_printf(dev, "Unable to create TX DMA map\n");
2997 goto fail;
2998 }
2999 }
3000
3001 return 0;
3002fail:
3003 /* We free all, it handles case where we are in the middle */
3004 ixgbe_free_transmit_structures(adapter);
3005 return (error);
3006}
3007
3008/*********************************************************************
3009 *
3010 * Initialize a transmit ring.
3011 *
3012 **********************************************************************/
3013static void
3014ixgbe_setup_transmit_ring(struct tx_ring *txr)
3015{
3016 struct adapter *adapter = txr->adapter;
3017 struct ixgbe_tx_buf *txbuf;
3018 int i;
3019#ifdef DEV_NETMAP
3020 struct netmap_adapter *na = NA(adapter->ifp);
3021 struct netmap_slot *slot;
3022#endif /* DEV_NETMAP */
3023
3024 /* Clear the old ring contents */
3025 IXGBE_TX_LOCK(txr);
3026#ifdef DEV_NETMAP
3027 /*
3028 * (under lock): if in netmap mode, do some consistency
3029 * checks and set slot to entry 0 of the netmap ring.
3030 */
3031 slot = netmap_reset(na, NR_TX, txr->me, 0);
3032#endif /* DEV_NETMAP */
3033 bzero((void *)txr->tx_base,
3034 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3035 /* Reset indices */
3036 txr->next_avail_desc = 0;
3037 txr->next_to_clean = 0;
3038
3039 /* Free any existing tx buffers. */
3040 txbuf = txr->tx_buffers;
3041 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3042 if (txbuf->m_head != NULL) {
3043 bus_dmamap_sync(txr->txtag, txbuf->map,
3044 BUS_DMASYNC_POSTWRITE);
3045 bus_dmamap_unload(txr->txtag, txbuf->map);
3046 m_freem(txbuf->m_head);
3047 txbuf->m_head = NULL;
3048 }
3049#ifdef DEV_NETMAP
3050 /*
3051 * In netmap mode, set the map for the packet buffer.
3052 * NOTE: Some drivers (not this one) also need to set
3053 * the physical buffer address in the NIC ring.
3054 * Slots in the netmap ring (indexed by "si") are
3055 * kring->nkr_hwofs positions "ahead" wrt the
3056 * corresponding slot in the NIC ring. In some drivers
3057 * (not here) nkr_hwofs can be negative. Function
3058 * netmap_idx_n2k() handles wraparounds properly.
3059 */
3060 if (slot) {
3061 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3062 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3063 }
3064#endif /* DEV_NETMAP */
3065 /* Clear the EOP index */
3066 txbuf->eop_index = -1;
3067 }
3068
3069#ifdef IXGBE_FDIR
3070 /* Set the rate at which we sample packets */
3071 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3072 txr->atr_sample = atr_sample_rate;
3073#endif
3074
3075 /* Set number of descriptors available */
3076 txr->tx_avail = adapter->num_tx_desc;
3077
3078 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3079 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3080 IXGBE_TX_UNLOCK(txr);
3081}
3082
3083/*********************************************************************
3084 *
3085 * Initialize all transmit rings.
3086 *
3087 **********************************************************************/
3088static int
3089ixgbe_setup_transmit_structures(struct adapter *adapter)
3090{
3091 struct tx_ring *txr = adapter->tx_rings;
3092
3093 for (int i = 0; i < adapter->num_queues; i++, txr++)
3094 ixgbe_setup_transmit_ring(txr);
3095
3096 return (0);
3097}
3098
3099/*********************************************************************
3100 *
3101 * Enable transmit unit.
3102 *
3103 **********************************************************************/
3104static void
3105ixgbe_initialize_transmit_units(struct adapter *adapter)
3106{
3107 struct tx_ring *txr = adapter->tx_rings;
3108 struct ixgbe_hw *hw = &adapter->hw;
3109
3110 /* Setup the Base and Length of the Tx Descriptor Ring */
3111
3112 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3113 u64 tdba = txr->txdma.dma_paddr;
3114 u32 txctrl;
3115
3116 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3117 (tdba & 0x00000000ffffffffULL));
3118 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3119 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3120 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3121
3122 /* Setup the HW Tx Head and Tail descriptor pointers */
3123 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3124 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3125
3126 /* Setup Transmit Descriptor Cmd Settings */
3127 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3128 txr->queue_status = IXGBE_QUEUE_IDLE;
3129
3130 /* Disable Head Writeback */
3131 switch (hw->mac.type) {
3132 case ixgbe_mac_82598EB:
3133 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3134 break;
3135 case ixgbe_mac_82599EB:
3136 case ixgbe_mac_X540:
3137 default:
3138 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3139 break;
3140 }
ce451236 3141 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
9407f759
FT
3142 switch (hw->mac.type) {
3143 case ixgbe_mac_82598EB:
3144 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3145 break;
3146 case ixgbe_mac_82599EB:
3147 case ixgbe_mac_X540:
3148 default:
3149 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3150 break;
3151 }
3152
3153 }
3154
3155 if (hw->mac.type != ixgbe_mac_82598EB) {
3156 u32 dmatxctl, rttdcs;
3157 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3158 dmatxctl |= IXGBE_DMATXCTL_TE;
3159 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3160 /* Disable arbiter to set MTQC */
3161 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3162 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3163 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3164 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3165 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3166 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3167 }
3168
3169 return;
3170}
3171
3172/*********************************************************************
3173 *
3174 * Free all transmit rings.
3175 *
3176 **********************************************************************/
3177static void
3178ixgbe_free_transmit_structures(struct adapter *adapter)
3179{
3180 struct tx_ring *txr = adapter->tx_rings;
3181
3182 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3183 IXGBE_TX_LOCK(txr);
3184 ixgbe_free_transmit_buffers(txr);
3185 ixgbe_dma_free(adapter, &txr->txdma);
3186 IXGBE_TX_UNLOCK(txr);
3187 IXGBE_TX_LOCK_DESTROY(txr);
3188 }
3189 kfree(adapter->tx_rings, M_DEVBUF);
3190}
3191
3192/*********************************************************************
3193 *
3194 * Free transmit ring related data structures.
3195 *
3196 **********************************************************************/
3197static void
3198ixgbe_free_transmit_buffers(struct tx_ring *txr)
3199{
3200 struct adapter *adapter = txr->adapter;
3201 struct ixgbe_tx_buf *tx_buffer;
3202 int i;
3203
3204 INIT_DEBUGOUT("free_transmit_ring: begin");
3205
3206 if (txr->tx_buffers == NULL)
3207 return;
3208
3209 tx_buffer = txr->tx_buffers;
3210 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3211 if (tx_buffer->m_head != NULL) {
3212 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3213 BUS_DMASYNC_POSTWRITE);
3214 bus_dmamap_unload(txr->txtag,
3215 tx_buffer->map);
3216 m_freem(tx_buffer->m_head);
3217 tx_buffer->m_head = NULL;
3218 if (tx_buffer->map != NULL) {
3219 bus_dmamap_destroy(txr->txtag,
3220 tx_buffer->map);
3221 tx_buffer->map = NULL;
3222 }
3223 } else if (tx_buffer->map != NULL) {
3224 bus_dmamap_unload(txr->txtag,
3225 tx_buffer->map);
3226 bus_dmamap_destroy(txr->txtag,
3227 tx_buffer->map);
3228 tx_buffer->map = NULL;
3229 }
3230 }
3231#if 0 /* __FreeBSD_version >= 800000 */
3232 if (txr->br != NULL)
3233 buf_ring_free(txr->br, M_DEVBUF);
3234#endif
3235 if (txr->tx_buffers != NULL) {
3236 kfree(txr->tx_buffers, M_DEVBUF);
3237 txr->tx_buffers = NULL;
3238 }
3239 if (txr->txtag != NULL) {
3240 bus_dma_tag_destroy(txr->txtag);
3241 txr->txtag = NULL;
3242 }
3243 return;
3244}
3245
3246/*********************************************************************
3247 *
3248 * Advanced Context Descriptor setup for VLAN or CSUM
3249 *
3250 **********************************************************************/
3251
3252static bool
3253ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3254{
3255 struct adapter *adapter = txr->adapter;
3256 struct ixgbe_adv_tx_context_desc *TXD;
3257 struct ixgbe_tx_buf *tx_buffer;
3258 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3259 struct ether_vlan_header *eh;
3260 struct ip *ip;
3261 struct ip6_hdr *ip6;
3262 int ehdrlen, ip_hlen = 0;
3263 u16 etype;
3264 u8 ipproto = 0;
3265 bool offload = TRUE;
3266 int ctxd = txr->next_avail_desc;
3267#ifdef NET_VLAN
3268 u16 vtag = 0;
3269#endif
3270
3271
3272 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3273 offload = FALSE;
3274
3275 tx_buffer = &txr->tx_buffers[ctxd];
3276 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3277
3278 /*
3279 ** In advanced descriptors the vlan tag must
3280 ** be placed into the descriptor itself.
3281 */
3282#ifdef NET_VLAN
3283 if (mp->m_flags & M_VLANTAG) {
3284 vtag = htole16(mp->m_pkthdr.ether_vtag);
3285 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3286 } else if (offload == FALSE)
3287 return FALSE;
3288#endif
3289
3290 /*
3291 * Determine where frame payload starts.
3292 * Jump over vlan headers if already present,
3293 * helpful for QinQ too.
3294 */
3295 eh = mtod(mp, struct ether_vlan_header *);
3296 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3297 etype = ntohs(eh->evl_proto);
3298 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3299 } else {
3300 etype = ntohs(eh->evl_encap_proto);
3301 ehdrlen = ETHER_HDR_LEN;
3302 }
3303
3304 /* Set the ether header length */
3305 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3306
3307 switch (etype) {
3308 case ETHERTYPE_IP:
3309 ip = (struct ip *)(mp->m_data + ehdrlen);
3310 ip_hlen = ip->ip_hl << 2;
3311 ipproto = ip->ip_p;
3312 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3313 break;
3314 case ETHERTYPE_IPV6:
3315 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3316 ip_hlen = sizeof(struct ip6_hdr);
3317 /* XXX-BZ this will go badly in case of ext hdrs. */
3318 ipproto = ip6->ip6_nxt;
3319 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3320 break;
3321 default:
3322 offload = FALSE;
3323 break;
3324 }
3325
3326 vlan_macip_lens |= ip_hlen;
3327 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3328
3329 switch (ipproto) {
3330 case IPPROTO_TCP:
3331 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3332 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3333 break;
3334
3335 case IPPROTO_UDP:
3336 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3337 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3338 break;
3339
3340#if 0
3341 case IPPROTO_SCTP:
3342 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3343 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3344 break;
3345#endif
3346 default:
3347 offload = FALSE;
3348 break;
3349 }
3350
3351 /* Now copy bits into descriptor */
3352 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3353 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3354 TXD->seqnum_seed = htole32(0);
3355 TXD->mss_l4len_idx = htole32(0);
3356
3357 tx_buffer->m_head = NULL;
3358 tx_buffer->eop_index = -1;
3359
3360 /* We've consumed the first desc, adjust counters */
3361 if (++ctxd == adapter->num_tx_desc)
3362 ctxd = 0;
3363 txr->next_avail_desc = ctxd;
3364 --txr->tx_avail;
3365
3366 return (offload);
3367}
3368
3369/**********************************************************************
3370 *
3371 * Setup work for hardware segmentation offload (TSO) on
3372 * adapters using advanced tx descriptors
3373 *
3374 **********************************************************************/
3375#if 0 /* NET_TSO */
3376static bool
3377ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3378 u32 *olinfo_status)
3379{
3380 struct adapter *adapter = txr->adapter;
3381 struct ixgbe_adv_tx_context_desc *TXD;
3382 struct ixgbe_tx_buf *tx_buffer;
3383#ifdef NET_VLAN
3384 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3385 u16 vtag = 0, eh_type;
3386#else
3387 u16 eh_type;
3388 u32 type_tucmd_mlhl = 0;
3389#endif
3390 u32 mss_l4len_idx = 0, len;
3391 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3392 struct ether_vlan_header *eh;
3393#ifdef INET6
3394 struct ip6_hdr *ip6;
3395#endif
3396#ifdef INET
3397 struct ip *ip;
3398#endif
3399 struct tcphdr *th;
3400
3401
3402 /*
3403 * Determine where frame payload starts.
3404 * Jump over vlan headers if already present
3405 */
3406 eh = mtod(mp, struct ether_vlan_header *);
3407 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3408 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3409 eh_type = eh->evl_proto;
3410 } else {
3411 ehdrlen = ETHER_HDR_LEN;
3412 eh_type = eh->evl_encap_proto;
3413 }
3414
3415 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3416 len = ehdrlen + sizeof(struct tcphdr);
3417 switch (ntohs(eh_type)) {
3418#ifdef INET6
3419 case ETHERTYPE_IPV6:
3420 if (mp->m_len < len + sizeof(struct ip6_hdr))
3421 return FALSE;
3422 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3423 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3424 if (ip6->ip6_nxt != IPPROTO_TCP)
3425 return FALSE;
3426 ip_hlen = sizeof(struct ip6_hdr);
3427 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3428 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3429 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3430 break;
3431#endif
3432#ifdef INET
3433 case ETHERTYPE_IP:
3434 if (mp->m_len < len + sizeof(struct ip))
3435 return FALSE;
3436 ip = (struct ip *)(mp->m_data + ehdrlen);
3437 if (ip->ip_p != IPPROTO_TCP)
3438 return FALSE;
3439 ip->ip_sum = 0;
3440 ip_hlen = ip->ip_hl << 2;
3441 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3442 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3443 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3444 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3445 /* Tell transmit desc to also do IPv4 checksum. */
3446 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3447 break;
3448#endif
3449 default:
3450 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3451 __func__, ntohs(eh_type));
3452 break;
3453 }
3454
3455 ctxd = txr->next_avail_desc;
3456 tx_buffer = &txr->tx_buffers[ctxd];
3457 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3458
3459 tcp_hlen = th->th_off << 2;
3460
3461 /* This is used in the transmit desc in encap */
3462 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3463
3464 /* VLAN MACLEN IPLEN */
3465#ifdef NET_VLAN
3466 if (mp->m_flags & M_VLANTAG) {
3467 vtag = htole16(mp->m_pkthdr.ether_vtag);
3468 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3469 }
3470
3471 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3472 vlan_macip_lens |= ip_hlen;
3473 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3474#endif
3475
3476 /* ADV DTYPE TUCMD */
3477 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3478 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3479 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3480
3481 /* MSS L4LEN IDX */
3482 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3483 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3484 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3485
3486 TXD->seqnum_seed = htole32(0);
3487 tx_buffer->m_head = NULL;
3488 tx_buffer->eop_index = -1;
3489
3490 if (++ctxd == adapter->num_tx_desc)
3491 ctxd = 0;
3492
3493 txr->tx_avail--;
3494 txr->next_avail_desc = ctxd;
3495 return TRUE;
3496}
3497#endif
3498
3499#ifdef IXGBE_FDIR
3500/*
3501** This routine parses packet headers so that Flow
3502** Director can make a hashed filter table entry
3503** allowing traffic flows to be identified and kept
3504** on the same cpu. This would be a performance
3505** hit, but we only do it at IXGBE_FDIR_RATE of
3506** packets.
3507*/
3508static void
3509ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3510{
3511 struct adapter *adapter = txr->adapter;
3512 struct ix_queue *que;
3513 struct ip *ip;
3514 struct tcphdr *th;
3515 struct udphdr *uh;
3516 struct ether_vlan_header *eh;
3517 union ixgbe_atr_hash_dword input = {.dword = 0};
3518 union ixgbe_atr_hash_dword common = {.dword = 0};
3519 int ehdrlen, ip_hlen;
3520 u16 etype;
3521
3522 eh = mtod(mp, struct ether_vlan_header *);
3523 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3524 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3525 etype = eh->evl_proto;
3526 } else {
3527 ehdrlen = ETHER_HDR_LEN;
3528 etype = eh->evl_encap_proto;
3529 }
3530
3531 /* Only handling IPv4 */
3532 if (etype != htons(ETHERTYPE_IP))
3533 return;
3534
3535 ip = (struct ip *)(mp->m_data + ehdrlen);
3536 ip_hlen = ip->ip_hl << 2;
3537
3538 /* check if we're UDP or TCP */
3539 switch (ip->ip_p) {
3540 case IPPROTO_TCP:
3541 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3542 /* src and dst are inverted */
3543 common.port.dst ^= th->th_sport;
3544 common.port.src ^= th->th_dport;
3545 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3546 break;
3547 case IPPROTO_UDP:
3548 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3549 /* src and dst are inverted */
3550 common.port.dst ^= uh->uh_sport;
3551 common.port.src ^= uh->uh_dport;
3552 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3553 break;
3554 default:
3555 return;
3556 }
3557
3558 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3559 if (mp->m_pkthdr.ether_vtag)
3560 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3561 else
3562 common.flex_bytes ^= etype;
3563 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3564
3565 que = &adapter->queues[txr->me];
3566 /*
3567 ** This assumes the Rx queue and Tx
3568 ** queue are bound to the same CPU
3569 */
3570 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3571 input, common, que->msix);
3572}
3573#endif /* IXGBE_FDIR */
3574
3575/**********************************************************************
3576 *
3577 * Examine each tx_buffer in the used queue. If the hardware is done
3578 * processing the packet then free associated resources. The
3579 * tx_buffer is put back on the free queue.
3580 *
3581 **********************************************************************/
3582static bool
3583ixgbe_txeof(struct tx_ring *txr)
3584{
3585 struct adapter *adapter = txr->adapter;
3586 struct ifnet *ifp = adapter->ifp;
3587 u32 first, last, done, processed;
3588 struct ixgbe_tx_buf *tx_buffer;
3589 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3590
3591 KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3592
3593#ifdef DEV_NETMAP
3594 if (ifp->if_capenable & IFCAP_NETMAP) {
3595 struct netmap_adapter *na = NA(ifp);
3596 struct netmap_kring *kring = &na->tx_rings[txr->me];
3597
3598 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3599
3600 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3601 BUS_DMASYNC_POSTREAD);
3602 /*
3603 * In netmap mode, all the work is done in the context
3604 * of the client thread. Interrupt handlers only wake up
3605 * clients, which may be sleeping on individual rings
3606 * or on a global resource for all rings.
3607 * To implement tx interrupt mitigation, we wake up the client
3608 * thread roughly every half ring, even if the NIC interrupts
3609 * more frequently. This is implemented as follows:
3610 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3611 * the slot that should wake up the thread (nkr_num_slots
3612 * means the user thread should not be woken up);
3613 * - the driver ignores tx interrupts unless netmap_mitigate=0
3614 * or the slot has the DD bit set.
3615 *
3616 * When the driver has separate locks, we need to
3617 * release and re-acquire txlock to avoid deadlocks.
3618 * XXX see if we can find a better way.
3619 */
3620 if (!netmap_mitigate ||
3621 (kring->nr_kflags < kring->nkr_num_slots &&
3622 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3623 kring->nr_kflags = kring->nkr_num_slots;
3624 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3625 IXGBE_TX_UNLOCK(txr);
3626 IXGBE_CORE_LOCK(adapter);
3627 selwakeuppri(&na->tx_si, PI_NET);
3628 IXGBE_CORE_UNLOCK(adapter);
3629 IXGBE_TX_LOCK(txr);
3630 }
3631 return FALSE;
3632 }
3633#endif /* DEV_NETMAP */
3634
3635 if (txr->tx_avail == adapter->num_tx_desc) {
3636 txr->queue_status = IXGBE_QUEUE_IDLE;
3637 return FALSE;
3638 }
3639
3640 processed = 0;
3641 first = txr->next_to_clean;
3642 tx_buffer = &txr->tx_buffers[first];
3643 /* For cleanup we just use legacy struct */
3644 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3645 last = tx_buffer->eop_index;
3646 if (last == -1)
3647 return FALSE;
3648 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3649
3650 /*
3651 ** Get the index of the first descriptor
3652 ** BEYOND the EOP and call that 'done'.
3653 ** I do this so the comparison in the
3654 ** inner while loop below can be simple
3655 */
3656 if (++last == adapter->num_tx_desc) last = 0;
3657 done = last;
3658
3659 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3660 BUS_DMASYNC_POSTREAD);
3661 /*
3662 ** Only the EOP descriptor of a packet now has the DD
3663 ** bit set, this is what we look for...
3664 */
3665 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3666 /* We clean the range of the packet */
3667 while (first != done) {
3668 tx_desc->upper.data = 0;
3669 tx_desc->lower.data = 0;
3670 tx_desc->buffer_addr = 0;
3671 ++txr->tx_avail;
3672 ++processed;
3673
3674 if (tx_buffer->m_head) {
3675 txr->bytes +=
3676 tx_buffer->m_head->m_pkthdr.len;
3677 bus_dmamap_sync(txr->txtag,
3678 tx_buffer->map,
3679 BUS_DMASYNC_POSTWRITE);
3680 bus_dmamap_unload(txr->txtag,
3681 tx_buffer->map);
3682 m_freem(tx_buffer->m_head);
3683 tx_buffer->m_head = NULL;
3684 tx_buffer->map = NULL;
3685 }
3686 tx_buffer->eop_index = -1;
3687 txr->watchdog_time = ticks;
3688
3689 if (++first == adapter->num_tx_desc)
3690 first = 0;
3691
3692 tx_buffer = &txr->tx_buffers[first];
3693 tx_desc =
3694 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3695 }
3696 ++txr->packets;
3697 ++ifp->if_opackets;
3698 /* See if there is more work now */
3699 last = tx_buffer->eop_index;
3700 if (last != -1) {
3701 eop_desc =
3702 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3703 /* Get next done point */
3704 if (++last == adapter->num_tx_desc) last = 0;
3705 done = last;
3706 } else
3707 break;
3708 }
3709 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3710 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3711
3712 txr->next_to_clean = first;
3713
3714 /*
3715 ** Watchdog calculation, we know there's
3716 ** work outstanding or the first return
3717 ** would have been taken, so none processed
3718 ** for too long indicates a hang.
3719 */
3720 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3721 txr->queue_status = IXGBE_QUEUE_HUNG;
3722
3723 /* With a minimum free clear the depleted state bit. */
3724 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3725 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3726
3727 if (txr->tx_avail == adapter->num_tx_desc) {
3728 txr->queue_status = IXGBE_QUEUE_IDLE;
3729 return (FALSE);
3730 }
3731
3732 return TRUE;
3733}
3734
3735/*********************************************************************
3736 *
3737 * Refresh mbuf buffers for RX descriptor rings
3738 * - now keeps its own state so discards due to resource
3739 * exhaustion are unnecessary, if an mbuf cannot be obtained
3740 * it just returns, keeping its placeholder, thus it can simply
3741 * be recalled to try again.
3742 *
3743 **********************************************************************/
3744static void
3745ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3746{
3747 struct adapter *adapter = rxr->adapter;
3748 bus_dma_segment_t hseg[1];
3749 bus_dma_segment_t pseg[1];
3750 struct ixgbe_rx_buf *rxbuf;
3751 struct mbuf *mh, *mp;
3752 int i, j, nsegs, error;
3753 bool refreshed = FALSE;
3754
3755 i = j = rxr->next_to_refresh;
3756 /* Control the loop with one beyond */
3757 if (++j == adapter->num_rx_desc)
3758 j = 0;
3759
3760 while (j != limit) {
3761 rxbuf = &rxr->rx_buffers[i];
3762 if (rxr->hdr_split == FALSE)
3763 goto no_split;
3764
3765 if (rxbuf->m_head == NULL) {
3766 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3767 if (mh == NULL)
3768 goto update;
3769 } else
3770 mh = rxbuf->m_head;
3771
3772 mh->m_pkthdr.len = mh->m_len = MHLEN;
3773 mh->m_len = MHLEN;
3774 mh->m_flags |= M_PKTHDR;
3775 /* Get the memory mapping */
3776 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3777 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3778 if (error != 0) {
3779 kprintf("Refresh mbufs: hdr dmamap load"
3780 " failure - %d\n", error);
3781 m_free(mh);
3782 rxbuf->m_head = NULL;
3783 goto update;
3784 }
3785 rxbuf->m_head = mh;
3786 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3787 BUS_DMASYNC_PREREAD);
3788 rxr->rx_base[i].read.hdr_addr =
3789 htole64(hseg[0].ds_addr);
3790
3791no_split:
3792 if (rxbuf->m_pack == NULL) {
3793 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3794 M_PKTHDR, adapter->rx_mbuf_sz);
3795 if (mp == NULL)
3796 goto update;
3797 } else
3798 mp = rxbuf->m_pack;
3799
3800 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3801 /* Get the memory mapping */
3802 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3803 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3804 if (error != 0) {
3805 kprintf("Refresh mbufs: payload dmamap load"
3806 " failure - %d\n", error);
3807 m_free(mp);
3808 rxbuf->m_pack = NULL;
3809 goto update;
3810 }
3811 rxbuf->m_pack = mp;
3812 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3813 BUS_DMASYNC_PREREAD);
3814 rxr->rx_base[i].read.pkt_addr =
3815 htole64(pseg[0].ds_addr);
3816
3817 refreshed = TRUE;
3818 /* Next is precalculated */
3819 i = j;
3820 rxr->next_to_refresh = i;
3821 if (++j == adapter->num_rx_desc)
3822 j = 0;
3823 }
3824update:
3825 if (refreshed) /* Update hardware tail index */
3826 IXGBE_WRITE_REG(&adapter->hw,
3827 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3828 return;
3829}
3830
3831/*********************************************************************
3832 *
3833 * Allocate memory for rx_buffer structures. Since we use one
3834 * rx_buffer per received packet, the maximum number of rx_buffer's
3835 * that we'll need is equal to the number of receive descriptors
3836 * that we've allocated.
3837 *
3838 **********************************************************************/
3839static int
3840ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3841{
3842 struct adapter *adapter = rxr->adapter;
3843 device_t dev = adapter->dev;
3844 struct ixgbe_rx_buf *rxbuf;
3845 int i, bsize, error;
3846
3847 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3848 if (!(rxr->rx_buffers =
3849 (struct ixgbe_rx_buf *) kmalloc(bsize,
3850 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3851 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3852 error = ENOMEM;
3853 goto fail;
3854 }
3855
3856 if ((error = bus_dma_tag_create(NULL, /* parent */
3857 1, 0, /* alignment, bounds */
3858 BUS_SPACE_MAXADDR, /* lowaddr */
3859 BUS_SPACE_MAXADDR, /* highaddr */
3860 NULL, NULL, /* filter, filterarg */
3861 MSIZE, /* maxsize */
3862 1, /* nsegments */
3863 MSIZE, /* maxsegsize */
3864 0, /* flags */
3865 &rxr->htag))) {
3866 device_printf(dev, "Unable to create RX DMA tag\n");
3867 goto fail;
3868 }
3869
3870 if ((error = bus_dma_tag_create(NULL, /* parent */
3871 1, 0, /* alignment, bounds */
3872 BUS_SPACE_MAXADDR, /* lowaddr */
3873 BUS_SPACE_MAXADDR, /* highaddr */
3874 NULL, NULL, /* filter, filterarg */
3875 MJUM16BYTES, /* maxsize */
3876 1, /* nsegments */
3877 MJUM16BYTES, /* maxsegsize */
3878 0, /* flags */
3879 &rxr->ptag))) {
3880 device_printf(dev, "Unable to create RX DMA tag\n");
3881 goto fail;
3882 }
3883
3884 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3885 rxbuf = &rxr->rx_buffers[i];
3886 error = bus_dmamap_create(rxr->htag,
3887 BUS_DMA_NOWAIT, &rxbuf->hmap);
3888 if (error) {
3889 device_printf(dev, "Unable to create RX head map\n");
3890 goto fail;
3891 }
3892 error = bus_dmamap_create(rxr->ptag,
3893 BUS_DMA_NOWAIT, &rxbuf->pmap);
3894 if (error) {
3895 device_printf(dev, "Unable to create RX pkt map\n");
3896 goto fail;
3897 }
3898 }
3899
3900 return (0);
3901
3902fail:
3903 /* Frees all, but can handle partial completion */
3904 ixgbe_free_receive_structures(adapter);
3905 return (error);
3906}
3907
3908/*
3909** Used to detect a descriptor that has
3910** been merged by Hardware RSC.
3911*/
3912static inline u32
3913ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3914{
3915 return (le32toh(rx->wb.lower.lo_dword.data) &
3916 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3917}
3918
3919/*********************************************************************
3920 *
3921 * Initialize Hardware RSC (LRO) feature on 82599
3922 * for an RX ring, this is toggled by the LRO capability
3923 * even though it is transparent to the stack.
3924 *
3925 **********************************************************************/
3926#if 0 /* NET_LRO */
3927static void
3928ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3929{
3930 struct adapter *adapter = rxr->adapter;
3931 struct ixgbe_hw *hw = &adapter->hw;
3932 u32 rscctrl, rdrxctl;
3933
3934 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3935 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3936#ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3937 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3938#endif /* DEV_NETMAP */
3939 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3940 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3941 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3942
3943 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3944 rscctrl |= IXGBE_RSCCTL_RSCEN;
3945 /*
3946 ** Limit the total number of descriptors that
3947 ** can be combined, so it does not exceed 64K
3948 */
3949 if (adapter->rx_mbuf_sz == MCLBYTES)
3950 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3951 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3952 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3953 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3954 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3955 else /* Using 16K cluster */
3956 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3957
3958 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3959
3960 /* Enable TCP header recognition */
3961 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3962 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3963 IXGBE_PSRTYPE_TCPHDR));
3964
3965 /* Disable RSC for ACK packets */
3966 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3967 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3968
3969 rxr->hw_rsc = TRUE;
3970}
3971#endif
3972
3973static void
3974ixgbe_free_receive_ring(struct rx_ring *rxr)
3975{
3976 struct adapter *adapter;
3977 struct ixgbe_rx_buf *rxbuf;
3978 int i;
3979
3980 adapter = rxr->adapter;
3981 for (i = 0; i < adapter->num_rx_desc; i++) {
3982 rxbuf = &rxr->rx_buffers[i];
3983 if (rxbuf->m_head != NULL) {
3984 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3985 BUS_DMASYNC_POSTREAD);
3986 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3987 rxbuf->m_head->m_flags |= M_PKTHDR;
3988 m_freem(rxbuf->m_head);
3989 }
3990 if (rxbuf->m_pack != NULL) {
3991 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3992 BUS_DMASYNC_POSTREAD);
3993 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3994 rxbuf->m_pack->m_flags |= M_PKTHDR;
3995 m_freem(rxbuf->m_pack);
3996 }
3997 rxbuf->m_head = NULL;
3998 rxbuf->m_pack = NULL;
3999 }
4000}
4001
4002
4003/*********************************************************************
4004 *
4005 * Initialize a receive ring and its buffers.
4006 *
4007 **********************************************************************/
4008static int
4009ixgbe_setup_receive_ring(struct rx_ring *rxr)
4010{
4011 struct adapter *adapter;
4012 struct ifnet *ifp;
4013 device_t dev;
4014 struct ixgbe_rx_buf *rxbuf;
4015 bus_dma_segment_t pseg[1], hseg[1];
4016#if 0 /* NET_LRO */
4017 struct lro_ctrl *lro = &rxr->lro;
4018#endif
4019 int rsize, nsegs, error = 0;
4020#ifdef DEV_NETMAP
4021 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4022 struct netmap_slot *slot;
4023#endif /* DEV_NETMAP */
4024
4025 adapter = rxr->adapter;
4026 ifp = adapter->ifp;
4027 dev = adapter->dev;
4028
4029 /* Clear the ring contents */
4030 IXGBE_RX_LOCK(rxr);
4031#ifdef DEV_NETMAP
4032 /* same as in ixgbe_setup_transmit_ring() */
4033 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4034#endif /* DEV_NETMAP */
4035 rsize = roundup2(adapter->num_rx_desc *
4036 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4037 bzero((void *)rxr->rx_base, rsize);
4038
4039 /* Free current RX buffer structs and their mbufs */
4040 ixgbe_free_receive_ring(rxr);
4041
4042 /* Configure header split? */
4043 if (ixgbe_header_split)
4044 rxr->hdr_split = TRUE;
4045
4046 /* Now replenish the mbufs */
4047 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4048 struct mbuf *mh, *mp;
4049
4050 rxbuf = &rxr->rx_buffers[j];
4051#ifdef DEV_NETMAP
4052 /*
4053 * In netmap mode, fill the map and set the buffer
4054 * address in the NIC ring, considering the offset
4055 * between the netmap and NIC rings (see comment in
4056 * ixgbe_setup_transmit_ring() ). No need to allocate
4057 * an mbuf, so end the block with a continue;
4058 */
4059 if (slot) {
4060 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4061 uint64_t paddr;
4062 void *addr;
4063
4064 addr = PNMB(slot + sj, &paddr);
4065 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4066 /* Update descriptor */
4067 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4068 continue;
4069 }
4070#endif /* DEV_NETMAP */
4071 /*
4072 ** Don't allocate mbufs if not
4073 ** doing header split, its wasteful
4074 */
4075 if (rxr->hdr_split == FALSE)
4076 goto skip_head;
4077
4078 /* First the header */
4079 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4080 if (rxbuf->m_head == NULL) {
4081 error = ENOBUFS;
4082 goto fail;
4083 }
4084 m_adj(rxbuf->m_head, ETHER_ALIGN);
4085 mh = rxbuf->m_head;
4086 mh->m_len = mh->m_pkthdr.len = MHLEN;
4087 mh->m_flags |= M_PKTHDR;
4088 /* Get the memory mapping */
4089 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4090 rxbuf->hmap, rxbuf->m_head, hseg, 1,
4091 &nsegs, BUS_DMA_NOWAIT);
4092
4093 if (error != 0) /* Nothing elegant to do here */
4094 goto fail;
4095 bus_dmamap_sync(rxr->htag,
4096 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4097 /* Update descriptor */
4098 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4099
4100skip_head:
4101 /* Now the payload cluster */
4102 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4103 M_PKTHDR, adapter->rx_mbuf_sz);
4104 if (rxbuf->m_pack == NULL) {
4105 error = ENOBUFS;
4106 goto fail;
4107 }
4108 mp = rxbuf->m_pack;
4109 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4110 /* Get the memory mapping */
4111 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4112 rxbuf->pmap, mp, hseg, 1,
4113 &nsegs, BUS_DMA_NOWAIT);
4114 if (error != 0)
4115 goto fail;
4116 bus_dmamap_sync(rxr->ptag,
4117 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4118 /* Update descriptor */
4119 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4120 }
4121
4122
4123 /* Setup our descriptor indices */