ixgbe: remove unneeded lockuninit call
[dragonfly.git] / sys / dev / netif / ixgbe / ixgbe.c
CommitLineData
9407f759
FT
1/******************************************************************************
2
3 Copyright (c) 2001-2012, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
ce451236 33/*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
9407f759
FT
34
35#include "opt_inet.h"
36#include "opt_inet6.h"
37
38#include "ixgbe.h"
39
40/*********************************************************************
41 * Set this to one to display debug statistics
42 *********************************************************************/
43int ixgbe_display_debug_stats = 0;
44
45/*********************************************************************
46 * Driver version
47 *********************************************************************/
ce451236 48char ixgbe_driver_version[] = "2.4.8";
9407f759
FT
49
50/*********************************************************************
51 * PCI Device ID Table
52 *
53 * Used by probe to select devices to load on
54 * Last field stores an index into ixgbe_strings
55 * Last entry must be all 0s
56 *
57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
58 *********************************************************************/
59
60static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
61{
62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
74 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
75 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
76 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
77 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
78 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
79 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
80 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
ce451236 81 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
9407f759
FT
82 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
83 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
ce451236 84 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
9407f759
FT
85 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
86 /* required last entry */
87 {0, 0, 0, 0, 0}
88};
89
90/*********************************************************************
91 * Table of branding strings
92 *********************************************************************/
93
94static char *ixgbe_strings[] = {
95 "Intel(R) PRO/10GbE PCI-Express Network Driver"
96};
97
98/*********************************************************************
99 * Function prototypes
100 *********************************************************************/
101static int ixgbe_probe(device_t);
102static int ixgbe_attach(device_t);
103static int ixgbe_detach(device_t);
104static int ixgbe_shutdown(device_t);
105static void ixgbe_start(struct ifnet *);
106static void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
107#if 0 /* __FreeBSD_version >= 800000 */
108static int ixgbe_mq_start(struct ifnet *, struct mbuf *);
109static int ixgbe_mq_start_locked(struct ifnet *,
110 struct tx_ring *, struct mbuf *);
111static void ixgbe_qflush(struct ifnet *);
112#endif
113static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
114static void ixgbe_init(void *);
115static void ixgbe_init_locked(struct adapter *);
116static void ixgbe_stop(void *);
117static void ixgbe_media_status(struct ifnet *, struct ifmediareq *);
118static int ixgbe_media_change(struct ifnet *);
119static void ixgbe_identify_hardware(struct adapter *);
120static int ixgbe_allocate_pci_resources(struct adapter *);
121static int ixgbe_allocate_msix(struct adapter *);
122static int ixgbe_allocate_legacy(struct adapter *);
123static int ixgbe_allocate_queues(struct adapter *);
124#if 0 /* HAVE_MSIX */
125static int ixgbe_setup_msix(struct adapter *);
126#endif
127static void ixgbe_free_pci_resources(struct adapter *);
128static void ixgbe_local_timer(void *);
129static int ixgbe_setup_interface(device_t, struct adapter *);
130static void ixgbe_config_link(struct adapter *);
131
132static int ixgbe_allocate_transmit_buffers(struct tx_ring *);
133static int ixgbe_setup_transmit_structures(struct adapter *);
134static void ixgbe_setup_transmit_ring(struct tx_ring *);
135static void ixgbe_initialize_transmit_units(struct adapter *);
136static void ixgbe_free_transmit_structures(struct adapter *);
137static void ixgbe_free_transmit_buffers(struct tx_ring *);
138
139static int ixgbe_allocate_receive_buffers(struct rx_ring *);
140static int ixgbe_setup_receive_structures(struct adapter *);
141static int ixgbe_setup_receive_ring(struct rx_ring *);
142static void ixgbe_initialize_receive_units(struct adapter *);
143static void ixgbe_free_receive_structures(struct adapter *);
144static void ixgbe_free_receive_buffers(struct rx_ring *);
145#if 0 /* NET_LRO */
146static void ixgbe_setup_hw_rsc(struct rx_ring *);
147#endif
148
149static void ixgbe_enable_intr(struct adapter *);
150static void ixgbe_disable_intr(struct adapter *);
151static void ixgbe_update_stats_counters(struct adapter *);
152static bool ixgbe_txeof(struct tx_ring *);
153static bool ixgbe_rxeof(struct ix_queue *, int);
154static void ixgbe_rx_checksum(u32, struct mbuf *, u32);
155static void ixgbe_set_promisc(struct adapter *);
156static void ixgbe_set_multi(struct adapter *);
157static void ixgbe_update_link_status(struct adapter *);
158static void ixgbe_refresh_mbufs(struct rx_ring *, int);
159static int ixgbe_xmit(struct tx_ring *, struct mbuf **);
160static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
161static int ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
162static int ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
163static int ixgbe_dma_malloc(struct adapter *, bus_size_t,
164 struct ixgbe_dma_alloc *, int);
165static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
166static void ixgbe_add_rx_process_limit(struct adapter *, const char *,
167 const char *, int *, int);
168static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
169#if 0 /* NET_TSO */
170static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
171#endif
172static void ixgbe_set_ivar(struct adapter *, u8, u8, s8);
173static void ixgbe_configure_ivars(struct adapter *);
174static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
175
176static void ixgbe_setup_vlan_hw_support(struct adapter *);
177static void ixgbe_register_vlan(void *, struct ifnet *, u16);
178static void ixgbe_unregister_vlan(void *, struct ifnet *, u16);
179
180static void ixgbe_add_hw_stats(struct adapter *adapter);
181
182static __inline void ixgbe_rx_discard(struct rx_ring *, int);
183static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
184 struct mbuf *, u32);
185
186/* Support for pluggable optic modules */
187static bool ixgbe_sfp_probe(struct adapter *);
188static void ixgbe_setup_optics(struct adapter *);
189
190/* Legacy (single vector interrupt handler */
191static void ixgbe_legacy_irq(void *);
192
193/* The MSI/X Interrupt handlers */
194static void ixgbe_msix_que(void *);
195static void ixgbe_msix_link(void *);
196
197/* Deferred interrupt tasklets */
198static void ixgbe_handle_que(void *, int);
199static void ixgbe_handle_link(void *, int);
200static void ixgbe_handle_msf(void *, int);
201static void ixgbe_handle_mod(void *, int);
202
203#ifdef IXGBE_FDIR
204static void ixgbe_atr(struct tx_ring *, struct mbuf *);
205static void ixgbe_reinit_fdir(void *, int);
206#endif
207
208/*********************************************************************
209 * FreeBSD Device Interface Entry Points
210 *********************************************************************/
211
212static device_method_t ixgbe_methods[] = {
213 /* Device interface */
214 DEVMETHOD(device_probe, ixgbe_probe),
215 DEVMETHOD(device_attach, ixgbe_attach),
216 DEVMETHOD(device_detach, ixgbe_detach),
217 DEVMETHOD(device_shutdown, ixgbe_shutdown),
218 {0, 0}
219};
220
221static driver_t ixgbe_driver = {
222 "ix", ixgbe_methods, sizeof(struct adapter),
223};
224
225devclass_t ixgbe_devclass;
226DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
227
228MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
229MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
230
231/*
232** TUNEABLE PARAMETERS:
233*/
234
235/*
236** AIM: Adaptive Interrupt Moderation
237** which means that the interrupt rate
238** is varied over time based on the
239** traffic for that interrupt vector
240*/
241static int ixgbe_enable_aim = TRUE;
242TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
243
244static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
245TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
246
247/* How many packets rxeof tries to clean at a time */
248static int ixgbe_rx_process_limit = 128;
249TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
250
9407f759
FT
251/*
252** Smart speed setting, default to on
253** this only works as a compile option
254** right now as its during attach, set
255** this to 'ixgbe_smart_speed_off' to
256** disable.
257*/
258static int ixgbe_smart_speed = ixgbe_smart_speed_on;
259
7e665ea1
FT
260static int ixgbe_msi_enable = 1;
261TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
262
9407f759
FT
263/*
264 * MSIX should be the default for best performance,
265 * but this allows it to be forced off for testing.
266 */
267static int ixgbe_enable_msix = 1;
268TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
269
270/*
271 * Header split: this causes the hardware to DMA
272 * the header into a separate mbuf from the payload,
273 * it can be a performance win in some workloads, but
274 * in others it actually hurts, its off by default.
275 */
276static int ixgbe_header_split = FALSE;
277TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
278
279/*
280 * Number of Queues, can be set to 0,
281 * it then autoconfigures based on the
282 * number of cpus with a max of 8. This
283 * can be overriden manually here.
284 */
285static int ixgbe_num_queues = 0;
286TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
287
288/*
289** Number of TX descriptors per ring,
290** setting higher than RX as this seems
291** the better performing choice.
292*/
293static int ixgbe_txd = PERFORM_TXD;
294TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
295
296/* Number of RX descriptors per ring */
297static int ixgbe_rxd = PERFORM_RXD;
298TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
299
300/* Keep running tab on them for sanity check */
301static int ixgbe_total_ports;
302
303#ifdef IXGBE_FDIR
304/*
305** For Flow Director: this is the
306** number of TX packets we sample
307** for the filter pool, this means
308** every 20th packet will be probed.
309**
310** This feature can be disabled by
311** setting this to 0.
312*/
313static int atr_sample_rate = 20;
314/*
315** Flow Director actually 'steals'
316** part of the packet buffer as its
317** filter pool, this variable controls
318** how much it uses:
319** 0 = 64K, 1 = 128K, 2 = 256K
320*/
321static int fdir_pballoc = 1;
322#endif
323
324#ifdef DEV_NETMAP
325/*
326 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
327 * be a reference on how to implement netmap support in a driver.
328 * Additional comments are in ixgbe_netmap.h .
329 *
330 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
331 * that extend the standard driver.
332 */
333#include <dev/netmap/ixgbe_netmap.h>
334#endif /* DEV_NETMAP */
335
336/*********************************************************************
337 * Device identification routine
338 *
339 * ixgbe_probe determines if the driver should be loaded on
340 * adapter based on PCI vendor/device id of the adapter.
341 *
342 * return BUS_PROBE_DEFAULT on success, positive on failure
343 *********************************************************************/
344
345static int
346ixgbe_probe(device_t dev)
347{
348 ixgbe_vendor_info_t *ent;
349
350 u16 pci_vendor_id = 0;
351 u16 pci_device_id = 0;
352 u16 pci_subvendor_id = 0;
353 u16 pci_subdevice_id = 0;
354 char adapter_name[256];
355
356 INIT_DEBUGOUT("ixgbe_probe: begin");
357
358 pci_vendor_id = pci_get_vendor(dev);
359 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
360 return (ENXIO);
361
362 pci_device_id = pci_get_device(dev);
363 pci_subvendor_id = pci_get_subvendor(dev);
364 pci_subdevice_id = pci_get_subdevice(dev);
365
366 ent = ixgbe_vendor_info_array;
367 while (ent->vendor_id != 0) {
368 if ((pci_vendor_id == ent->vendor_id) &&
369 (pci_device_id == ent->device_id) &&
370
371 ((pci_subvendor_id == ent->subvendor_id) ||
372 (ent->subvendor_id == 0)) &&
373
374 ((pci_subdevice_id == ent->subdevice_id) ||
375 (ent->subdevice_id == 0))) {
376 ksprintf(adapter_name, "%s, Version - %s",
377 ixgbe_strings[ent->index],
378 ixgbe_driver_version);
379 device_set_desc_copy(dev, adapter_name);
380 ++ixgbe_total_ports;
381 return (BUS_PROBE_DEFAULT);
382 }
383 ent++;
384 }
385 return (ENXIO);
386}
387
388/*********************************************************************
389 * Device initialization routine
390 *
391 * The attach entry point is called when the driver is being loaded.
392 * This routine identifies the type of hardware, allocates all resources
393 * and initializes the hardware.
394 *
395 * return 0 on success, positive on failure
396 *********************************************************************/
397
398static int
399ixgbe_attach(device_t dev)
400{
401 struct adapter *adapter;
402 struct ixgbe_hw *hw;
403 int error = 0;
404 u16 csum;
405 u32 ctrl_ext;
406
407 INIT_DEBUGOUT("ixgbe_attach: begin");
408
409 if (resource_disabled("ixgbe", device_get_unit(dev))) {
410 device_printf(dev, "Disabled by device hint\n");
411 return (ENXIO);
412 }
413
414 /* Allocate, clear, and link in our adapter structure */
415 adapter = device_get_softc(dev);
416 adapter->dev = adapter->osdep.dev = dev;
417 hw = &adapter->hw;
418
419 /* Core Lock Init*/
420 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
9407f759
FT
421
422 /* SYSCTL APIs */
423
424 sysctl_ctx_init(&adapter->sysctl_ctx);
425 adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
426 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
427 device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
428 if (adapter->sysctl_tree == NULL) {
429 device_printf(adapter->dev, "can't add sysctl node\n");
430 return (EINVAL);
431 }
432 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
433 SYSCTL_CHILDREN(adapter->sysctl_tree),
434 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
435 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
436
437 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
438 SYSCTL_CHILDREN(adapter->sysctl_tree),
439 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
440 &ixgbe_enable_aim, 1, "Interrupt Moderation");
441
442 /*
443 ** Allow a kind of speed control by forcing the autoneg
444 ** advertised speed list to only a certain value, this
445 ** supports 1G on 82599 devices, and 100Mb on x540.
446 */
447 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
448 SYSCTL_CHILDREN(adapter->sysctl_tree),
449 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
450 adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
451
452 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
453 SYSCTL_CHILDREN(adapter->sysctl_tree),
454 OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
455 0, ixgbe_set_thermal_test, "I", "Thermal Test");
456
457 /* Set up the timer callout */
9d1692ad 458 callout_init_mp(&adapter->timer);
9407f759
FT
459
460 /* Determine hardware revision */
461 ixgbe_identify_hardware(adapter);
462
463 /* Do base PCI setup - map BAR0 */
464 if (ixgbe_allocate_pci_resources(adapter)) {
465 device_printf(dev, "Allocation of PCI resources failed\n");
466 error = ENXIO;
467 goto err_out;
468 }
469
470 /* Do descriptor calc and sanity checks */
471 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
472 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
473 device_printf(dev, "TXD config issue, using default!\n");
474 adapter->num_tx_desc = DEFAULT_TXD;
475 } else
476 adapter->num_tx_desc = ixgbe_txd;
477
478 /*
479 ** With many RX rings it is easy to exceed the
480 ** system mbuf allocation. Tuning nmbclusters
481 ** can alleviate this.
482 */
483 if (nmbclusters > 0 ) {
484 int s;
485 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
486 if (s > nmbclusters) {
487 device_printf(dev, "RX Descriptors exceed "
488 "system mbuf max, using default instead!\n");
489 ixgbe_rxd = DEFAULT_RXD;
490 }
491 }
492
493 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
494 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
495 device_printf(dev, "RXD config issue, using default!\n");
496 adapter->num_rx_desc = DEFAULT_RXD;
497 } else
498 adapter->num_rx_desc = ixgbe_rxd;
499
500 /* Allocate our TX/RX Queues */
501 if (ixgbe_allocate_queues(adapter)) {
502 error = ENOMEM;
503 goto err_out;
504 }
505
506 /* Allocate multicast array memory. */
507 adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
508 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
509 if (adapter->mta == NULL) {
510 device_printf(dev, "Can not allocate multicast setup array\n");
511 error = ENOMEM;
512 goto err_late;
513 }
514
515 /* Initialize the shared code */
516 error = ixgbe_init_shared_code(hw);
517 if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
518 /*
519 ** No optics in this port, set up
520 ** so the timer routine will probe
521 ** for later insertion.
522 */
523 adapter->sfp_probe = TRUE;
524 error = 0;
525 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
526 device_printf(dev,"Unsupported SFP+ module detected!\n");
527 error = EIO;
528 goto err_late;
529 } else if (error) {
530 device_printf(dev,"Unable to initialize the shared code\n");
531 error = EIO;
532 goto err_late;
533 }
534
535 /* Make sure we have a good EEPROM before we read from it */
536 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
537 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
538 error = EIO;
539 goto err_late;
540 }
541
9407f759 542 error = ixgbe_init_hw(hw);
ce451236
FT
543 switch (error) {
544 case IXGBE_ERR_EEPROM_VERSION:
9407f759
FT
545 device_printf(dev, "This device is a pre-production adapter/"
546 "LOM. Please be aware there may be issues associated "
547 "with your hardware.\n If you are experiencing problems "
548 "please contact your Intel or hardware representative "
549 "who provided you with this hardware.\n");
ce451236
FT
550 break;
551 case IXGBE_ERR_SFP_NOT_SUPPORTED:
9407f759 552 device_printf(dev,"Unsupported SFP+ Module\n");
9407f759
FT
553 error = EIO;
554 device_printf(dev,"Hardware Initialization Failure\n");
555 goto err_late;
ce451236
FT
556 case IXGBE_ERR_SFP_NOT_PRESENT:
557 device_printf(dev,"No SFP+ Module found\n");
558 /* falls thru */
559 default:
560 break;
9407f759
FT
561 }
562
563 /* Detect and set physical type */
564 ixgbe_setup_optics(adapter);
565
566 if ((adapter->msix > 1) && (ixgbe_enable_msix))
567 error = ixgbe_allocate_msix(adapter);
568 else
569 error = ixgbe_allocate_legacy(adapter);
570 if (error)
571 goto err_late;
572
573 /* Setup OS specific network interface */
574 if (ixgbe_setup_interface(dev, adapter) != 0)
575 goto err_late;
576
577 /* Sysctl for limiting the amount of work done in the taskqueue */
578 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
579 "max number of rx packets to process", &adapter->rx_process_limit,
580 ixgbe_rx_process_limit);
581
582 /* Initialize statistics */
583 ixgbe_update_stats_counters(adapter);
584
585 /* Register for VLAN events */
586 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
587 ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
588 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
589 ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
590
591 /* Print PCIE bus type/speed/width info */
592 ixgbe_get_bus_info(hw);
593 device_printf(dev,"PCI Express Bus: Speed %s %s\n",
594 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
595 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
596 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
597 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
598 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
599 ("Unknown"));
600
601 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
602 (hw->bus.speed == ixgbe_bus_speed_2500)) {
603 device_printf(dev, "PCI-Express bandwidth available"
604 " for this card\n is not sufficient for"
605 " optimal performance.\n");
606 device_printf(dev, "For optimal performance a x8 "
607 "PCIE, or x4 PCIE 2 slot is required.\n");
608 }
609
610 /* let hardware know driver is loaded */
611 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
612 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
613 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
614
615 ixgbe_add_hw_stats(adapter);
616
617#ifdef DEV_NETMAP
618 ixgbe_netmap_attach(adapter);
619#endif /* DEV_NETMAP */
620 INIT_DEBUGOUT("ixgbe_attach: end");
621 return (0);
622err_late:
623 ixgbe_free_transmit_structures(adapter);
624 ixgbe_free_receive_structures(adapter);
625err_out:
626 if (adapter->ifp != NULL)
627 if_free(adapter->ifp);
628 ixgbe_free_pci_resources(adapter);
629 kfree(adapter->mta, M_DEVBUF);
630 return (error);
631
632}
633
634/*********************************************************************
635 * Device removal routine
636 *
637 * The detach entry point is called when the driver is being removed.
638 * This routine stops the adapter and deallocates all the resources
639 * that were allocated for driver operation.
640 *
641 * return 0 on success, positive on failure
642 *********************************************************************/
643
644static int
645ixgbe_detach(device_t dev)
646{
647 struct adapter *adapter = device_get_softc(dev);
648 struct ix_queue *que = adapter->queues;
649 u32 ctrl_ext;
650
651 INIT_DEBUGOUT("ixgbe_detach: begin");
652
653#ifdef NET_VLAN
654 /* Make sure VLANS are not using driver */
655 if (adapter->ifp->if_vlantrunk != NULL) {
656 device_printf(dev,"Vlan in use, detach first\n");
657 return (EBUSY);
658 }
659#endif
660
661 IXGBE_CORE_LOCK(adapter);
662 ixgbe_stop(adapter);
663 IXGBE_CORE_UNLOCK(adapter);
664
665 for (int i = 0; i < adapter->num_queues; i++, que++) {
666 if (que->tq) {
667 taskqueue_drain(que->tq, &que->que_task);
668 taskqueue_free(que->tq);
669 }
670 }
671
672 /* Drain the Link queue */
673 if (adapter->tq) {
674 taskqueue_drain(adapter->tq, &adapter->link_task);
675 taskqueue_drain(adapter->tq, &adapter->mod_task);
676 taskqueue_drain(adapter->tq, &adapter->msf_task);
677#ifdef IXGBE_FDIR
678 taskqueue_drain(adapter->tq, &adapter->fdir_task);
679#endif
680 taskqueue_free(adapter->tq);
681 }
682
683 /* let hardware know driver is unloading */
684 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
685 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
686 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
687
688 /* Unregister VLAN events */
689#ifdef NET_VLAN
690 if (adapter->vlan_attach != NULL)
691 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
692 if (adapter->vlan_detach != NULL)
693 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
694#endif
695
696 ether_ifdetach(adapter->ifp);
697 callout_stop(&adapter->timer);
9407f759
FT
698#ifdef DEV_NETMAP
699 netmap_detach(adapter->ifp);
700#endif /* DEV_NETMAP */
701 ixgbe_free_pci_resources(adapter);
702 bus_generic_detach(dev);
703 if_free(adapter->ifp);
704
705 ixgbe_free_transmit_structures(adapter);
706 ixgbe_free_receive_structures(adapter);
707 kfree(adapter->mta, M_DEVBUF);
708 sysctl_ctx_free(&adapter->sysctl_ctx);
709
9407f759
FT
710 IXGBE_CORE_LOCK_DESTROY(adapter);
711 return (0);
712}
713
714/*********************************************************************
715 *
716 * Shutdown entry point
717 *
718 **********************************************************************/
719
720static int
721ixgbe_shutdown(device_t dev)
722{
723 struct adapter *adapter = device_get_softc(dev);
724 IXGBE_CORE_LOCK(adapter);
725 ixgbe_stop(adapter);
726 IXGBE_CORE_UNLOCK(adapter);
727 return (0);
728}
729
730
731/*********************************************************************
732 * Transmit entry point
733 *
734 * ixgbe_start is called by the stack to initiate a transmit.
735 * The driver will remain in this routine as long as there are
736 * packets to transmit and transmit resources are available.
737 * In case resources are not available stack is notified and
738 * the packet is requeued.
739 **********************************************************************/
740
741static void
742ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
743{
744 struct mbuf *m_head;
745 struct adapter *adapter = txr->adapter;
746
747 IXGBE_TX_LOCK_ASSERT(txr);
748
749 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
750 return;
751 if (!adapter->link_active)
752 return;
753
754 while (!ifq_is_empty(&ifp->if_snd)) {
755 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
756 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
757 break;
758 }
759
760 m_head = ifq_dequeue(&ifp->if_snd, NULL);
761 if (m_head == NULL)
762 break;
763
764 if (ixgbe_xmit(txr, &m_head)) {
765#if 0 /* XXX: prepend to an ALTQ queue ? */
766 if (m_head != NULL)
767 IF_PREPEND(&ifp->if_snd, m_head);
768#endif
769 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
770 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
771 break;
772 }
773 /* Send a copy of the frame to the BPF listener */
774 ETHER_BPF_MTAP(ifp, m_head);
775
776 /* Set watchdog on */
777 txr->watchdog_time = ticks;
778 txr->queue_status = IXGBE_QUEUE_WORKING;
779
780 }
781 return;
782}
783
784/*
785 * Legacy TX start - called by the stack, this
786 * always uses the first tx ring, and should
787 * not be used with multiqueue tx enabled.
788 */
789static void
790ixgbe_start(struct ifnet *ifp)
791{
792 struct adapter *adapter = ifp->if_softc;
793 struct tx_ring *txr = adapter->tx_rings;
794
795 if (ifp->if_flags & IFF_RUNNING) {
796 IXGBE_TX_LOCK(txr);
797 ixgbe_start_locked(txr, ifp);
798 IXGBE_TX_UNLOCK(txr);
799 }
800 return;
801}
802
803#if 0 /* __FreeBSD_version >= 800000 */
804/*
805** Multiqueue Transmit driver
806**
807*/
808static int
809ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
810{
811 struct adapter *adapter = ifp->if_softc;
812 struct ix_queue *que;
813 struct tx_ring *txr;
814 int i = 0, err = 0;
815
816 /* Which queue to use */
817 if ((m->m_flags & M_FLOWID) != 0)
818 i = m->m_pkthdr.flowid % adapter->num_queues;
819 else
820 i = curcpu % adapter->num_queues;
821
822 txr = &adapter->tx_rings[i];
823 que = &adapter->queues[i];
824
825 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
826 IXGBE_TX_TRYLOCK(txr)) {
827 err = ixgbe_mq_start_locked(ifp, txr, m);
828 IXGBE_TX_UNLOCK(txr);
829 } else {
830 err = drbr_enqueue(ifp, txr->br, m);
831 taskqueue_enqueue(que->tq, &que->que_task);
832 }
833
834 return (err);
835}
836
837static int
838ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
839{
840 struct adapter *adapter = txr->adapter;
841 struct mbuf *next;
842 int enqueued, err = 0;
843
844 if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
845 (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
846 adapter->link_active == 0) {
847 if (m != NULL)
848 err = drbr_enqueue(ifp, txr->br, m);
849 return (err);
850 }
851
852 enqueued = 0;
853 if (m == NULL) {
854 next = drbr_dequeue(ifp, txr->br);
855 } else if (drbr_needs_enqueue(ifp, txr->br)) {
856 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
857 return (err);
858 next = drbr_dequeue(ifp, txr->br);
859 } else
860 next = m;
861
862 /* Process the queue */
863 while (next != NULL) {
864 if ((err = ixgbe_xmit(txr, &next)) != 0) {
865 if (next != NULL)
866 err = drbr_enqueue(ifp, txr->br, next);
867 break;
868 }
869 enqueued++;
870 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
871 /* Send a copy of the frame to the BPF listener */
872 ETHER_BPF_MTAP(ifp, next);
873 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
874 break;
875 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
876 ixgbe_txeof(txr);
877 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
878 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
879 break;
880 }
881 next = drbr_dequeue(ifp, txr->br);
882 }
883
884 if (enqueued > 0) {
885 /* Set watchdog on */
886 txr->queue_status |= IXGBE_QUEUE_WORKING;
887 txr->watchdog_time = ticks;
888 }
889
890 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
891 ixgbe_txeof(txr);
892
893 return (err);
894}
895
896/*
897** Flush all ring buffers
898*/
899static void
900ixgbe_qflush(struct ifnet *ifp)
901{
902 struct adapter *adapter = ifp->if_softc;
903 struct tx_ring *txr = adapter->tx_rings;
904 struct mbuf *m;
905
906 for (int i = 0; i < adapter->num_queues; i++, txr++) {
907 IXGBE_TX_LOCK(txr);
908 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
909 m_freem(m);
910 IXGBE_TX_UNLOCK(txr);
911 }
912 if_qflush(ifp);
913}
914#endif /* __FreeBSD_version >= 800000 */
915
916/*********************************************************************
917 * Ioctl entry point
918 *
919 * ixgbe_ioctl is called when the user wants to configure the
920 * interface.
921 *
922 * return 0 on success, positive on failure
923 **********************************************************************/
924
925static int
926ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
927{
928 struct adapter *adapter = ifp->if_softc;
929 struct ifreq *ifr = (struct ifreq *) data;
930#if defined(INET) || defined(INET6)
931 struct ifaddr *ifa = (struct ifaddr *)data;
932 bool avoid_reset = FALSE;
933#endif
934 int error = 0;
935
936 switch (command) {
937
938 case SIOCSIFADDR:
939#ifdef INET
940 if (ifa->ifa_addr->sa_family == AF_INET)
941 avoid_reset = TRUE;
942#endif
943#ifdef INET6
944 if (ifa->ifa_addr->sa_family == AF_INET6)
945 avoid_reset = TRUE;
946#endif
947#if defined(INET) || defined(INET6)
948 /*
949 ** Calling init results in link renegotiation,
950 ** so we avoid doing it when possible.
951 */
952 if (avoid_reset) {
953 ifp->if_flags |= IFF_UP;
954 if (!(ifp->if_flags & IFF_RUNNING))
955 ixgbe_init(adapter);
956 if (!(ifp->if_flags & IFF_NOARP))
957 arp_ifinit(ifp, ifa);
958 } else
959 error = ether_ioctl(ifp, command, data);
960#endif
961 break;
962 case SIOCSIFMTU:
963 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
964 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
965 error = EINVAL;
966 } else {
967 IXGBE_CORE_LOCK(adapter);
968 ifp->if_mtu = ifr->ifr_mtu;
969 adapter->max_frame_size =
970 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
971 ixgbe_init_locked(adapter);
972 IXGBE_CORE_UNLOCK(adapter);
973 }
974 break;
975 case SIOCSIFFLAGS:
976 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
977 IXGBE_CORE_LOCK(adapter);
978 if (ifp->if_flags & IFF_UP) {
979 if ((ifp->if_flags & IFF_RUNNING)) {
980 if ((ifp->if_flags ^ adapter->if_flags) &
981 (IFF_PROMISC | IFF_ALLMULTI)) {
982 ixgbe_set_promisc(adapter);
983 }
984 } else
985 ixgbe_init_locked(adapter);
986 } else
987 if (ifp->if_flags & IFF_RUNNING)
988 ixgbe_stop(adapter);
989 adapter->if_flags = ifp->if_flags;
990 IXGBE_CORE_UNLOCK(adapter);
991 break;
992 case SIOCADDMULTI:
993 case SIOCDELMULTI:
994 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
995 if (ifp->if_flags & IFF_RUNNING) {
996 IXGBE_CORE_LOCK(adapter);
997 ixgbe_disable_intr(adapter);
998 ixgbe_set_multi(adapter);
999 ixgbe_enable_intr(adapter);
1000 IXGBE_CORE_UNLOCK(adapter);
1001 }
1002 break;
1003 case SIOCSIFMEDIA:
1004 case SIOCGIFMEDIA:
1005 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1006 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1007 break;
1008 case SIOCSIFCAP:
1009 {
1010 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1011 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1012 if (mask & IFCAP_HWCSUM)
1013 ifp->if_capenable ^= IFCAP_HWCSUM;
1014#if 0 /* NET_TSO */
1015 if (mask & IFCAP_TSO4)
1016 ifp->if_capenable ^= IFCAP_TSO4;
1017 if (mask & IFCAP_TSO6)
1018 ifp->if_capenable ^= IFCAP_TSO6;
1019#endif
1020#if 0 /* NET_LRO */
1021 if (mask & IFCAP_LRO)
1022 ifp->if_capenable ^= IFCAP_LRO;
1023#endif
1024 if (mask & IFCAP_VLAN_HWTAGGING)
1025 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1026 if (mask & IFCAP_VLAN_HWFILTER)
1027 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1028#if 0 /* NET_TSO */
1029 if (mask & IFCAP_VLAN_HWTSO)
1030 ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1031#endif
1032 if (ifp->if_flags & IFF_RUNNING) {
1033 IXGBE_CORE_LOCK(adapter);
1034 ixgbe_init_locked(adapter);
1035 IXGBE_CORE_UNLOCK(adapter);
1036 }
1037#if 0
1038 VLAN_CAPABILITIES(ifp);
1039#endif
1040 break;
1041 }
1042
1043 default:
1044 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1045 error = ether_ioctl(ifp, command, data);
1046 break;
1047 }
1048
1049 return (error);
1050}
1051
1052/*********************************************************************
1053 * Init entry point
1054 *
1055 * This routine is used in two ways. It is used by the stack as
1056 * init entry point in network interface structure. It is also used
1057 * by the driver as a hw/sw initialization routine to get to a
1058 * consistent state.
1059 *
1060 * return 0 on success, positive on failure
1061 **********************************************************************/
1062#define IXGBE_MHADD_MFS_SHIFT 16
1063
1064static void
1065ixgbe_init_locked(struct adapter *adapter)
1066{
1067 struct ifnet *ifp = adapter->ifp;
1068 device_t dev = adapter->dev;
1069 struct ixgbe_hw *hw = &adapter->hw;
1070 u32 k, txdctl, mhadd, gpie;
1071 u32 rxdctl, rxctrl;
1072
1073 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1074 INIT_DEBUGOUT("ixgbe_init: begin");
1075 hw->adapter_stopped = FALSE;
1076 ixgbe_stop_adapter(hw);
1077 callout_stop(&adapter->timer);
1078
1079 /* reprogram the RAR[0] in case user changed it. */
1080 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1081
1082 /* Get the latest mac address, User can use a LAA */
1083 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1084 IXGBE_ETH_LENGTH_OF_ADDRESS);
1085 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1086 hw->addr_ctrl.rar_used_count = 1;
1087
1088 /* Set the various hardware offload abilities */
1089 ifp->if_hwassist = 0;
1090#if 0 /* NET_TSO */
1091 if (ifp->if_capenable & IFCAP_TSO)
1092 ifp->if_hwassist |= CSUM_TSO;
1093#endif
1094 if (ifp->if_capenable & IFCAP_TXCSUM) {
1095 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1096#if 0
1097 if (hw->mac.type != ixgbe_mac_82598EB)
1098 ifp->if_hwassist |= CSUM_SCTP;
1099#endif
1100 }
1101
1102 /* Prepare transmit descriptors and buffers */
1103 if (ixgbe_setup_transmit_structures(adapter)) {
1104 device_printf(dev,"Could not setup transmit structures\n");
1105 ixgbe_stop(adapter);
1106 return;
1107 }
1108
1109 ixgbe_init_hw(hw);
1110 ixgbe_initialize_transmit_units(adapter);
1111
1112 /* Setup Multicast table */
1113 ixgbe_set_multi(adapter);
1114
1115 /*
1116 ** Determine the correct mbuf pool
1117 ** for doing jumbo/headersplit
1118 */
1119 if (adapter->max_frame_size <= 2048)
1120 adapter->rx_mbuf_sz = MCLBYTES;
1121 else if (adapter->max_frame_size <= 4096)
1122 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1123 else if (adapter->max_frame_size <= 9216)
1124 adapter->rx_mbuf_sz = MJUM9BYTES;
1125 else
1126 adapter->rx_mbuf_sz = MJUM16BYTES;
1127
1128 /* Prepare receive descriptors and buffers */
1129 if (ixgbe_setup_receive_structures(adapter)) {
1130 device_printf(dev,"Could not setup receive structures\n");
1131 ixgbe_stop(adapter);
1132 return;
1133 }
1134
1135 /* Configure RX settings */
1136 ixgbe_initialize_receive_units(adapter);
1137
1138 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1139
1140 /* Enable Fan Failure Interrupt */
1141 gpie |= IXGBE_SDP1_GPIEN;
1142
1143 /* Add for Module detection */
1144 if (hw->mac.type == ixgbe_mac_82599EB)
1145 gpie |= IXGBE_SDP2_GPIEN;
1146
1147 /* Thermal Failure Detection */
1148 if (hw->mac.type == ixgbe_mac_X540)
1149 gpie |= IXGBE_SDP0_GPIEN;
1150
1151 if (adapter->msix > 1) {
1152 /* Enable Enhanced MSIX mode */
1153 gpie |= IXGBE_GPIE_MSIX_MODE;
1154 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1155 IXGBE_GPIE_OCD;
1156 }
1157 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1158
1159 /* Set MTU size */
1160 if (ifp->if_mtu > ETHERMTU) {
1161 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1162 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1163 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1164 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1165 }
1166
1167 /* Now enable all the queues */
1168
1169 for (int i = 0; i < adapter->num_queues; i++) {
1170 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1171 txdctl |= IXGBE_TXDCTL_ENABLE;
1172 /* Set WTHRESH to 8, burst writeback */
1173 txdctl |= (8 << 16);
1174 /*
1175 * When the internal queue falls below PTHRESH (32),
1176 * start prefetching as long as there are at least
1177 * HTHRESH (1) buffers ready. The values are taken
1178 * from the Intel linux driver 3.8.21.
1179 * Prefetching enables tx line rate even with 1 queue.
1180 */
1181 txdctl |= (32 << 0) | (1 << 8);
1182 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1183 }
1184
1185 for (int i = 0; i < adapter->num_queues; i++) {
1186 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1187 if (hw->mac.type == ixgbe_mac_82598EB) {
1188 /*
1189 ** PTHRESH = 21
1190 ** HTHRESH = 4
1191 ** WTHRESH = 8
1192 */
1193 rxdctl &= ~0x3FFFFF;
1194 rxdctl |= 0x080420;
1195 }
1196 rxdctl |= IXGBE_RXDCTL_ENABLE;
1197 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1198 for (k = 0; k < 10; k++) {
1199 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1200 IXGBE_RXDCTL_ENABLE)
1201 break;
1202 else
1203 msec_delay(1);
1204 }
1205 wmb();
1206#ifdef DEV_NETMAP
1207 /*
1208 * In netmap mode, we must preserve the buffers made
1209 * available to userspace before the if_init()
1210 * (this is true by default on the TX side, because
1211 * init makes all buffers available to userspace).
1212 *
1213 * netmap_reset() and the device specific routines
1214 * (e.g. ixgbe_setup_receive_rings()) map these
1215 * buffers at the end of the NIC ring, so here we
1216 * must set the RDT (tail) register to make sure
1217 * they are not overwritten.
1218 *
1219 * In this driver the NIC ring starts at RDH = 0,
1220 * RDT points to the last slot available for reception (?),
1221 * so RDT = num_rx_desc - 1 means the whole ring is available.
1222 */
1223 if (ifp->if_capenable & IFCAP_NETMAP) {
1224 struct netmap_adapter *na = NA(adapter->ifp);
1225 struct netmap_kring *kring = &na->rx_rings[i];
1226 int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1227
1228 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1229 } else
1230#endif /* DEV_NETMAP */
1231 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1232 }
1233
1234 /* Set up VLAN support and filter */
1235 ixgbe_setup_vlan_hw_support(adapter);
1236
1237 /* Enable Receive engine */
1238 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1239 if (hw->mac.type == ixgbe_mac_82598EB)
1240 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1241 rxctrl |= IXGBE_RXCTRL_RXEN;
1242 ixgbe_enable_rx_dma(hw, rxctrl);
1243
1244 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1245
1246 /* Set up MSI/X routing */
1247 if (ixgbe_enable_msix) {
1248 ixgbe_configure_ivars(adapter);
1249 /* Set up auto-mask */
1250 if (hw->mac.type == ixgbe_mac_82598EB)
1251 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1252 else {
1253 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1254 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1255 }
1256 } else { /* Simple settings for Legacy/MSI */
1257 ixgbe_set_ivar(adapter, 0, 0, 0);
1258 ixgbe_set_ivar(adapter, 0, 0, 1);
1259 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1260 }
1261
1262#ifdef IXGBE_FDIR
1263 /* Init Flow director */
1264 if (hw->mac.type != ixgbe_mac_82598EB) {
ce451236 1265 u32 hdrm = 32 << fdir_pballoc;
9407f759
FT
1266
1267 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1268 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1269 }
1270#endif
1271
1272 /*
1273 ** Check on any SFP devices that
1274 ** need to be kick-started
1275 */
1276 if (hw->phy.type == ixgbe_phy_none) {
1277 int err = hw->phy.ops.identify(hw);
1278 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1279 device_printf(dev,
1280 "Unsupported SFP+ module type was detected.\n");
1281 return;
1282 }
1283 }
1284
1285 /* Set moderation on the Link interrupt */
1286 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1287
1288 /* Config/Enable Link */
1289 ixgbe_config_link(adapter);
1290
ce451236
FT
1291 /* Hardware Packet Buffer & Flow Control setup */
1292 {
1293 u32 rxpb, frame, size, tmp;
1294
1295 frame = adapter->max_frame_size;
1296
1297 /* Calculate High Water */
1298 if (hw->mac.type == ixgbe_mac_X540)
1299 tmp = IXGBE_DV_X540(frame, frame);
1300 else
1301 tmp = IXGBE_DV(frame, frame);
1302 size = IXGBE_BT2KB(tmp);
1303 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1304 hw->fc.high_water[0] = rxpb - size;
1305
1306 /* Now calculate Low Water */
1307 if (hw->mac.type == ixgbe_mac_X540)
1308 tmp = IXGBE_LOW_DV_X540(frame);
1309 else
1310 tmp = IXGBE_LOW_DV(frame);
1311 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1312
1313 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1314 hw->fc.pause_time = IXGBE_FC_PAUSE;
1315 hw->fc.send_xon = TRUE;
1316 }
1317 /* Initialize the FC settings */
1318 ixgbe_start_hw(hw);
1319
9407f759
FT
1320 /* And now turn on interrupts */
1321 ixgbe_enable_intr(adapter);
1322
1323 /* Now inform the stack we're ready */
1324 ifp->if_flags |= IFF_RUNNING;
1325 ifp->if_flags &= ~IFF_OACTIVE;
1326
1327 return;
1328}
1329
1330static void
1331ixgbe_init(void *arg)
1332{
1333 struct adapter *adapter = arg;
1334
1335 IXGBE_CORE_LOCK(adapter);
1336 ixgbe_init_locked(adapter);
1337 IXGBE_CORE_UNLOCK(adapter);
1338 return;
1339}
1340
1341
1342/*
1343**
1344** MSIX Interrupt Handlers and Tasklets
1345**
1346*/
1347
1348static inline void
1349ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1350{
1351 struct ixgbe_hw *hw = &adapter->hw;
1352 u64 queue = (u64)(1 << vector);
1353 u32 mask;
1354
1355 if (hw->mac.type == ixgbe_mac_82598EB) {
1356 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1357 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1358 } else {
1359 mask = (queue & 0xFFFFFFFF);
1360 if (mask)
1361 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1362 mask = (queue >> 32);
1363 if (mask)
1364 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1365 }
1366}
1367
1368static inline void
1369ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1370{
1371 struct ixgbe_hw *hw = &adapter->hw;
1372 u64 queue = (u64)(1 << vector);
1373 u32 mask;
1374
1375 if (hw->mac.type == ixgbe_mac_82598EB) {
1376 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1377 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1378 } else {
1379 mask = (queue & 0xFFFFFFFF);
1380 if (mask)
1381 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1382 mask = (queue >> 32);
1383 if (mask)
1384 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1385 }
1386}
1387
1388static inline void
1389ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1390{
1391 u32 mask;
1392
1393 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1394 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1395 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1396 } else {
1397 mask = (queues & 0xFFFFFFFF);
1398 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1399 mask = (queues >> 32);
1400 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1401 }
1402}
1403
1404
1405static void
1406ixgbe_handle_que(void *context, int pending)
1407{
1408 struct ix_queue *que = context;
1409 struct adapter *adapter = que->adapter;
1410 struct tx_ring *txr = que->txr;
1411 struct ifnet *ifp = adapter->ifp;
1412 bool more;
1413
1414 if (ifp->if_flags & IFF_RUNNING) {
1415 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1416 IXGBE_TX_LOCK(txr);
1417 ixgbe_txeof(txr);
1418#if 0 /*__FreeBSD_version >= 800000*/
1419 if (!drbr_empty(ifp, txr->br))
1420 ixgbe_mq_start_locked(ifp, txr, NULL);
1421#else
1422 if (!ifq_is_empty(&ifp->if_snd))
1423 ixgbe_start_locked(txr, ifp);
1424#endif
1425 IXGBE_TX_UNLOCK(txr);
1426 if (more) {
1427 taskqueue_enqueue(que->tq, &que->que_task);
1428 return;
1429 }
1430 }
1431
1432 /* Reenable this interrupt */
1433 ixgbe_enable_queue(adapter, que->msix);
1434 return;
1435}
1436
1437
1438/*********************************************************************
1439 *
1440 * Legacy Interrupt Service routine
1441 *
1442 **********************************************************************/
1443
1444static void
1445ixgbe_legacy_irq(void *arg)
1446{
1447 struct ix_queue *que = arg;
1448 struct adapter *adapter = que->adapter;
1449 struct ixgbe_hw *hw = &adapter->hw;
1450 struct tx_ring *txr = adapter->tx_rings;
1451 bool more_tx, more_rx;
1452 u32 reg_eicr, loop = MAX_LOOP;
1453
1454
1455 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1456
1457 ++que->irqs;
1458 if (reg_eicr == 0) {
1459 ixgbe_enable_intr(adapter);
1460 return;
1461 }
1462
1463 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1464
1465 IXGBE_TX_LOCK(txr);
1466 do {
1467 more_tx = ixgbe_txeof(txr);
1468 } while (loop-- && more_tx);
1469 IXGBE_TX_UNLOCK(txr);
1470
1471 if (more_rx || more_tx)
1472 taskqueue_enqueue(que->tq, &que->que_task);
1473
1474 /* Check for fan failure */
1475 if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1476 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1477 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1478 "REPLACE IMMEDIATELY!!\n");
1479 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1480 }
1481
1482 /* Link status change */
1483 if (reg_eicr & IXGBE_EICR_LSC)
1484 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1485
1486 ixgbe_enable_intr(adapter);
1487 return;
1488}
1489
1490
1491/*********************************************************************
1492 *
1493 * MSIX Queue Interrupt Service routine
1494 *
1495 **********************************************************************/
1496void
1497ixgbe_msix_que(void *arg)
1498{
1499 struct ix_queue *que = arg;
1500 struct adapter *adapter = que->adapter;
1501 struct tx_ring *txr = que->txr;
1502 struct rx_ring *rxr = que->rxr;
1503 bool more_tx, more_rx;
1504 u32 newitr = 0;
1505
1506 ixgbe_disable_queue(adapter, que->msix);
1507 ++que->irqs;
1508
1509 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1510
1511 IXGBE_TX_LOCK(txr);
1512 more_tx = ixgbe_txeof(txr);
1513 /*
1514 ** Make certain that if the stack
1515 ** has anything queued the task gets
1516 ** scheduled to handle it.
1517 */
1518#if 0
1519#if __FreeBSD_version < 800000
1520 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1521#else
1522 if (!drbr_empty(adapter->ifp, txr->br))
1523#endif
1524#endif
1525 if (!ifq_is_empty(&adapter->ifp->if_snd))
1526 more_tx = 1;
1527 IXGBE_TX_UNLOCK(txr);
1528
1529 /* Do AIM now? */
1530
1531 if (ixgbe_enable_aim == FALSE)
1532 goto no_calc;
1533 /*
1534 ** Do Adaptive Interrupt Moderation:
1535 ** - Write out last calculated setting
1536 ** - Calculate based on average size over
1537 ** the last interval.
1538 */
1539 if (que->eitr_setting)
1540 IXGBE_WRITE_REG(&adapter->hw,
1541 IXGBE_EITR(que->msix), que->eitr_setting);
1542
1543 que->eitr_setting = 0;
1544
1545 /* Idle, do nothing */
1546 if ((txr->bytes == 0) && (rxr->bytes == 0))
1547 goto no_calc;
1548
1549 if ((txr->bytes) && (txr->packets))
1550 newitr = txr->bytes/txr->packets;
1551 if ((rxr->bytes) && (rxr->packets))
1552 newitr = max(newitr,
1553 (rxr->bytes / rxr->packets));
1554 newitr += 24; /* account for hardware frame, crc */
1555
1556 /* set an upper boundary */
1557 newitr = min(newitr, 3000);
1558
1559 /* Be nice to the mid range */
1560 if ((newitr > 300) && (newitr < 1200))
1561 newitr = (newitr / 3);
1562 else
1563 newitr = (newitr / 2);
1564
1565 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1566 newitr |= newitr << 16;
1567 else
1568 newitr |= IXGBE_EITR_CNT_WDIS;
1569
1570 /* save for next interrupt */
1571 que->eitr_setting = newitr;
1572
1573 /* Reset state */
1574 txr->bytes = 0;
1575 txr->packets = 0;
1576 rxr->bytes = 0;
1577 rxr->packets = 0;
1578
1579no_calc:
1580 if (more_tx || more_rx)
1581 taskqueue_enqueue(que->tq, &que->que_task);
1582 else /* Reenable this interrupt */
1583 ixgbe_enable_queue(adapter, que->msix);
1584 return;
1585}
1586
1587
1588static void
1589ixgbe_msix_link(void *arg)
1590{
1591 struct adapter *adapter = arg;
1592 struct ixgbe_hw *hw = &adapter->hw;
1593 u32 reg_eicr;
1594
1595 ++adapter->link_irq;
1596
1597 /* First get the cause */
1598 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1599 /* Clear interrupt with write */
1600 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1601
1602 /* Link status change */
1603 if (reg_eicr & IXGBE_EICR_LSC)
1604 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1605
1606 if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1607#ifdef IXGBE_FDIR
1608 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1609 /* This is probably overkill :) */
1610 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1611 return;
ce451236
FT
1612 /* Disable the interrupt */
1613 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
9407f759
FT
1614 taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1615 } else
1616#endif
1617 if (reg_eicr & IXGBE_EICR_ECC) {
1618 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1619 "Please Reboot!!\n");
1620 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1621 } else
1622
1623 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1624 /* Clear the interrupt */
1625 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1626 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1627 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1628 /* Clear the interrupt */
1629 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1630 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1631 }
1632 }
1633
1634 /* Check for fan failure */
1635 if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1636 (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1637 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1638 "REPLACE IMMEDIATELY!!\n");
1639 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1640 }
1641
1642 /* Check for over temp condition */
1643 if ((hw->mac.type == ixgbe_mac_X540) &&
1644 (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1645 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1646 "PHY IS SHUT DOWN!!\n");
1647 device_printf(adapter->dev, "System shutdown required\n");
1648 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1649 }
1650
1651 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1652 return;
1653}
1654
1655/*********************************************************************
1656 *
1657 * Media Ioctl callback
1658 *
1659 * This routine is called whenever the user queries the status of
1660 * the interface using ifconfig.
1661 *
1662 **********************************************************************/
1663static void
1664ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1665{
1666 struct adapter *adapter = ifp->if_softc;
1667
1668 INIT_DEBUGOUT("ixgbe_media_status: begin");
1669 IXGBE_CORE_LOCK(adapter);
1670 ixgbe_update_link_status(adapter);
1671
1672 ifmr->ifm_status = IFM_AVALID;
1673 ifmr->ifm_active = IFM_ETHER;
1674
1675 if (!adapter->link_active) {
1676 IXGBE_CORE_UNLOCK(adapter);
1677 return;
1678 }
1679
1680 ifmr->ifm_status |= IFM_ACTIVE;
1681
1682 switch (adapter->link_speed) {
1683 case IXGBE_LINK_SPEED_100_FULL:
1684 ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1685 break;
1686 case IXGBE_LINK_SPEED_1GB_FULL:
1687 ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1688 break;
1689 case IXGBE_LINK_SPEED_10GB_FULL:
1690 ifmr->ifm_active |= adapter->optics | IFM_FDX;
1691 break;
1692 }
1693
1694 IXGBE_CORE_UNLOCK(adapter);
1695
1696 return;
1697}
1698
1699/*********************************************************************
1700 *
1701 * Media Ioctl callback
1702 *
1703 * This routine is called when the user changes speed/duplex using
1704 * media/mediopt option with ifconfig.
1705 *
1706 **********************************************************************/
1707static int
1708ixgbe_media_change(struct ifnet * ifp)
1709{
1710 struct adapter *adapter = ifp->if_softc;
1711 struct ifmedia *ifm = &adapter->media;
1712
1713 INIT_DEBUGOUT("ixgbe_media_change: begin");
1714
1715 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1716 return (EINVAL);
1717
1718 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1719 case IFM_AUTO:
1720 adapter->hw.phy.autoneg_advertised =
1721 IXGBE_LINK_SPEED_100_FULL |
1722 IXGBE_LINK_SPEED_1GB_FULL |
1723 IXGBE_LINK_SPEED_10GB_FULL;
1724 break;
1725 default:
1726 device_printf(adapter->dev, "Only auto media type\n");
1727 return (EINVAL);
1728 }
1729
1730 return (0);
1731}
1732
1733/*********************************************************************
1734 *
1735 * This routine maps the mbufs to tx descriptors, allowing the
1736 * TX engine to transmit the packets.
1737 * - return 0 on success, positive on failure
1738 *
1739 **********************************************************************/
1740
1741static int
1742ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1743{
1744 struct adapter *adapter = txr->adapter;
1745 u32 olinfo_status = 0, cmd_type_len;
1746 u32 paylen = 0;
1747 int i, j, error, nsegs;
1748 int first, last = 0;
1749 struct mbuf *m_head;
1750 bus_dma_segment_t segs[1];
1751 bus_dmamap_t map;
1752 struct ixgbe_tx_buf *txbuf;
1753 union ixgbe_adv_tx_desc *txd = NULL;
1754
1755 m_head = *m_headp;
1756
1757 /* Basic descriptor defines */
1758 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1759 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1760
1761 if (m_head->m_flags & M_VLANTAG)
1762 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1763
1764 /*
1765 * Important to capture the first descriptor
1766 * used because it will contain the index of
1767 * the one we tell the hardware to report back
1768 */
1769 first = txr->next_avail_desc;
1770 txbuf = &txr->tx_buffers[first];
1771 map = txbuf->map;
1772
1773 /*
1774 * Map the packet for DMA.
1775 */
1776 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1777 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1778
1779 if (error == EFBIG) {
1780 struct mbuf *m;
1781
1782 m = m_defrag(*m_headp, MB_DONTWAIT);
1783 if (m == NULL) {
1784 adapter->mbuf_defrag_failed++;
1785 m_freem(*m_headp);
1786 *m_headp = NULL;
1787 return (ENOBUFS);
1788 }
1789 *m_headp = m;
1790
1791 /* Try it again */
1792 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1793 *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1794
1795 if (error == ENOMEM) {
1796 adapter->no_tx_dma_setup++;
1797 return (error);
1798 } else if (error != 0) {
1799 adapter->no_tx_dma_setup++;
1800 m_freem(*m_headp);
1801 *m_headp = NULL;
1802 return (error);
1803 }
1804 } else if (error == ENOMEM) {
1805 adapter->no_tx_dma_setup++;
1806 return (error);
1807 } else if (error != 0) {
1808 adapter->no_tx_dma_setup++;
1809 m_freem(*m_headp);
1810 *m_headp = NULL;
1811 return (error);
1812 }
1813
1814 /* Make certain there are enough descriptors */
1815 if (nsegs > txr->tx_avail - 2) {
1816 txr->no_desc_avail++;
1817 error = ENOBUFS;
1818 goto xmit_fail;
1819 }
1820 m_head = *m_headp;
1821
1822 /*
1823 ** Set up the appropriate offload context
1824 ** this becomes the first descriptor of
1825 ** a packet.
1826 */
1827#if 0 /* NET_TSO */
1828 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1829 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1830 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1831 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1832 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1833 ++adapter->tso_tx;
1834 } else
1835 return (ENXIO);
1836 } else if (ixgbe_tx_ctx_setup(txr, m_head))
1837#endif
1838 if (ixgbe_tx_ctx_setup(txr, m_head))
1839 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1840
1841#ifdef IXGBE_IEEE1588
1842 /* This is changing soon to an mtag detection */
1843 if (we detect this mbuf has a TSTAMP mtag)
1844 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1845#endif
1846
1847#ifdef IXGBE_FDIR
1848 /* Do the flow director magic */
1849 if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1850 ++txr->atr_count;
1851 if (txr->atr_count >= atr_sample_rate) {
1852 ixgbe_atr(txr, m_head);
1853 txr->atr_count = 0;
1854 }
1855 }
1856#endif
1857 /* Record payload length */
1858 if (paylen == 0)
1859 olinfo_status |= m_head->m_pkthdr.len <<
1860 IXGBE_ADVTXD_PAYLEN_SHIFT;
1861
1862 i = txr->next_avail_desc;
1863 for (j = 0; j < nsegs; j++) {
1864 bus_size_t seglen;
1865 bus_addr_t segaddr;
1866
1867 txbuf = &txr->tx_buffers[i];
1868 txd = &txr->tx_base[i];
1869 seglen = segs[j].ds_len;
1870 segaddr = htole64(segs[j].ds_addr);
1871
1872 txd->read.buffer_addr = segaddr;
1873 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1874 cmd_type_len |seglen);
1875 txd->read.olinfo_status = htole32(olinfo_status);
1876 last = i; /* descriptor that will get completion IRQ */
1877
1878 if (++i == adapter->num_tx_desc)
1879 i = 0;
1880
1881 txbuf->m_head = NULL;
1882 txbuf->eop_index = -1;
1883 }
1884
1885 txd->read.cmd_type_len |=
1886 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1887 txr->tx_avail -= nsegs;
1888 txr->next_avail_desc = i;
1889
1890 txbuf->m_head = m_head;
1891 /* Swap the dma map between the first and last descriptor */
1892 txr->tx_buffers[first].map = txbuf->map;
1893 txbuf->map = map;
1894 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1895
1896 /* Set the index of the descriptor that will be marked done */
1897 txbuf = &txr->tx_buffers[first];
1898 txbuf->eop_index = last;
1899
1900 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1901 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1902 /*
1903 * Advance the Transmit Descriptor Tail (Tdt), this tells the
1904 * hardware that this frame is available to transmit.
1905 */
1906 ++txr->total_packets;
1907 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1908
1909 return (0);
1910
1911xmit_fail:
1912 bus_dmamap_unload(txr->txtag, txbuf->map);
1913 return (error);
1914
1915}
1916
1917static void
1918ixgbe_set_promisc(struct adapter *adapter)
1919{
1920 u_int32_t reg_rctl;
1921 struct ifnet *ifp = adapter->ifp;
1922
1923 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1924 reg_rctl &= (~IXGBE_FCTRL_UPE);
1925 reg_rctl &= (~IXGBE_FCTRL_MPE);
1926 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1927
1928 if (ifp->if_flags & IFF_PROMISC) {
1929 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1930 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1931 } else if (ifp->if_flags & IFF_ALLMULTI) {
1932 reg_rctl |= IXGBE_FCTRL_MPE;
1933 reg_rctl &= ~IXGBE_FCTRL_UPE;
1934 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1935 }
1936 return;
1937}
1938
1939
1940/*********************************************************************
1941 * Multicast Update
1942 *
1943 * This routine is called whenever multicast address list is updated.
1944 *
1945 **********************************************************************/
1946#define IXGBE_RAR_ENTRIES 16
1947
1948static void
1949ixgbe_set_multi(struct adapter *adapter)
1950{
1951 u32 fctrl;
1952 u8 *mta;
1953 u8 *update_ptr;
1954 struct ifmultiaddr *ifma;
1955 int mcnt = 0;
1956 struct ifnet *ifp = adapter->ifp;
1957
1958 IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1959
1960 mta = adapter->mta;
1961 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1962 MAX_NUM_MULTICAST_ADDRESSES);
1963
1964 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1965 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1966 if (ifp->if_flags & IFF_PROMISC)
1967 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1968 else if (ifp->if_flags & IFF_ALLMULTI) {
1969 fctrl |= IXGBE_FCTRL_MPE;
1970 fctrl &= ~IXGBE_FCTRL_UPE;
1971 } else
1972 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1973
1974 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1975
9407f759
FT
1976 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1977 if (ifma->ifma_addr->sa_family != AF_LINK)
1978 continue;
1979 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1980 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1981 IXGBE_ETH_LENGTH_OF_ADDRESS);
1982 mcnt++;
1983 }
9407f759
FT
1984
1985 update_ptr = mta;
1986 ixgbe_update_mc_addr_list(&adapter->hw,
1987 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1988
1989 return;
1990}
1991
1992/*
1993 * This is an iterator function now needed by the multicast
1994 * shared code. It simply feeds the shared code routine the
1995 * addresses in the array of ixgbe_set_multi() one by one.
1996 */
1997static u8 *
1998ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1999{
2000 u8 *addr = *update_ptr;
2001 u8 *newptr;
2002 *vmdq = 0;
2003
2004 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2005 *update_ptr = newptr;
2006 return addr;
2007}
2008
2009
2010/*********************************************************************
2011 * Timer routine
2012 *
2013 * This routine checks for link status,updates statistics,
2014 * and runs the watchdog check.
2015 *
2016 **********************************************************************/
2017
2018static void
2019ixgbe_local_timer(void *arg)
2020{
2021 struct adapter *adapter = arg;
2022 device_t dev = adapter->dev;
2023 struct ifnet *ifp = adapter->ifp;
2024 struct ix_queue *que = adapter->queues;
2025 struct tx_ring *txr = adapter->tx_rings;
2026 int hung, busy, paused;
2027
fe609108 2028 IXGBE_CORE_LOCK(adapter);
9407f759
FT
2029 hung = busy = paused = 0;
2030
2031 /* Check for pluggable optics */
2032 if (adapter->sfp_probe)
2033 if (!ixgbe_sfp_probe(adapter))
2034 goto out; /* Nothing to do */
2035
2036 ixgbe_update_link_status(adapter);
2037 ixgbe_update_stats_counters(adapter);
2038
2039 /*
2040 * If the interface has been paused
2041 * then don't do the watchdog check
2042 */
2043 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2044 paused = 1;
2045
2046 /*
2047 ** Check the TX queues status
2048 ** - central locked handling of OACTIVE
2049 ** - watchdog only if all queues show hung
2050 */
2051 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2052 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2053 (paused == 0))
2054 ++hung;
2055 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2056 ++busy;
2057 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2058 taskqueue_enqueue(que->tq, &que->que_task);
2059 }
2060 /* Only truely watchdog if all queues show hung */
2061 if (hung == adapter->num_queues)
2062 goto watchdog;
2063 /* Only turn off the stack flow when ALL are depleted */
2064 if (busy == adapter->num_queues)
2065 ifp->if_flags |= IFF_OACTIVE;
2066 else if ((ifp->if_flags & IFF_OACTIVE) &&
2067 (busy < adapter->num_queues))
2068 ifp->if_flags &= ~IFF_OACTIVE;
2069
2070out:
2071 ixgbe_rearm_queues(adapter, adapter->que_mask);
2072 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
fe609108 2073 IXGBE_CORE_UNLOCK(adapter);
9407f759
FT
2074 return;
2075
2076watchdog:
2077 device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2078 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2079 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2080 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2081 device_printf(dev,"TX(%d) desc avail = %d,"
2082 "Next TX to Clean = %d\n",
2083 txr->me, txr->tx_avail, txr->next_to_clean);
2084 adapter->ifp->if_flags &= ~IFF_RUNNING;
2085 adapter->watchdog_events++;
2086 ixgbe_init_locked(adapter);
2087
fe609108 2088 IXGBE_CORE_UNLOCK(adapter);
9407f759
FT
2089}
2090
2091/*
2092** Note: this routine updates the OS on the link state
2093** the real check of the hardware only happens with
2094** a link interrupt.
2095*/
2096static void
2097ixgbe_update_link_status(struct adapter *adapter)
2098{
2099 struct ifnet *ifp = adapter->ifp;
2100 struct tx_ring *txr = adapter->tx_rings;
2101 device_t dev = adapter->dev;
2102
2103
2104 if (adapter->link_up){
2105 if (adapter->link_active == FALSE) {
2106 if (bootverbose)
2107 device_printf(dev,"Link is up %d Gbps %s \n",
2108 ((adapter->link_speed == 128)? 10:1),
2109 "Full Duplex");
2110 adapter->link_active = TRUE;
ce451236
FT
2111 /* Update any Flow Control changes */
2112 ixgbe_fc_enable(&adapter->hw);
9407f759
FT
2113 ifp->if_link_state = LINK_STATE_UP;
2114 if_link_state_change(ifp);
2115 }
2116 } else { /* Link down */
2117 if (adapter->link_active == TRUE) {
2118 if (bootverbose)
2119 device_printf(dev,"Link is Down\n");
2120 ifp->if_link_state = LINK_STATE_DOWN;
2121 if_link_state_change(ifp);
2122 adapter->link_active = FALSE;
2123 for (int i = 0; i < adapter->num_queues;
2124 i++, txr++)
2125 txr->queue_status = IXGBE_QUEUE_IDLE;
2126 }
2127 }
2128
2129 return;
2130}
2131
2132
2133/*********************************************************************
2134 *
2135 * This routine disables all traffic on the adapter by issuing a
2136 * global reset on the MAC and deallocates TX/RX buffers.
2137 *
2138 **********************************************************************/
2139
2140static void
2141ixgbe_stop(void *arg)
2142{
2143 struct ifnet *ifp;
2144 struct adapter *adapter = arg;
2145 struct ixgbe_hw *hw = &adapter->hw;
2146 ifp = adapter->ifp;
2147
2148 KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2149
2150 INIT_DEBUGOUT("ixgbe_stop: begin\n");
2151 ixgbe_disable_intr(adapter);
2152 callout_stop(&adapter->timer);
2153
2154 /* Let the stack know...*/
2155 ifp->if_flags &= ~IFF_RUNNING;
2156 ifp->if_flags |= IFF_OACTIVE;
2157
2158 ixgbe_reset_hw(hw);
2159 hw->adapter_stopped = FALSE;
2160 ixgbe_stop_adapter(hw);
2161 /* Turn off the laser */
2162 if (hw->phy.multispeed_fiber)
2163 ixgbe_disable_tx_laser(hw);
2164
2165 /* reprogram the RAR[0] in case user changed it. */
2166 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2167
2168 return;
2169}
2170
2171
2172/*********************************************************************
2173 *
2174 * Determine hardware revision.
2175 *
2176 **********************************************************************/
2177static void
2178ixgbe_identify_hardware(struct adapter *adapter)
2179{
2180 device_t dev = adapter->dev;
2181 struct ixgbe_hw *hw = &adapter->hw;
2182
2183 /* Save off the information about this board */
2184 hw->vendor_id = pci_get_vendor(dev);
2185 hw->device_id = pci_get_device(dev);
2186 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2187 hw->subsystem_vendor_id =
2188 pci_read_config(dev, PCIR_SUBVEND_0, 2);
2189 hw->subsystem_device_id =
2190 pci_read_config(dev, PCIR_SUBDEV_0, 2);
2191
2192 /* We need this here to set the num_segs below */
2193 ixgbe_set_mac_type(hw);
2194
2195 /* Pick up the 82599 and VF settings */
2196 if (hw->mac.type != ixgbe_mac_82598EB) {
2197 hw->phy.smart_speed = ixgbe_smart_speed;
2198 adapter->num_segs = IXGBE_82599_SCATTER;
2199 } else
2200 adapter->num_segs = IXGBE_82598_SCATTER;
2201
2202 return;
2203}
2204
2205/*********************************************************************
2206 *
2207 * Determine optic type
2208 *
2209 **********************************************************************/
2210static void
2211ixgbe_setup_optics(struct adapter *adapter)
2212{
2213 struct ixgbe_hw *hw = &adapter->hw;
2214 int layer;
2215
2216 layer = ixgbe_get_supported_physical_layer(hw);
2217
2218 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2219 adapter->optics = IFM_10G_T;
2220 return;
2221 }
2222
2223 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2224 adapter->optics = IFM_1000_T;
2225 return;
2226 }
2227
2228 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2229 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2230 adapter->optics = IFM_10G_LR;
2231 return;
2232 }
2233
2234 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2235 adapter->optics = IFM_10G_SR;
2236 return;
2237 }
2238
2239 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2240 adapter->optics = IFM_10G_TWINAX;
2241 return;
2242 }
2243
2244 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2245 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2246 adapter->optics = IFM_10G_CX4;
2247 return;
2248 }
2249
2250 /* If we get here just set the default */
2251 adapter->optics = IFM_ETHER | IFM_AUTO;
2252 return;
2253}
2254
2255/*********************************************************************
2256 *
2257 * Setup the Legacy or MSI Interrupt handler
2258 *
2259 **********************************************************************/
2260static int
2261ixgbe_allocate_legacy(struct adapter *adapter)
2262{
2263 device_t dev = adapter->dev;
2264 struct ix_queue *que = adapter->queues;
2265 int error, rid = 0;
7e665ea1 2266 unsigned int intr_flags;
9407f759
FT
2267
2268 /* MSI RID at 1 */
2269 if (adapter->msix == 1)
2270 rid = 1;
2271
7e665ea1
FT
2272 /* Try allocating a MSI interrupt first */
2273 adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2274 &rid, &intr_flags);
2275
9407f759
FT
2276 /* We allocate a single interrupt resource */
2277 adapter->res = bus_alloc_resource_any(dev,
7e665ea1 2278 SYS_RES_IRQ, &rid, intr_flags);
9407f759
FT
2279 if (adapter->res == NULL) {
2280 device_printf(dev, "Unable to allocate bus resource: "
2281 "interrupt\n");
2282 return (ENXIO);
2283 }
2284
2285 /*
2286 * Try allocating a fast interrupt and the associated deferred
2287 * processing contexts.
2288 */
2289 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2290 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2291 taskqueue_thread_enqueue, &que->tq);
2292 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2293 device_get_nameunit(adapter->dev));
2294
2295 /* Tasklets for Link, SFP and Multispeed Fiber */
2296 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2297 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2298 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2299#ifdef IXGBE_FDIR
2300 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2301#endif
2302 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2303 taskqueue_thread_enqueue, &adapter->tq);
2304 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2305 device_get_nameunit(adapter->dev));
2306
2307 if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2308 ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2309 device_printf(dev, "Failed to register fast interrupt "
2310 "handler: %d\n", error);
2311 taskqueue_free(que->tq);
2312 taskqueue_free(adapter->tq);
2313 que->tq = NULL;
2314 adapter->tq = NULL;
2315 return (error);
2316 }
2317 /* For simplicity in the handlers */
2318 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2319
2320 return (0);
2321}
2322
2323
2324/*********************************************************************
2325 *
2326 * Setup MSIX Interrupt resources and handlers
2327 *
2328 **********************************************************************/
2329static int
2330ixgbe_allocate_msix(struct adapter *adapter)
2331{
2332 device_t dev = adapter->dev;
2333 struct ix_queue *que = adapter->queues;
2334 int error, rid, vector = 0;
2335
2336 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2337 rid = vector + 1;
2338 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2339 RF_SHAREABLE | RF_ACTIVE);
2340 if (que->res == NULL) {
2341 device_printf(dev,"Unable to allocate"
2342 " bus resource: que interrupt [%d]\n", vector);
2343 return (ENXIO);
2344 }
2345 /* Set the handler function */
2346 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2347 ixgbe_msix_que, que, &que->tag, &que->serializer);
2348 if (error) {
2349 que->res = NULL;
2350 device_printf(dev, "Failed to register QUE handler");
2351 return (error);
2352 }
2353#if 0 /* __FreeBSD_version >= 800504 */
2354 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2355#endif
2356 que->msix = vector;
2357 adapter->que_mask |= (u64)(1 << que->msix);
2358 /*
2359 ** Bind the msix vector, and thus the
2360 ** ring to the corresponding cpu.
2361 */
2362#if 0 /* XXX */
2363 if (adapter->num_queues > 1)
2364 bus_bind_intr(dev, que->res, i);
2365#endif
2366
2367 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2368 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2369 taskqueue_thread_enqueue, &que->tq);
2370 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2371 device_get_nameunit(adapter->dev));
2372 }
2373
2374 /* and Link */
2375 rid = vector + 1;
2376 adapter->res = bus_alloc_resource_any(dev,
2377 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2378 if (!adapter->res) {
2379 device_printf(dev,"Unable to allocate"
2380 " bus resource: Link interrupt [%d]\n", rid);
2381 return (ENXIO);
2382 }
2383 /* Set the link handler function */
2384 error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2385 ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2386 if (error) {
2387 adapter->res = NULL;
2388 device_printf(dev, "Failed to register LINK handler");
2389 return (error);
2390 }
2391#if 0 /* __FreeBSD_version >= 800504 */
2392 bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2393#endif
2394 adapter->linkvec = vector;
2395 /* Tasklets for Link, SFP and Multispeed Fiber */
2396 TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2397 TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2398 TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2399#ifdef IXGBE_FDIR
2400 TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2401#endif
2402 adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2403 taskqueue_thread_enqueue, &adapter->tq);
2404 taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2405 device_get_nameunit(adapter->dev));
2406
2407 return (0);
2408}
2409
2410#if 0 /* HAVE_MSIX */
2411/*
2412 * Setup Either MSI/X or MSI
2413 */
2414static int
2415ixgbe_setup_msix(struct adapter *adapter)
2416{
2417 device_t dev = adapter->dev;
2418 int rid, want, queues, msgs;
2419
2420 /* Override by tuneable */
2421 if (ixgbe_enable_msix == 0)
2422 goto msi;
2423
2424 /* First try MSI/X */
2425 rid = PCIR_BAR(MSIX_82598_BAR);
2426 adapter->msix_mem = bus_alloc_resource_any(dev,
2427 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2428 if (!adapter->msix_mem) {
2429 rid += 4; /* 82599 maps in higher BAR */
2430 adapter->msix_mem = bus_alloc_resource_any(dev,
2431 SYS_RES_MEMORY, &rid, RF_ACTIVE);
2432 }
2433 if (!adapter->msix_mem) {
2434 /* May not be enabled */
2435 device_printf(adapter->dev,
2436 "Unable to map MSIX table \n");
2437 goto msi;
2438 }
2439
2440 msgs = pci_msix_count(dev);
2441 if (msgs == 0) { /* system has msix disabled */
2442 bus_release_resource(dev, SYS_RES_MEMORY,
2443 rid, adapter->msix_mem);
2444 adapter->msix_mem = NULL;
2445 goto msi;
2446 }
2447
2448 /* Figure out a reasonable auto config value */
2449 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2450
2451 if (ixgbe_num_queues != 0)
2452 queues = ixgbe_num_queues;
2453 /* Set max queues to 8 when autoconfiguring */
2454 else if ((ixgbe_num_queues == 0) && (queues > 8))
2455 queues = 8;
2456
2457 /*
2458 ** Want one vector (RX/TX pair) per queue
2459 ** plus an additional for Link.
2460 */
2461 want = queues + 1;
2462 if (msgs >= want)
2463 msgs = want;
2464 else {
2465 device_printf(adapter->dev,
2466 "MSIX Configuration Problem, "
2467 "%d vectors but %d queues wanted!\n",
2468 msgs, want);
2469 return (0); /* Will go to Legacy setup */
2470 }
2471 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2472 device_printf(adapter->dev,
2473 "Using MSIX interrupts with %d vectors\n", msgs);
2474 adapter->num_queues = queues;
2475 return (msgs);
2476 }
2477msi:
2478 msgs = pci_msi_count(dev);
2479 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2480 device_printf(adapter->dev,"Using an MSI interrupt\n");
2481 else
2482 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2483 return (msgs);
2484}
2485#endif
2486
2487
2488static int
2489ixgbe_allocate_pci_resources(struct adapter *adapter)
2490{
2491 int rid;
2492 device_t dev = adapter->dev;
2493
2494 rid = PCIR_BAR(0);
2495 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2496 &rid, RF_ACTIVE);
2497
2498 if (!(adapter->pci_mem)) {
2499 device_printf(dev,"Unable to allocate bus resource: memory\n");
2500 return (ENXIO);
2501 }
2502
2503 adapter->osdep.mem_bus_space_tag =
2504 rman_get_bustag(adapter->pci_mem);
2505 adapter->osdep.mem_bus_space_handle =
2506 rman_get_bushandle(adapter->pci_mem);
2507 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2508
2509 /* Legacy defaults */
2510 adapter->num_queues = 1;
2511 adapter->hw.back = &adapter->osdep;
2512
2513 /*
2514 ** Now setup MSI or MSI/X, should
2515 ** return us the number of supported
2516 ** vectors. (Will be 1 for MSI)
2517 */
2518#if 0 /* HAVE_MSIX */
2519 adapter->msix = ixgbe_setup_msix(adapter);
2520#endif
2521 return (0);
2522}
2523
2524static void
2525ixgbe_free_pci_resources(struct adapter * adapter)
2526{
2527 struct ix_queue *que = adapter->queues;
2528 device_t dev = adapter->dev;
2529 int rid, memrid;
2530
2531 if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2532 memrid = PCIR_BAR(MSIX_82598_BAR);
2533 else
2534 memrid = PCIR_BAR(MSIX_82599_BAR);
2535
2536 /*
2537 ** There is a slight possibility of a failure mode
2538 ** in attach that will result in entering this function
2539 ** before interrupt resources have been initialized, and
2540 ** in that case we do not want to execute the loops below
2541 ** We can detect this reliably by the state of the adapter
2542 ** res pointer.
2543 */
2544 if (adapter->res == NULL)
2545 goto mem;
2546
2547 /*
2548 ** Release all msix queue resources:
2549 */
2550 for (int i = 0; i < adapter->num_queues; i++, que++) {
2551 rid = que->msix + 1;
2552 if (que->tag != NULL) {
2553 bus_teardown_intr(dev, que->res, que->tag);
2554 que->tag = NULL;
2555 }
2556 if (que->res != NULL)
2557 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2558 }
2559
2560
2561 /* Clean the Legacy or Link interrupt last */
2562 if (adapter->linkvec) /* we are doing MSIX */
2563 rid = adapter->linkvec + 1;
2564 else
2565 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2566
2567 if (adapter->tag != NULL) {
2568 bus_teardown_intr(dev, adapter->res, adapter->tag);
2569 adapter->tag = NULL;
2570 }
2571 if (adapter->res != NULL)
2572 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
7e665ea1
FT
2573 if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2574 pci_release_msi(adapter->dev);
9407f759
FT
2575
2576mem:
2577 if (adapter->msix)
2578 pci_release_msi(dev);
2579
2580 if (adapter->msix_mem != NULL)
2581 bus_release_resource(dev, SYS_RES_MEMORY,
2582 memrid, adapter->msix_mem);
2583
2584 if (adapter->pci_mem != NULL)
2585 bus_release_resource(dev, SYS_RES_MEMORY,
2586 PCIR_BAR(0), adapter->pci_mem);
2587
2588 return;
2589}
2590
2591/*********************************************************************
2592 *
2593 * Setup networking device structure and register an interface.
2594 *
2595 **********************************************************************/
2596static int
2597ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2598{
2599 struct ixgbe_hw *hw = &adapter->hw;
2600 struct ifnet *ifp;
2601
2602 INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2603
2604 ifp = adapter->ifp = if_alloc(IFT_ETHER);
2605 if (ifp == NULL) {
2606 device_printf(dev, "can not allocate ifnet structure\n");
2607 return (-1);
2608 }
2609 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2610 ifp->if_baudrate = 1000000000;
2611 ifp->if_init = ixgbe_init;
2612 ifp->if_softc = adapter;
2613 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2614 ifp->if_ioctl = ixgbe_ioctl;
2615 ifp->if_start = ixgbe_start;
2616#if 0 /* __FreeBSD_version >= 800000 */
2617 ifp->if_transmit = ixgbe_mq_start;
2618 ifp->if_qflush = ixgbe_qflush;
2619#endif
2620 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2621
2622 ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2623
2624 adapter->max_frame_size =
2625 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2626
2627 /*
2628 * Tell the upper layer(s) we support long frames.
2629 */
2630 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2631
2632#if 0 /* NET_TSO */
2633 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2634#endif
2635 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2636 ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2637 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2638#if 0 /* NET_TSO */
2639 | IFCAP_VLAN_HWTSO
2640#endif
2641 | IFCAP_VLAN_MTU;
2642 ifp->if_capenable = ifp->if_capabilities;
2643
2644 /* Don't enable LRO by default */
2645#if 0 /* NET_LRO */
2646 ifp->if_capabilities |= IFCAP_LRO;
2647#endif
2648
2649 /*
2650 ** Don't turn this on by default, if vlans are
2651 ** created on another pseudo device (eg. lagg)
2652 ** then vlan events are not passed thru, breaking
2653 ** operation, but with HW FILTER off it works. If
2654 ** using vlans directly on the ixgbe driver you can
2655 ** enable this and get full hardware tag filtering.
2656 */
2657 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2658
2659 /*
2660 * Specify the media types supported by this adapter and register
2661 * callbacks to update media and link information
2662 */
2663 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2664 ixgbe_media_status);
2665 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2666 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2667 if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2668 ifmedia_add(&adapter->media,
2669 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2670 ifmedia_add(&adapter->media,
2671 IFM_ETHER | IFM_1000_T, 0, NULL);
2672 }
2673 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2674 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2675
2676 return (0);
2677}
2678
2679static void
2680ixgbe_config_link(struct adapter *adapter)
2681{
2682 struct ixgbe_hw *hw = &adapter->hw;
2683 u32 autoneg, err = 0;
2684 bool sfp, negotiate;
2685
2686 sfp = ixgbe_is_sfp(hw);
2687
2688 if (sfp) {
2689 if (hw->phy.multispeed_fiber) {
2690 hw->mac.ops.setup_sfp(hw);
2691 ixgbe_enable_tx_laser(hw);
2692 taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2693 } else
2694 taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2695 } else {
2696 if (hw->mac.ops.check_link)
2697 err = ixgbe_check_link(hw, &autoneg,
2698 &adapter->link_up, FALSE);
2699 if (err)
2700 goto out;
2701 autoneg = hw->phy.autoneg_advertised;
2702 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2703 err = hw->mac.ops.get_link_capabilities(hw,
2704 &autoneg, &negotiate);
2705 if (err)
2706 goto out;
2707 if (hw->mac.ops.setup_link)
2708 err = hw->mac.ops.setup_link(hw, autoneg,
2709 negotiate, adapter->link_up);
2710 }
2711out:
2712 return;
2713}
2714
2715/********************************************************************
2716 * Manage DMA'able memory.
2717 *******************************************************************/
2718static void
2719ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2720{
2721 if (error)
2722 return;
2723 *(bus_addr_t *) arg = segs->ds_addr;
2724 return;
2725}
2726
2727static int
2728ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2729 struct ixgbe_dma_alloc *dma, int mapflags)
2730{
2731 device_t dev = adapter->dev;
2732 int r;
2733
2734 r = bus_dma_tag_create(NULL, /* parent */
2735 DBA_ALIGN, 0, /* alignment, bounds */
2736 BUS_SPACE_MAXADDR, /* lowaddr */
2737 BUS_SPACE_MAXADDR, /* highaddr */
2738 NULL, NULL, /* filter, filterarg */
2739 size, /* maxsize */
2740 1, /* nsegments */
2741 size, /* maxsegsize */
2742 BUS_DMA_ALLOCNOW, /* flags */
2743 &dma->dma_tag);
2744 if (r != 0) {
2745 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2746 "error %u\n", r);
2747 goto fail_0;
2748 }
2749 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2750 BUS_DMA_NOWAIT, &dma->dma_map);
2751 if (r != 0) {
2752 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2753 "error %u\n", r);
2754 goto fail_1;
2755 }
2756 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2757 size,
2758 ixgbe_dmamap_cb,
2759 &dma->dma_paddr,
2760 mapflags | BUS_DMA_NOWAIT);
2761 if (r != 0) {
2762 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2763 "error %u\n", r);
2764 goto fail_2;
2765 }
2766 dma->dma_size = size;
2767 return (0);
2768fail_2:
2769 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2770fail_1:
2771 bus_dma_tag_destroy(dma->dma_tag);
2772fail_0:
2773 dma->dma_map = NULL;
2774 dma->dma_tag = NULL;
2775 return (r);
2776}
2777
2778static void
2779ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2780{
2781 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2782 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2783 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2784 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2785 bus_dma_tag_destroy(dma->dma_tag);
2786}
2787
2788
2789/*********************************************************************
2790 *
2791 * Allocate memory for the transmit and receive rings, and then
2792 * the descriptors associated with each, called only once at attach.
2793 *
2794 **********************************************************************/
2795static int
2796ixgbe_allocate_queues(struct adapter *adapter)
2797{
2798 device_t dev = adapter->dev;
2799 struct ix_queue *que;
2800 struct tx_ring *txr;
2801 struct rx_ring *rxr;
2802 int rsize, tsize, error = IXGBE_SUCCESS;
2803 int txconf = 0, rxconf = 0;
2804
2805 /* First allocate the top level queue structs */
2806 if (!(adapter->queues =
2807 (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2808 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2809 device_printf(dev, "Unable to allocate queue memory\n");
2810 error = ENOMEM;
2811 goto fail;
2812 }
2813
2814 /* First allocate the TX ring struct memory */
2815 if (!(adapter->tx_rings =
2816 (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2817 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2818 device_printf(dev, "Unable to allocate TX ring memory\n");
2819 error = ENOMEM;
2820 goto tx_fail;
2821 }
2822
2823 /* Next allocate the RX */
2824 if (!(adapter->rx_rings =
2825 (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2826 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2827 device_printf(dev, "Unable to allocate RX ring memory\n");
2828 error = ENOMEM;
2829 goto rx_fail;
2830 }
2831
2832 /* For the ring itself */
2833 tsize = roundup2(adapter->num_tx_desc *
2834 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2835
2836 /*
2837 * Now set up the TX queues, txconf is needed to handle the
2838 * possibility that things fail midcourse and we need to
2839 * undo memory gracefully
2840 */
2841 for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2842 /* Set up some basics */
2843 txr = &adapter->tx_rings[i];
2844 txr->adapter = adapter;
2845 txr->me = i;
2846
2847 /* Initialize the TX side lock */
2848 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2849 device_get_nameunit(dev), txr->me);
2850 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2851
2852 if (ixgbe_dma_malloc(adapter, tsize,
2853 &txr->txdma, BUS_DMA_NOWAIT)) {
2854 device_printf(dev,
2855 "Unable to allocate TX Descriptor memory\n");
2856 error = ENOMEM;
2857 goto err_tx_desc;
2858 }
2859 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2860 bzero((void *)txr->tx_base, tsize);
2861
2862 /* Now allocate transmit buffers for the ring */
2863 if (ixgbe_allocate_transmit_buffers(txr)) {
2864 device_printf(dev,
2865 "Critical Failure setting up transmit buffers\n");
2866 error = ENOMEM;
2867 goto err_tx_desc;
2868 }
2869#if 0 /* __FreeBSD_version >= 800000 */
2870 /* Allocate a buf ring */
2871 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2872 M_WAITOK, &txr->tx_mtx);
2873 if (txr->br == NULL) {
2874 device_printf(dev,
2875 "Critical Failure setting up buf ring\n");
2876 error = ENOMEM;
2877 goto err_tx_desc;
2878 }
2879#endif
2880 }
2881
2882 /*
2883 * Next the RX queues...
2884 */
2885 rsize = roundup2(adapter->num_rx_desc *
2886 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2887 for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2888 rxr = &adapter->rx_rings[i];
2889 /* Set up some basics */
2890 rxr->adapter = adapter;
2891 rxr->me = i;
2892
2893 /* Initialize the RX side lock */
2894 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2895 device_get_nameunit(dev), rxr->me);
2896 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2897
2898 if (ixgbe_dma_malloc(adapter, rsize,
2899 &rxr->rxdma, BUS_DMA_NOWAIT)) {
2900 device_printf(dev,
2901 "Unable to allocate RxDescriptor memory\n");
2902 error = ENOMEM;
2903 goto err_rx_desc;
2904 }
2905 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2906 bzero((void *)rxr->rx_base, rsize);
2907
2908 /* Allocate receive buffers for the ring*/
2909 if (ixgbe_allocate_receive_buffers(rxr)) {
2910 device_printf(dev,
2911 "Critical Failure setting up receive buffers\n");
2912 error = ENOMEM;
2913 goto err_rx_desc;
2914 }
2915 }
2916
2917 /*
2918 ** Finally set up the queue holding structs
2919 */
2920 for (int i = 0; i < adapter->num_queues; i++) {
2921 que = &adapter->queues[i];
2922 que->adapter = adapter;
2923 que->txr = &adapter->tx_rings[i];
2924 que->rxr = &adapter->rx_rings[i];
2925 }
2926
2927 return (0);
2928
2929err_rx_desc:
2930 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2931 ixgbe_dma_free(adapter, &rxr->rxdma);
2932err_tx_desc:
2933 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2934 ixgbe_dma_free(adapter, &txr->txdma);
2935 kfree(adapter->rx_rings, M_DEVBUF);
2936rx_fail:
2937 kfree(adapter->tx_rings, M_DEVBUF);
2938tx_fail:
2939 kfree(adapter->queues, M_DEVBUF);
2940fail:
2941 return (error);
2942}
2943
2944/*********************************************************************
2945 *
2946 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2947 * the information needed to transmit a packet on the wire. This is
2948 * called only once at attach, setup is done every reset.
2949 *
2950 **********************************************************************/
2951static int
2952ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2953{
2954 struct adapter *adapter = txr->adapter;
2955 device_t dev = adapter->dev;
2956 struct ixgbe_tx_buf *txbuf;
2957 int error, i;
2958
2959 /*
2960 * Setup DMA descriptor areas.
2961 */
2962 if ((error = bus_dma_tag_create(
2963 NULL, /* parent */
2964 1, 0, /* alignment, bounds */
2965 BUS_SPACE_MAXADDR, /* lowaddr */
2966 BUS_SPACE_MAXADDR, /* highaddr */
2967 NULL, NULL, /* filter, filterarg */
2968 IXGBE_TSO_SIZE, /* maxsize */
2969 adapter->num_segs, /* nsegments */
2970 PAGE_SIZE, /* maxsegsize */
2971 0, /* flags */
2972 &txr->txtag))) {
2973 device_printf(dev,"Unable to allocate TX DMA tag\n");
2974 goto fail;
2975 }
2976
2977 if (!(txr->tx_buffers =
2978 (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2979 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2980 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2981 error = ENOMEM;
2982 goto fail;
2983 }
2984
2985 /* Create the descriptor buffer dma maps */
2986 txbuf = txr->tx_buffers;
2987 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2988 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2989 if (error != 0) {
2990 device_printf(dev, "Unable to create TX DMA map\n");
2991 goto fail;
2992 }
2993 }
2994
2995 return 0;
2996fail:
2997 /* We free all, it handles case where we are in the middle */
2998 ixgbe_free_transmit_structures(adapter);
2999 return (error);
3000}
3001
3002/*********************************************************************
3003 *
3004 * Initialize a transmit ring.
3005 *
3006 **********************************************************************/
3007static void
3008ixgbe_setup_transmit_ring(struct tx_ring *txr)
3009{
3010 struct adapter *adapter = txr->adapter;
3011 struct ixgbe_tx_buf *txbuf;
3012 int i;
3013#ifdef DEV_NETMAP
3014 struct netmap_adapter *na = NA(adapter->ifp);
3015 struct netmap_slot *slot;
3016#endif /* DEV_NETMAP */
3017
3018 /* Clear the old ring contents */
3019 IXGBE_TX_LOCK(txr);
3020#ifdef DEV_NETMAP
3021 /*
3022 * (under lock): if in netmap mode, do some consistency
3023 * checks and set slot to entry 0 of the netmap ring.
3024 */
3025 slot = netmap_reset(na, NR_TX, txr->me, 0);
3026#endif /* DEV_NETMAP */
3027 bzero((void *)txr->tx_base,
3028 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3029 /* Reset indices */
3030 txr->next_avail_desc = 0;
3031 txr->next_to_clean = 0;
3032
3033 /* Free any existing tx buffers. */
3034 txbuf = txr->tx_buffers;
3035 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3036 if (txbuf->m_head != NULL) {
3037 bus_dmamap_sync(txr->txtag, txbuf->map,
3038 BUS_DMASYNC_POSTWRITE);
3039 bus_dmamap_unload(txr->txtag, txbuf->map);
3040 m_freem(txbuf->m_head);
3041 txbuf->m_head = NULL;
3042 }
3043#ifdef DEV_NETMAP
3044 /*
3045 * In netmap mode, set the map for the packet buffer.
3046 * NOTE: Some drivers (not this one) also need to set
3047 * the physical buffer address in the NIC ring.
3048 * Slots in the netmap ring (indexed by "si") are
3049 * kring->nkr_hwofs positions "ahead" wrt the
3050 * corresponding slot in the NIC ring. In some drivers
3051 * (not here) nkr_hwofs can be negative. Function
3052 * netmap_idx_n2k() handles wraparounds properly.
3053 */
3054 if (slot) {
3055 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3056 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3057 }
3058#endif /* DEV_NETMAP */
3059 /* Clear the EOP index */
3060 txbuf->eop_index = -1;
3061 }
3062
3063#ifdef IXGBE_FDIR
3064 /* Set the rate at which we sample packets */
3065 if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3066 txr->atr_sample = atr_sample_rate;
3067#endif
3068
3069 /* Set number of descriptors available */
3070 txr->tx_avail = adapter->num_tx_desc;
3071
3072 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3073 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3074 IXGBE_TX_UNLOCK(txr);
3075}
3076
3077/*********************************************************************
3078 *
3079 * Initialize all transmit rings.
3080 *
3081 **********************************************************************/
3082static int
3083ixgbe_setup_transmit_structures(struct adapter *adapter)
3084{
3085 struct tx_ring *txr = adapter->tx_rings;
3086
3087 for (int i = 0; i < adapter->num_queues; i++, txr++)
3088 ixgbe_setup_transmit_ring(txr);
3089
3090 return (0);
3091}
3092
3093/*********************************************************************
3094 *
3095 * Enable transmit unit.
3096 *
3097 **********************************************************************/
3098static void
3099ixgbe_initialize_transmit_units(struct adapter *adapter)
3100{
3101 struct tx_ring *txr = adapter->tx_rings;
3102 struct ixgbe_hw *hw = &adapter->hw;
3103
3104 /* Setup the Base and Length of the Tx Descriptor Ring */
3105
3106 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3107 u64 tdba = txr->txdma.dma_paddr;
3108 u32 txctrl;
3109
3110 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3111 (tdba & 0x00000000ffffffffULL));
3112 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3113 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3114 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3115
3116 /* Setup the HW Tx Head and Tail descriptor pointers */
3117 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3118 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3119
3120 /* Setup Transmit Descriptor Cmd Settings */
3121 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3122 txr->queue_status = IXGBE_QUEUE_IDLE;
3123
3124 /* Disable Head Writeback */
3125 switch (hw->mac.type) {
3126 case ixgbe_mac_82598EB:
3127 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3128 break;
3129 case ixgbe_mac_82599EB:
3130 case ixgbe_mac_X540:
3131 default:
3132 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3133 break;
3134 }
ce451236 3135 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
9407f759
FT
3136 switch (hw->mac.type) {
3137 case ixgbe_mac_82598EB:
3138 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3139 break;
3140 case ixgbe_mac_82599EB:
3141 case ixgbe_mac_X540:
3142 default:
3143 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3144 break;
3145 }
3146
3147 }
3148
3149 if (hw->mac.type != ixgbe_mac_82598EB) {
3150 u32 dmatxctl, rttdcs;
3151 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3152 dmatxctl |= IXGBE_DMATXCTL_TE;
3153 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3154 /* Disable arbiter to set MTQC */
3155 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3156 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3157 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3158 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3159 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3160 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3161 }
3162
3163 return;
3164}
3165
3166/*********************************************************************
3167 *
3168 * Free all transmit rings.
3169 *
3170 **********************************************************************/
3171static void
3172ixgbe_free_transmit_structures(struct adapter *adapter)
3173{
3174 struct tx_ring *txr = adapter->tx_rings;
3175
3176 for (int i = 0; i < adapter->num_queues; i++, txr++) {
3177 IXGBE_TX_LOCK(txr);
3178 ixgbe_free_transmit_buffers(txr);
3179 ixgbe_dma_free(adapter, &txr->txdma);
3180 IXGBE_TX_UNLOCK(txr);
3181 IXGBE_TX_LOCK_DESTROY(txr);
3182 }
3183 kfree(adapter->tx_rings, M_DEVBUF);
3184}
3185
3186/*********************************************************************
3187 *
3188 * Free transmit ring related data structures.
3189 *
3190 **********************************************************************/
3191static void
3192ixgbe_free_transmit_buffers(struct tx_ring *txr)
3193{
3194 struct adapter *adapter = txr->adapter;
3195 struct ixgbe_tx_buf *tx_buffer;
3196 int i;
3197
3198 INIT_DEBUGOUT("free_transmit_ring: begin");
3199
3200 if (txr->tx_buffers == NULL)
3201 return;
3202
3203 tx_buffer = txr->tx_buffers;
3204 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3205 if (tx_buffer->m_head != NULL) {
3206 bus_dmamap_sync(txr->txtag, tx_buffer->map,
3207 BUS_DMASYNC_POSTWRITE);
3208 bus_dmamap_unload(txr->txtag,
3209 tx_buffer->map);
3210 m_freem(tx_buffer->m_head);
3211 tx_buffer->m_head = NULL;
3212 if (tx_buffer->map != NULL) {
3213 bus_dmamap_destroy(txr->txtag,
3214 tx_buffer->map);
3215 tx_buffer->map = NULL;
3216 }
3217 } else if (tx_buffer->map != NULL) {
3218 bus_dmamap_unload(txr->txtag,
3219 tx_buffer->map);
3220 bus_dmamap_destroy(txr->txtag,
3221 tx_buffer->map);
3222 tx_buffer->map = NULL;
3223 }
3224 }
3225#if 0 /* __FreeBSD_version >= 800000 */
3226 if (txr->br != NULL)
3227 buf_ring_free(txr->br, M_DEVBUF);
3228#endif
3229 if (txr->tx_buffers != NULL) {
3230 kfree(txr->tx_buffers, M_DEVBUF);
3231 txr->tx_buffers = NULL;
3232 }
3233 if (txr->txtag != NULL) {
3234 bus_dma_tag_destroy(txr->txtag);
3235 txr->txtag = NULL;
3236 }
3237 return;
3238}
3239
3240/*********************************************************************
3241 *
3242 * Advanced Context Descriptor setup for VLAN or CSUM
3243 *
3244 **********************************************************************/
3245
3246static bool
3247ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3248{
3249 struct adapter *adapter = txr->adapter;
3250 struct ixgbe_adv_tx_context_desc *TXD;
3251 struct ixgbe_tx_buf *tx_buffer;
3252 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3253 struct ether_vlan_header *eh;
3254 struct ip *ip;
3255 struct ip6_hdr *ip6;
3256 int ehdrlen, ip_hlen = 0;
3257 u16 etype;
3258 u8 ipproto = 0;
3259 bool offload = TRUE;
3260 int ctxd = txr->next_avail_desc;
3261#ifdef NET_VLAN
3262 u16 vtag = 0;
3263#endif
3264
3265
3266 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3267 offload = FALSE;
3268
3269 tx_buffer = &txr->tx_buffers[ctxd];
3270 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3271
3272 /*
3273 ** In advanced descriptors the vlan tag must
3274 ** be placed into the descriptor itself.
3275 */
3276#ifdef NET_VLAN
3277 if (mp->m_flags & M_VLANTAG) {
3278 vtag = htole16(mp->m_pkthdr.ether_vtag);
3279 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3280 } else if (offload == FALSE)
3281 return FALSE;
3282#endif
3283
3284 /*
3285 * Determine where frame payload starts.
3286 * Jump over vlan headers if already present,
3287 * helpful for QinQ too.
3288 */
3289 eh = mtod(mp, struct ether_vlan_header *);
3290 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3291 etype = ntohs(eh->evl_proto);
3292 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3293 } else {
3294 etype = ntohs(eh->evl_encap_proto);
3295 ehdrlen = ETHER_HDR_LEN;
3296 }
3297
3298 /* Set the ether header length */
3299 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3300
3301 switch (etype) {
3302 case ETHERTYPE_IP:
3303 ip = (struct ip *)(mp->m_data + ehdrlen);
3304 ip_hlen = ip->ip_hl << 2;
3305 ipproto = ip->ip_p;
3306 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3307 break;
3308 case ETHERTYPE_IPV6:
3309 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3310 ip_hlen = sizeof(struct ip6_hdr);
3311 /* XXX-BZ this will go badly in case of ext hdrs. */
3312 ipproto = ip6->ip6_nxt;
3313 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3314 break;
3315 default:
3316 offload = FALSE;
3317 break;
3318 }
3319
3320 vlan_macip_lens |= ip_hlen;
3321 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3322
3323 switch (ipproto) {
3324 case IPPROTO_TCP:
3325 if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3326 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3327 break;
3328
3329 case IPPROTO_UDP:
3330 if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3331 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3332 break;
3333
3334#if 0
3335 case IPPROTO_SCTP:
3336 if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3337 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3338 break;
3339#endif
3340 default:
3341 offload = FALSE;
3342 break;
3343 }
3344
3345 /* Now copy bits into descriptor */
3346 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3347 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3348 TXD->seqnum_seed = htole32(0);
3349 TXD->mss_l4len_idx = htole32(0);
3350
3351 tx_buffer->m_head = NULL;
3352 tx_buffer->eop_index = -1;
3353
3354 /* We've consumed the first desc, adjust counters */
3355 if (++ctxd == adapter->num_tx_desc)
3356 ctxd = 0;
3357 txr->next_avail_desc = ctxd;
3358 --txr->tx_avail;
3359
3360 return (offload);
3361}
3362
3363/**********************************************************************
3364 *
3365 * Setup work for hardware segmentation offload (TSO) on
3366 * adapters using advanced tx descriptors
3367 *
3368 **********************************************************************/
3369#if 0 /* NET_TSO */
3370static bool
3371ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3372 u32 *olinfo_status)
3373{
3374 struct adapter *adapter = txr->adapter;
3375 struct ixgbe_adv_tx_context_desc *TXD;
3376 struct ixgbe_tx_buf *tx_buffer;
3377#ifdef NET_VLAN
3378 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3379 u16 vtag = 0, eh_type;
3380#else
3381 u16 eh_type;
3382 u32 type_tucmd_mlhl = 0;
3383#endif
3384 u32 mss_l4len_idx = 0, len;
3385 int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3386 struct ether_vlan_header *eh;
3387#ifdef INET6
3388 struct ip6_hdr *ip6;
3389#endif
3390#ifdef INET
3391 struct ip *ip;
3392#endif
3393 struct tcphdr *th;
3394
3395
3396 /*
3397 * Determine where frame payload starts.
3398 * Jump over vlan headers if already present
3399 */
3400 eh = mtod(mp, struct ether_vlan_header *);
3401 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3402 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3403 eh_type = eh->evl_proto;
3404 } else {
3405 ehdrlen = ETHER_HDR_LEN;
3406 eh_type = eh->evl_encap_proto;
3407 }
3408
3409 /* Ensure we have at least the IP+TCP header in the first mbuf. */
3410 len = ehdrlen + sizeof(struct tcphdr);
3411 switch (ntohs(eh_type)) {
3412#ifdef INET6
3413 case ETHERTYPE_IPV6:
3414 if (mp->m_len < len + sizeof(struct ip6_hdr))
3415 return FALSE;
3416 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3417 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3418 if (ip6->ip6_nxt != IPPROTO_TCP)
3419 return FALSE;
3420 ip_hlen = sizeof(struct ip6_hdr);
3421 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3422 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3423 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3424 break;
3425#endif
3426#ifdef INET
3427 case ETHERTYPE_IP:
3428 if (mp->m_len < len + sizeof(struct ip))
3429 return FALSE;
3430 ip = (struct ip *)(mp->m_data + ehdrlen);
3431 if (ip->ip_p != IPPROTO_TCP)
3432 return FALSE;
3433 ip->ip_sum = 0;
3434 ip_hlen = ip->ip_hl << 2;
3435 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3436 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3437 ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3438 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3439 /* Tell transmit desc to also do IPv4 checksum. */
3440 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3441 break;
3442#endif
3443 default:
3444 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3445 __func__, ntohs(eh_type));
3446 break;
3447 }
3448
3449 ctxd = txr->next_avail_desc;
3450 tx_buffer = &txr->tx_buffers[ctxd];
3451 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3452
3453 tcp_hlen = th->th_off << 2;
3454
3455 /* This is used in the transmit desc in encap */
3456 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3457
3458 /* VLAN MACLEN IPLEN */
3459#ifdef NET_VLAN
3460 if (mp->m_flags & M_VLANTAG) {
3461 vtag = htole16(mp->m_pkthdr.ether_vtag);
3462 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3463 }
3464
3465 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3466 vlan_macip_lens |= ip_hlen;
3467 TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3468#endif
3469
3470 /* ADV DTYPE TUCMD */
3471 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3472 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3473 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3474
3475 /* MSS L4LEN IDX */
3476 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3477 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3478 TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3479
3480 TXD->seqnum_seed = htole32(0);
3481 tx_buffer->m_head = NULL;
3482 tx_buffer->eop_index = -1;
3483
3484 if (++ctxd == adapter->num_tx_desc)
3485 ctxd = 0;
3486
3487 txr->tx_avail--;
3488 txr->next_avail_desc = ctxd;
3489 return TRUE;
3490}
3491#endif
3492
3493#ifdef IXGBE_FDIR
3494/*
3495** This routine parses packet headers so that Flow
3496** Director can make a hashed filter table entry
3497** allowing traffic flows to be identified and kept
3498** on the same cpu. This would be a performance
3499** hit, but we only do it at IXGBE_FDIR_RATE of
3500** packets.
3501*/
3502static void
3503ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3504{
3505 struct adapter *adapter = txr->adapter;
3506 struct ix_queue *que;
3507 struct ip *ip;
3508 struct tcphdr *th;
3509 struct udphdr *uh;
3510 struct ether_vlan_header *eh;
3511 union ixgbe_atr_hash_dword input = {.dword = 0};
3512 union ixgbe_atr_hash_dword common = {.dword = 0};
3513 int ehdrlen, ip_hlen;
3514 u16 etype;
3515
3516 eh = mtod(mp, struct ether_vlan_header *);
3517 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3518 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3519 etype = eh->evl_proto;
3520 } else {
3521 ehdrlen = ETHER_HDR_LEN;
3522 etype = eh->evl_encap_proto;
3523 }
3524
3525 /* Only handling IPv4 */
3526 if (etype != htons(ETHERTYPE_IP))
3527 return;
3528
3529 ip = (struct ip *)(mp->m_data + ehdrlen);
3530 ip_hlen = ip->ip_hl << 2;
3531
3532 /* check if we're UDP or TCP */
3533 switch (ip->ip_p) {
3534 case IPPROTO_TCP:
3535 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3536 /* src and dst are inverted */
3537 common.port.dst ^= th->th_sport;
3538 common.port.src ^= th->th_dport;
3539 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3540 break;
3541 case IPPROTO_UDP:
3542 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3543 /* src and dst are inverted */
3544 common.port.dst ^= uh->uh_sport;
3545 common.port.src ^= uh->uh_dport;
3546 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3547 break;
3548 default:
3549 return;
3550 }
3551
3552 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3553 if (mp->m_pkthdr.ether_vtag)
3554 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3555 else
3556 common.flex_bytes ^= etype;
3557 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3558
3559 que = &adapter->queues[txr->me];
3560 /*
3561 ** This assumes the Rx queue and Tx
3562 ** queue are bound to the same CPU
3563 */
3564 ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3565 input, common, que->msix);
3566}
3567#endif /* IXGBE_FDIR */
3568
3569/**********************************************************************
3570 *
3571 * Examine each tx_buffer in the used queue. If the hardware is done
3572 * processing the packet then free associated resources. The
3573 * tx_buffer is put back on the free queue.
3574 *
3575 **********************************************************************/
3576static bool
3577ixgbe_txeof(struct tx_ring *txr)
3578{
3579 struct adapter *adapter = txr->adapter;
3580 struct ifnet *ifp = adapter->ifp;
3581 u32 first, last, done, processed;
3582 struct ixgbe_tx_buf *tx_buffer;
3583 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3584
3585 KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3586
3587#ifdef DEV_NETMAP
3588 if (ifp->if_capenable & IFCAP_NETMAP) {
3589 struct netmap_adapter *na = NA(ifp);
3590 struct netmap_kring *kring = &na->tx_rings[txr->me];
3591
3592 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3593
3594 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3595 BUS_DMASYNC_POSTREAD);
3596 /*
3597 * In netmap mode, all the work is done in the context
3598 * of the client thread. Interrupt handlers only wake up
3599 * clients, which may be sleeping on individual rings
3600 * or on a global resource for all rings.
3601 * To implement tx interrupt mitigation, we wake up the client
3602 * thread roughly every half ring, even if the NIC interrupts
3603 * more frequently. This is implemented as follows:
3604 * - ixgbe_txsync() sets kring->nr_kflags with the index of
3605 * the slot that should wake up the thread (nkr_num_slots
3606 * means the user thread should not be woken up);
3607 * - the driver ignores tx interrupts unless netmap_mitigate=0
3608 * or the slot has the DD bit set.
3609 *
3610 * When the driver has separate locks, we need to
3611 * release and re-acquire txlock to avoid deadlocks.
3612 * XXX see if we can find a better way.
3613 */
3614 if (!netmap_mitigate ||
3615 (kring->nr_kflags < kring->nkr_num_slots &&
3616 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3617 kring->nr_kflags = kring->nkr_num_slots;
3618 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3619 IXGBE_TX_UNLOCK(txr);
3620 IXGBE_CORE_LOCK(adapter);
3621 selwakeuppri(&na->tx_si, PI_NET);
3622 IXGBE_CORE_UNLOCK(adapter);
3623 IXGBE_TX_LOCK(txr);
3624 }
3625 return FALSE;
3626 }
3627#endif /* DEV_NETMAP */
3628
3629 if (txr->tx_avail == adapter->num_tx_desc) {
3630 txr->queue_status = IXGBE_QUEUE_IDLE;
3631 return FALSE;
3632 }
3633
3634 processed = 0;
3635 first = txr->next_to_clean;
3636 tx_buffer = &txr->tx_buffers[first];
3637 /* For cleanup we just use legacy struct */
3638 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3639 last = tx_buffer->eop_index;
3640 if (last == -1)
3641 return FALSE;
3642 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3643
3644 /*
3645 ** Get the index of the first descriptor
3646 ** BEYOND the EOP and call that 'done'.
3647 ** I do this so the comparison in the
3648 ** inner while loop below can be simple
3649 */
3650 if (++last == adapter->num_tx_desc) last = 0;
3651 done = last;
3652
3653 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3654 BUS_DMASYNC_POSTREAD);
3655 /*
3656 ** Only the EOP descriptor of a packet now has the DD
3657 ** bit set, this is what we look for...
3658 */
3659 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3660 /* We clean the range of the packet */
3661 while (first != done) {
3662 tx_desc->upper.data = 0;
3663 tx_desc->lower.data = 0;
3664 tx_desc->buffer_addr = 0;
3665 ++txr->tx_avail;
3666 ++processed;
3667
3668 if (tx_buffer->m_head) {
3669 txr->bytes +=
3670 tx_buffer->m_head->m_pkthdr.len;
3671 bus_dmamap_sync(txr->txtag,
3672 tx_buffer->map,
3673 BUS_DMASYNC_POSTWRITE);
3674 bus_dmamap_unload(txr->txtag,
3675 tx_buffer->map);
3676 m_freem(tx_buffer->m_head);
3677 tx_buffer->m_head = NULL;
3678 tx_buffer->map = NULL;
3679 }
3680 tx_buffer->eop_index = -1;
3681 txr->watchdog_time = ticks;
3682
3683 if (++first == adapter->num_tx_desc)
3684 first = 0;
3685
3686 tx_buffer = &txr->tx_buffers[first];
3687 tx_desc =
3688 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3689 }
3690 ++txr->packets;
3691 ++ifp->if_opackets;
3692 /* See if there is more work now */
3693 last = tx_buffer->eop_index;
3694 if (last != -1) {
3695 eop_desc =
3696 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3697 /* Get next done point */
3698 if (++last == adapter->num_tx_desc) last = 0;
3699 done = last;
3700 } else
3701 break;
3702 }
3703 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3704 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3705
3706 txr->next_to_clean = first;
3707
3708 /*
3709 ** Watchdog calculation, we know there's
3710 ** work outstanding or the first return
3711 ** would have been taken, so none processed
3712 ** for too long indicates a hang.
3713 */
3714 if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3715 txr->queue_status = IXGBE_QUEUE_HUNG;
3716
3717 /* With a minimum free clear the depleted state bit. */
3718 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3719 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3720
3721 if (txr->tx_avail == adapter->num_tx_desc) {
3722 txr->queue_status = IXGBE_QUEUE_IDLE;
3723 return (FALSE);
3724 }
3725
3726 return TRUE;
3727}
3728
3729/*********************************************************************
3730 *
3731 * Refresh mbuf buffers for RX descriptor rings
3732 * - now keeps its own state so discards due to resource
3733 * exhaustion are unnecessary, if an mbuf cannot be obtained
3734 * it just returns, keeping its placeholder, thus it can simply
3735 * be recalled to try again.
3736 *
3737 **********************************************************************/
3738static void
3739ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3740{
3741 struct adapter *adapter = rxr->adapter;
3742 bus_dma_segment_t hseg[1];
3743 bus_dma_segment_t pseg[1];
3744 struct ixgbe_rx_buf *rxbuf;
3745 struct mbuf *mh, *mp;
3746 int i, j, nsegs, error;
3747 bool refreshed = FALSE;
3748
3749 i = j = rxr->next_to_refresh;
3750 /* Control the loop with one beyond */
3751 if (++j == adapter->num_rx_desc)
3752 j = 0;
3753
3754 while (j != limit) {
3755 rxbuf = &rxr->rx_buffers[i];
3756 if (rxr->hdr_split == FALSE)
3757 goto no_split;
3758
3759 if (rxbuf->m_head == NULL) {
3760 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3761 if (mh == NULL)
3762 goto update;
3763 } else
3764 mh = rxbuf->m_head;
3765
3766 mh->m_pkthdr.len = mh->m_len = MHLEN;
3767 mh->m_len = MHLEN;
3768 mh->m_flags |= M_PKTHDR;
3769 /* Get the memory mapping */
3770 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3771 rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3772 if (error != 0) {
3773 kprintf("Refresh mbufs: hdr dmamap load"
3774 " failure - %d\n", error);
3775 m_free(mh);
3776 rxbuf->m_head = NULL;
3777 goto update;
3778 }
3779 rxbuf->m_head = mh;
3780 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3781 BUS_DMASYNC_PREREAD);
3782 rxr->rx_base[i].read.hdr_addr =
3783 htole64(hseg[0].ds_addr);
3784
3785no_split:
3786 if (rxbuf->m_pack == NULL) {
3787 mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3788 M_PKTHDR, adapter->rx_mbuf_sz);
3789 if (mp == NULL)
3790 goto update;
3791 } else
3792 mp = rxbuf->m_pack;
3793
3794 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3795 /* Get the memory mapping */
3796 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3797 rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3798 if (error != 0) {
3799 kprintf("Refresh mbufs: payload dmamap load"
3800 " failure - %d\n", error);
3801 m_free(mp);
3802 rxbuf->m_pack = NULL;
3803 goto update;
3804 }
3805 rxbuf->m_pack = mp;
3806 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3807 BUS_DMASYNC_PREREAD);
3808 rxr->rx_base[i].read.pkt_addr =
3809 htole64(pseg[0].ds_addr);
3810
3811 refreshed = TRUE;
3812 /* Next is precalculated */
3813 i = j;
3814 rxr->next_to_refresh = i;
3815 if (++j == adapter->num_rx_desc)
3816 j = 0;
3817 }
3818update:
3819 if (refreshed) /* Update hardware tail index */
3820 IXGBE_WRITE_REG(&adapter->hw,
3821 IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3822 return;
3823}
3824
3825/*********************************************************************
3826 *
3827 * Allocate memory for rx_buffer structures. Since we use one
3828 * rx_buffer per received packet, the maximum number of rx_buffer's
3829 * that we'll need is equal to the number of receive descriptors
3830 * that we've allocated.
3831 *
3832 **********************************************************************/
3833static int
3834ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3835{
3836 struct adapter *adapter = rxr->adapter;
3837 device_t dev = adapter->dev;
3838 struct ixgbe_rx_buf *rxbuf;
3839 int i, bsize, error;
3840
3841 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3842 if (!(rxr->rx_buffers =
3843 (struct ixgbe_rx_buf *) kmalloc(bsize,
3844 M_DEVBUF, M_NOWAIT | M_ZERO))) {
3845 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3846 error = ENOMEM;
3847 goto fail;
3848 }
3849
3850 if ((error = bus_dma_tag_create(NULL, /* parent */
3851 1, 0, /* alignment, bounds */
3852 BUS_SPACE_MAXADDR, /* lowaddr */
3853 BUS_SPACE_MAXADDR, /* highaddr */
3854 NULL, NULL, /* filter, filterarg */
3855 MSIZE, /* maxsize */
3856 1, /* nsegments */
3857 MSIZE, /* maxsegsize */
3858 0, /* flags */
3859 &rxr->htag))) {
3860 device_printf(dev, "Unable to create RX DMA tag\n");
3861 goto fail;
3862 }
3863
3864 if ((error = bus_dma_tag_create(NULL, /* parent */
3865 1, 0, /* alignment, bounds */
3866 BUS_SPACE_MAXADDR, /* lowaddr */
3867 BUS_SPACE_MAXADDR, /* highaddr */
3868 NULL, NULL, /* filter, filterarg */
3869 MJUM16BYTES, /* maxsize */
3870 1, /* nsegments */
3871 MJUM16BYTES, /* maxsegsize */
3872 0, /* flags */
3873 &rxr->ptag))) {
3874 device_printf(dev, "Unable to create RX DMA tag\n");
3875 goto fail;
3876 }
3877
3878 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3879 rxbuf = &rxr->rx_buffers[i];
3880 error = bus_dmamap_create(rxr->htag,
3881 BUS_DMA_NOWAIT, &rxbuf->hmap);
3882 if (error) {
3883 device_printf(dev, "Unable to create RX head map\n");
3884 goto fail;
3885 }
3886 error = bus_dmamap_create(rxr->ptag,
3887 BUS_DMA_NOWAIT, &rxbuf->pmap);
3888 if (error) {
3889 device_printf(dev, "Unable to create RX pkt map\n");
3890 goto fail;
3891 }
3892 }
3893
3894 return (0);
3895
3896fail:
3897 /* Frees all, but can handle partial completion */
3898 ixgbe_free_receive_structures(adapter);
3899 return (error);
3900}
3901
3902/*
3903** Used to detect a descriptor that has
3904** been merged by Hardware RSC.
3905*/
3906static inline u32
3907ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3908{
3909 return (le32toh(rx->wb.lower.lo_dword.data) &
3910 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3911}
3912
3913/*********************************************************************
3914 *
3915 * Initialize Hardware RSC (LRO) feature on 82599
3916 * for an RX ring, this is toggled by the LRO capability
3917 * even though it is transparent to the stack.
3918 *
3919 **********************************************************************/
3920#if 0 /* NET_LRO */
3921static void
3922ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3923{
3924 struct adapter *adapter = rxr->adapter;
3925 struct ixgbe_hw *hw = &adapter->hw;
3926 u32 rscctrl, rdrxctl;
3927
3928 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3929 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3930#ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3931 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3932#endif /* DEV_NETMAP */
3933 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3934 rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3935 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3936
3937 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3938 rscctrl |= IXGBE_RSCCTL_RSCEN;
3939 /*
3940 ** Limit the total number of descriptors that
3941 ** can be combined, so it does not exceed 64K
3942 */
3943 if (adapter->rx_mbuf_sz == MCLBYTES)
3944 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3945 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3946 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3947 else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3948 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3949 else /* Using 16K cluster */
3950 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3951
3952 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3953
3954 /* Enable TCP header recognition */
3955 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3956 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3957 IXGBE_PSRTYPE_TCPHDR));
3958
3959 /* Disable RSC for ACK packets */
3960 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3961 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3962
3963 rxr->hw_rsc = TRUE;
3964}
3965#endif
3966
3967static void
3968ixgbe_free_receive_ring(struct rx_ring *rxr)
3969{
3970 struct adapter *adapter;
3971 struct ixgbe_rx_buf *rxbuf;
3972 int i;
3973
3974 adapter = rxr->adapter;
3975 for (i = 0; i < adapter->num_rx_desc; i++) {
3976 rxbuf = &rxr->rx_buffers[i];
3977 if (rxbuf->m_head != NULL) {
3978 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3979 BUS_DMASYNC_POSTREAD);
3980 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3981 rxbuf->m_head->m_flags |= M_PKTHDR;
3982 m_freem(rxbuf->m_head);
3983 }
3984 if (rxbuf->m_pack != NULL) {
3985 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3986 BUS_DMASYNC_POSTREAD);
3987 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3988 rxbuf->m_pack->m_flags |= M_PKTHDR;
3989 m_freem(rxbuf->m_pack);
3990 }
3991 rxbuf->m_head = NULL;
3992 rxbuf->m_pack = NULL;
3993 }
3994}
3995
3996
3997/*********************************************************************
3998 *
3999 * Initialize a receive ring and its buffers.
4000 *
4001 **********************************************************************/
4002static int
4003ixgbe_setup_receive_ring(struct rx_ring *rxr)
4004{
4005 struct adapter *adapter;
4006 struct ifnet *ifp;
4007 device_t dev;
4008 struct ixgbe_rx_buf *rxbuf;
4009 bus_dma_segment_t pseg[1], hseg[1];
4010#if 0 /* NET_LRO */
4011 struct lro_ctrl *lro = &rxr->lro;
4012#endif
4013 int rsize, nsegs, error = 0;
4014#ifdef DEV_NETMAP
4015 struct netmap_adapter *na = NA(rxr->adapter->ifp);
4016 struct netmap_slot *slot;
4017#endif /* DEV_NETMAP */
4018
4019 adapter = rxr->adapter;
4020 ifp = adapter->ifp;
4021 dev = adapter->dev;
4022
4023 /* Clear the ring contents */
4024 IXGBE_RX_LOCK(rxr);
4025#ifdef DEV_NETMAP
4026 /* same as in ixgbe_setup_transmit_ring() */
4027 slot = netmap_reset(na, NR_RX, rxr->me, 0);
4028#endif /* DEV_NETMAP */
4029 rsize = roundup2(adapter->num_rx_desc *
4030 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4031 bzero((void *)rxr->rx_base, rsize);
4032
4033 /* Free current RX buffer structs and their mbufs */
4034 ixgbe_free_receive_ring(rxr);
4035
4036 /* Configure header split? */
4037 if (ixgbe_header_split)
4038 rxr->hdr_split = TRUE;
4039
4040 /* Now replenish the mbufs */
4041 for (int j = 0; j != adapter->num_rx_desc; ++j) {
4042 struct mbuf *mh, *mp;
4043
4044 rxbuf = &rxr->rx_buffers[j];
4045#ifdef DEV_NETMAP
4046 /*
4047 * In netmap mode, fill the map and set the buffer
4048 * address in the NIC ring, considering the offset
4049 * between the netmap and NIC rings (see comment in
4050 * ixgbe_setup_transmit_ring() ). No need to allocate
4051 * an mbuf, so end the block with a continue;
4052 */
4053 if (slot) {
4054 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4055 uint64_t paddr;
4056 void *addr;
4057
4058 addr = PNMB(slot + sj, &paddr);
4059 netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4060 /* Update descriptor */
4061 rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4062 continue;
4063 }
4064#endif /* DEV_NETMAP */
4065 /*
4066 ** Don't allocate mbufs if not
4067 ** doing header split, its wasteful
4068 */
4069 if (rxr->hdr_split == FALSE)
4070 goto skip_head;
4071
4072 /* First the header */
4073 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4074 if (rxbuf->m_head == NULL) {
4075 error = ENOBUFS;
4076 goto fail;
4077 }
4078 m_adj(rxbuf->m_head, ETHER_ALIGN);
4079 mh = rxbuf->m_head;
4080 mh->m_len = mh->m_pkthdr.len = MHLEN;
4081 mh->m_flags |= M_PKTHDR;
4082 /* Get the memory mapping */
4083 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4084 rxbuf->hmap, rxbuf->m_head, hseg, 1,
4085 &nsegs, BUS_DMA_NOWAIT);
4086
4087 if (error != 0) /* Nothing elegant to do here */
4088 goto fail;
4089 bus_dmamap_sync(rxr->htag,
4090 rxbuf->hmap, BUS_DMASYNC_PREREAD);
4091 /* Update descriptor */
4092 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4093
4094skip_head:
4095 /* Now the payload cluster */
4096 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4097 M_PKTHDR, adapter->rx_mbuf_sz);
4098 if (rxbuf->m_pack == NULL) {
4099 error = ENOBUFS;
4100 goto fail;
4101 }
4102 mp = rxbuf->m_pack;
4103 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4104 /* Get the memory mapping */
4105 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4106 rxbuf->pmap, mp, hseg, 1,
4107 &nsegs, BUS_DMA_NOWAIT);
4108 if (error != 0)
4109 goto fail;
4110 bus_dmamap_sync(rxr->ptag,
4111 rxbuf->pmap, BUS_DMASYNC_PREREAD);
4112 /* Update descriptor */
4113 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4114 }