ixgbe: Only set gso_type to SKB_GSO_TCPV4 as RSC does not support IPv6
[~shefty/rdma-dev.git] / drivers / net / ethernet / intel / ixgbe / ixgbe_main.c
1 /*******************************************************************************
2
3   Intel 10 Gigabit PCI Express Linux driver
4   Copyright(c) 1999 - 2012 Intel Corporation.
5
6   This program is free software; you can redistribute it and/or modify it
7   under the terms and conditions of the GNU General Public License,
8   version 2, as published by the Free Software Foundation.
9
10   This program is distributed in the hope it will be useful, but WITHOUT
11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13   more details.
14
15   You should have received a copy of the GNU General Public License along with
16   this program; if not, write to the Free Software Foundation, Inc.,
17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19   The full GNU General Public License is included in this distribution in
20   the file called "COPYING".
21
22   Contact Information:
23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28 #include <linux/types.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/netdevice.h>
32 #include <linux/vmalloc.h>
33 #include <linux/string.h>
34 #include <linux/in.h>
35 #include <linux/interrupt.h>
36 #include <linux/ip.h>
37 #include <linux/tcp.h>
38 #include <linux/sctp.h>
39 #include <linux/pkt_sched.h>
40 #include <linux/ipv6.h>
41 #include <linux/slab.h>
42 #include <net/checksum.h>
43 #include <net/ip6_checksum.h>
44 #include <linux/ethtool.h>
45 #include <linux/if.h>
46 #include <linux/if_vlan.h>
47 #include <linux/if_bridge.h>
48 #include <linux/prefetch.h>
49 #include <scsi/fc/fc_fcoe.h>
50
51 #include "ixgbe.h"
52 #include "ixgbe_common.h"
53 #include "ixgbe_dcb_82599.h"
54 #include "ixgbe_sriov.h"
55
56 char ixgbe_driver_name[] = "ixgbe";
57 static const char ixgbe_driver_string[] =
58                               "Intel(R) 10 Gigabit PCI Express Network Driver";
59 #ifdef IXGBE_FCOE
60 char ixgbe_default_device_descr[] =
61                               "Intel(R) 10 Gigabit Network Connection";
62 #else
63 static char ixgbe_default_device_descr[] =
64                               "Intel(R) 10 Gigabit Network Connection";
65 #endif
66 #define DRV_VERSION "3.11.33-k"
67 const char ixgbe_driver_version[] = DRV_VERSION;
68 static const char ixgbe_copyright[] =
69                                 "Copyright (c) 1999-2012 Intel Corporation.";
70
71 static const struct ixgbe_info *ixgbe_info_tbl[] = {
72         [board_82598] = &ixgbe_82598_info,
73         [board_82599] = &ixgbe_82599_info,
74         [board_X540] = &ixgbe_X540_info,
75 };
76
77 /* ixgbe_pci_tbl - PCI Device ID Table
78  *
79  * Wildcard entries (PCI_ANY_ID) should come last
80  * Last entry must be all 0s
81  *
82  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
83  *   Class, Class Mask, private data (not used) }
84  */
85 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
86         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
87         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
88         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
89         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
90         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
91         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
92         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
93         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
94         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
95         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
96         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
97         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
98         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
99         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
100         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
101         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
102         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
103         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
104         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
105         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
106         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
107         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
108         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
109         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
110         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
111         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
112         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
113         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
114         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
115         /* required last entry */
116         {0, }
117 };
118 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
119
120 #ifdef CONFIG_IXGBE_DCA
121 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
122                             void *p);
123 static struct notifier_block dca_notifier = {
124         .notifier_call = ixgbe_notify_dca,
125         .next          = NULL,
126         .priority      = 0
127 };
128 #endif
129
130 #ifdef CONFIG_PCI_IOV
131 static unsigned int max_vfs;
132 module_param(max_vfs, uint, 0);
133 MODULE_PARM_DESC(max_vfs,
134                  "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63");
135 #endif /* CONFIG_PCI_IOV */
136
137 static unsigned int allow_unsupported_sfp;
138 module_param(allow_unsupported_sfp, uint, 0);
139 MODULE_PARM_DESC(allow_unsupported_sfp,
140                  "Allow unsupported and untested SFP+ modules on 82599-based adapters");
141
142 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
143 static int debug = -1;
144 module_param(debug, int, 0);
145 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
146
147 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
148 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
149 MODULE_LICENSE("GPL");
150 MODULE_VERSION(DRV_VERSION);
151
152 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
153 {
154         if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
155             !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
156                 schedule_work(&adapter->service_task);
157 }
158
159 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
160 {
161         BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
162
163         /* flush memory to make sure state is correct before next watchdog */
164         smp_mb__before_clear_bit();
165         clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
166 }
167
168 struct ixgbe_reg_info {
169         u32 ofs;
170         char *name;
171 };
172
173 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
174
175         /* General Registers */
176         {IXGBE_CTRL, "CTRL"},
177         {IXGBE_STATUS, "STATUS"},
178         {IXGBE_CTRL_EXT, "CTRL_EXT"},
179
180         /* Interrupt Registers */
181         {IXGBE_EICR, "EICR"},
182
183         /* RX Registers */
184         {IXGBE_SRRCTL(0), "SRRCTL"},
185         {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
186         {IXGBE_RDLEN(0), "RDLEN"},
187         {IXGBE_RDH(0), "RDH"},
188         {IXGBE_RDT(0), "RDT"},
189         {IXGBE_RXDCTL(0), "RXDCTL"},
190         {IXGBE_RDBAL(0), "RDBAL"},
191         {IXGBE_RDBAH(0), "RDBAH"},
192
193         /* TX Registers */
194         {IXGBE_TDBAL(0), "TDBAL"},
195         {IXGBE_TDBAH(0), "TDBAH"},
196         {IXGBE_TDLEN(0), "TDLEN"},
197         {IXGBE_TDH(0), "TDH"},
198         {IXGBE_TDT(0), "TDT"},
199         {IXGBE_TXDCTL(0), "TXDCTL"},
200
201         /* List Terminator */
202         {}
203 };
204
205
206 /*
207  * ixgbe_regdump - register printout routine
208  */
209 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
210 {
211         int i = 0, j = 0;
212         char rname[16];
213         u32 regs[64];
214
215         switch (reginfo->ofs) {
216         case IXGBE_SRRCTL(0):
217                 for (i = 0; i < 64; i++)
218                         regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
219                 break;
220         case IXGBE_DCA_RXCTRL(0):
221                 for (i = 0; i < 64; i++)
222                         regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
223                 break;
224         case IXGBE_RDLEN(0):
225                 for (i = 0; i < 64; i++)
226                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
227                 break;
228         case IXGBE_RDH(0):
229                 for (i = 0; i < 64; i++)
230                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
231                 break;
232         case IXGBE_RDT(0):
233                 for (i = 0; i < 64; i++)
234                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
235                 break;
236         case IXGBE_RXDCTL(0):
237                 for (i = 0; i < 64; i++)
238                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
239                 break;
240         case IXGBE_RDBAL(0):
241                 for (i = 0; i < 64; i++)
242                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
243                 break;
244         case IXGBE_RDBAH(0):
245                 for (i = 0; i < 64; i++)
246                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
247                 break;
248         case IXGBE_TDBAL(0):
249                 for (i = 0; i < 64; i++)
250                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
251                 break;
252         case IXGBE_TDBAH(0):
253                 for (i = 0; i < 64; i++)
254                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
255                 break;
256         case IXGBE_TDLEN(0):
257                 for (i = 0; i < 64; i++)
258                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
259                 break;
260         case IXGBE_TDH(0):
261                 for (i = 0; i < 64; i++)
262                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
263                 break;
264         case IXGBE_TDT(0):
265                 for (i = 0; i < 64; i++)
266                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
267                 break;
268         case IXGBE_TXDCTL(0):
269                 for (i = 0; i < 64; i++)
270                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
271                 break;
272         default:
273                 pr_info("%-15s %08x\n", reginfo->name,
274                         IXGBE_READ_REG(hw, reginfo->ofs));
275                 return;
276         }
277
278         for (i = 0; i < 8; i++) {
279                 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
280                 pr_err("%-15s", rname);
281                 for (j = 0; j < 8; j++)
282                         pr_cont(" %08x", regs[i*8+j]);
283                 pr_cont("\n");
284         }
285
286 }
287
288 /*
289  * ixgbe_dump - Print registers, tx-rings and rx-rings
290  */
291 static void ixgbe_dump(struct ixgbe_adapter *adapter)
292 {
293         struct net_device *netdev = adapter->netdev;
294         struct ixgbe_hw *hw = &adapter->hw;
295         struct ixgbe_reg_info *reginfo;
296         int n = 0;
297         struct ixgbe_ring *tx_ring;
298         struct ixgbe_tx_buffer *tx_buffer;
299         union ixgbe_adv_tx_desc *tx_desc;
300         struct my_u0 { u64 a; u64 b; } *u0;
301         struct ixgbe_ring *rx_ring;
302         union ixgbe_adv_rx_desc *rx_desc;
303         struct ixgbe_rx_buffer *rx_buffer_info;
304         u32 staterr;
305         int i = 0;
306
307         if (!netif_msg_hw(adapter))
308                 return;
309
310         /* Print netdevice Info */
311         if (netdev) {
312                 dev_info(&adapter->pdev->dev, "Net device Info\n");
313                 pr_info("Device Name     state            "
314                         "trans_start      last_rx\n");
315                 pr_info("%-15s %016lX %016lX %016lX\n",
316                         netdev->name,
317                         netdev->state,
318                         netdev->trans_start,
319                         netdev->last_rx);
320         }
321
322         /* Print Registers */
323         dev_info(&adapter->pdev->dev, "Register Dump\n");
324         pr_info(" Register Name   Value\n");
325         for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
326              reginfo->name; reginfo++) {
327                 ixgbe_regdump(hw, reginfo);
328         }
329
330         /* Print TX Ring Summary */
331         if (!netdev || !netif_running(netdev))
332                 goto exit;
333
334         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
335         pr_info(" %s     %s              %s        %s\n",
336                 "Queue [NTU] [NTC] [bi(ntc)->dma  ]",
337                 "leng", "ntw", "timestamp");
338         for (n = 0; n < adapter->num_tx_queues; n++) {
339                 tx_ring = adapter->tx_ring[n];
340                 tx_buffer = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
341                 pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",
342                            n, tx_ring->next_to_use, tx_ring->next_to_clean,
343                            (u64)dma_unmap_addr(tx_buffer, dma),
344                            dma_unmap_len(tx_buffer, len),
345                            tx_buffer->next_to_watch,
346                            (u64)tx_buffer->time_stamp);
347         }
348
349         /* Print TX Rings */
350         if (!netif_msg_tx_done(adapter))
351                 goto rx_ring_summary;
352
353         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
354
355         /* Transmit Descriptor Formats
356          *
357          * 82598 Advanced Transmit Descriptor
358          *   +--------------------------------------------------------------+
359          * 0 |         Buffer Address [63:0]                                |
360          *   +--------------------------------------------------------------+
361          * 8 |  PAYLEN  | POPTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
362          *   +--------------------------------------------------------------+
363          *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
364          *
365          * 82598 Advanced Transmit Descriptor (Write-Back Format)
366          *   +--------------------------------------------------------------+
367          * 0 |                          RSV [63:0]                          |
368          *   +--------------------------------------------------------------+
369          * 8 |            RSV           |  STA  |          NXTSEQ           |
370          *   +--------------------------------------------------------------+
371          *   63                       36 35   32 31                         0
372          *
373          * 82599+ Advanced Transmit Descriptor
374          *   +--------------------------------------------------------------+
375          * 0 |         Buffer Address [63:0]                                |
376          *   +--------------------------------------------------------------+
377          * 8 |PAYLEN  |POPTS|CC|IDX  |STA  |DCMD  |DTYP |MAC  |RSV  |DTALEN |
378          *   +--------------------------------------------------------------+
379          *   63     46 45 40 39 38 36 35 32 31  24 23 20 19 18 17 16 15     0
380          *
381          * 82599+ Advanced Transmit Descriptor (Write-Back Format)
382          *   +--------------------------------------------------------------+
383          * 0 |                          RSV [63:0]                          |
384          *   +--------------------------------------------------------------+
385          * 8 |            RSV           |  STA  |           RSV             |
386          *   +--------------------------------------------------------------+
387          *   63                       36 35   32 31                         0
388          */
389
390         for (n = 0; n < adapter->num_tx_queues; n++) {
391                 tx_ring = adapter->tx_ring[n];
392                 pr_info("------------------------------------\n");
393                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
394                 pr_info("------------------------------------\n");
395                 pr_info("%s%s    %s              %s        %s          %s\n",
396                         "T [desc]     [address 63:0  ] ",
397                         "[PlPOIdStDDt Ln] [bi->dma       ] ",
398                         "leng", "ntw", "timestamp", "bi->skb");
399
400                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
401                         tx_desc = IXGBE_TX_DESC(tx_ring, i);
402                         tx_buffer = &tx_ring->tx_buffer_info[i];
403                         u0 = (struct my_u0 *)tx_desc;
404                         if (dma_unmap_len(tx_buffer, len) > 0) {
405                                 pr_info("T [0x%03X]    %016llX %016llX %016llX %08X %p %016llX %p",
406                                         i,
407                                         le64_to_cpu(u0->a),
408                                         le64_to_cpu(u0->b),
409                                         (u64)dma_unmap_addr(tx_buffer, dma),
410                                         dma_unmap_len(tx_buffer, len),
411                                         tx_buffer->next_to_watch,
412                                         (u64)tx_buffer->time_stamp,
413                                         tx_buffer->skb);
414                                 if (i == tx_ring->next_to_use &&
415                                         i == tx_ring->next_to_clean)
416                                         pr_cont(" NTC/U\n");
417                                 else if (i == tx_ring->next_to_use)
418                                         pr_cont(" NTU\n");
419                                 else if (i == tx_ring->next_to_clean)
420                                         pr_cont(" NTC\n");
421                                 else
422                                         pr_cont("\n");
423
424                                 if (netif_msg_pktdata(adapter) &&
425                                     tx_buffer->skb)
426                                         print_hex_dump(KERN_INFO, "",
427                                                 DUMP_PREFIX_ADDRESS, 16, 1,
428                                                 tx_buffer->skb->data,
429                                                 dma_unmap_len(tx_buffer, len),
430                                                 true);
431                         }
432                 }
433         }
434
435         /* Print RX Rings Summary */
436 rx_ring_summary:
437         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
438         pr_info("Queue [NTU] [NTC]\n");
439         for (n = 0; n < adapter->num_rx_queues; n++) {
440                 rx_ring = adapter->rx_ring[n];
441                 pr_info("%5d %5X %5X\n",
442                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
443         }
444
445         /* Print RX Rings */
446         if (!netif_msg_rx_status(adapter))
447                 goto exit;
448
449         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
450
451         /* Receive Descriptor Formats
452          *
453          * 82598 Advanced Receive Descriptor (Read) Format
454          *    63                                           1        0
455          *    +-----------------------------------------------------+
456          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
457          *    +----------------------------------------------+------+
458          *  8 |       Header Buffer Address [63:1]           |  DD  |
459          *    +-----------------------------------------------------+
460          *
461          *
462          * 82598 Advanced Receive Descriptor (Write-Back) Format
463          *
464          *   63       48 47    32 31  30      21 20 16 15   4 3     0
465          *   +------------------------------------------------------+
466          * 0 |       RSS Hash /  |SPH| HDR_LEN  | RSV |Packet|  RSS |
467          *   | Packet   | IP     |   |          |     | Type | Type |
468          *   | Checksum | Ident  |   |          |     |      |      |
469          *   +------------------------------------------------------+
470          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
471          *   +------------------------------------------------------+
472          *   63       48 47    32 31            20 19               0
473          *
474          * 82599+ Advanced Receive Descriptor (Read) Format
475          *    63                                           1        0
476          *    +-----------------------------------------------------+
477          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
478          *    +----------------------------------------------+------+
479          *  8 |       Header Buffer Address [63:1]           |  DD  |
480          *    +-----------------------------------------------------+
481          *
482          *
483          * 82599+ Advanced Receive Descriptor (Write-Back) Format
484          *
485          *   63       48 47    32 31  30      21 20 17 16   4 3     0
486          *   +------------------------------------------------------+
487          * 0 |RSS / Frag Checksum|SPH| HDR_LEN  |RSC- |Packet|  RSS |
488          *   |/ RTT / PCoE_PARAM |   |          | CNT | Type | Type |
489          *   |/ Flow Dir Flt ID  |   |          |     |      |      |
490          *   +------------------------------------------------------+
491          * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |
492          *   +------------------------------------------------------+
493          *   63       48 47    32 31          20 19                 0
494          */
495
496         for (n = 0; n < adapter->num_rx_queues; n++) {
497                 rx_ring = adapter->rx_ring[n];
498                 pr_info("------------------------------------\n");
499                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
500                 pr_info("------------------------------------\n");
501                 pr_info("%s%s%s",
502                         "R  [desc]      [ PktBuf     A0] ",
503                         "[  HeadBuf   DD] [bi->dma       ] [bi->skb       ] ",
504                         "<-- Adv Rx Read format\n");
505                 pr_info("%s%s%s",
506                         "RWB[desc]      [PcsmIpSHl PtRs] ",
507                         "[vl er S cks ln] ---------------- [bi->skb       ] ",
508                         "<-- Adv Rx Write-Back format\n");
509
510                 for (i = 0; i < rx_ring->count; i++) {
511                         rx_buffer_info = &rx_ring->rx_buffer_info[i];
512                         rx_desc = IXGBE_RX_DESC(rx_ring, i);
513                         u0 = (struct my_u0 *)rx_desc;
514                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
515                         if (staterr & IXGBE_RXD_STAT_DD) {
516                                 /* Descriptor Done */
517                                 pr_info("RWB[0x%03X]     %016llX "
518                                         "%016llX ---------------- %p", i,
519                                         le64_to_cpu(u0->a),
520                                         le64_to_cpu(u0->b),
521                                         rx_buffer_info->skb);
522                         } else {
523                                 pr_info("R  [0x%03X]     %016llX "
524                                         "%016llX %016llX %p", i,
525                                         le64_to_cpu(u0->a),
526                                         le64_to_cpu(u0->b),
527                                         (u64)rx_buffer_info->dma,
528                                         rx_buffer_info->skb);
529
530                                 if (netif_msg_pktdata(adapter) &&
531                                     rx_buffer_info->dma) {
532                                         print_hex_dump(KERN_INFO, "",
533                                            DUMP_PREFIX_ADDRESS, 16, 1,
534                                            page_address(rx_buffer_info->page) +
535                                                     rx_buffer_info->page_offset,
536                                            ixgbe_rx_bufsz(rx_ring), true);
537                                 }
538                         }
539
540                         if (i == rx_ring->next_to_use)
541                                 pr_cont(" NTU\n");
542                         else if (i == rx_ring->next_to_clean)
543                                 pr_cont(" NTC\n");
544                         else
545                                 pr_cont("\n");
546
547                 }
548         }
549
550 exit:
551         return;
552 }
553
554 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
555 {
556         u32 ctrl_ext;
557
558         /* Let firmware take over control of h/w */
559         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
560         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
561                         ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
562 }
563
564 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
565 {
566         u32 ctrl_ext;
567
568         /* Let firmware know the driver has taken over */
569         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
570         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
571                         ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
572 }
573
574 /**
575  * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
576  * @adapter: pointer to adapter struct
577  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
578  * @queue: queue to map the corresponding interrupt to
579  * @msix_vector: the vector to map to the corresponding queue
580  *
581  */
582 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
583                            u8 queue, u8 msix_vector)
584 {
585         u32 ivar, index;
586         struct ixgbe_hw *hw = &adapter->hw;
587         switch (hw->mac.type) {
588         case ixgbe_mac_82598EB:
589                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
590                 if (direction == -1)
591                         direction = 0;
592                 index = (((direction * 64) + queue) >> 2) & 0x1F;
593                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
594                 ivar &= ~(0xFF << (8 * (queue & 0x3)));
595                 ivar |= (msix_vector << (8 * (queue & 0x3)));
596                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
597                 break;
598         case ixgbe_mac_82599EB:
599         case ixgbe_mac_X540:
600                 if (direction == -1) {
601                         /* other causes */
602                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
603                         index = ((queue & 1) * 8);
604                         ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
605                         ivar &= ~(0xFF << index);
606                         ivar |= (msix_vector << index);
607                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
608                         break;
609                 } else {
610                         /* tx or rx causes */
611                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
612                         index = ((16 * (queue & 1)) + (8 * direction));
613                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
614                         ivar &= ~(0xFF << index);
615                         ivar |= (msix_vector << index);
616                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
617                         break;
618                 }
619         default:
620                 break;
621         }
622 }
623
624 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
625                                           u64 qmask)
626 {
627         u32 mask;
628
629         switch (adapter->hw.mac.type) {
630         case ixgbe_mac_82598EB:
631                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
632                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
633                 break;
634         case ixgbe_mac_82599EB:
635         case ixgbe_mac_X540:
636                 mask = (qmask & 0xFFFFFFFF);
637                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
638                 mask = (qmask >> 32);
639                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
640                 break;
641         default:
642                 break;
643         }
644 }
645
646 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *ring,
647                                       struct ixgbe_tx_buffer *tx_buffer)
648 {
649         if (tx_buffer->skb) {
650                 dev_kfree_skb_any(tx_buffer->skb);
651                 if (dma_unmap_len(tx_buffer, len))
652                         dma_unmap_single(ring->dev,
653                                          dma_unmap_addr(tx_buffer, dma),
654                                          dma_unmap_len(tx_buffer, len),
655                                          DMA_TO_DEVICE);
656         } else if (dma_unmap_len(tx_buffer, len)) {
657                 dma_unmap_page(ring->dev,
658                                dma_unmap_addr(tx_buffer, dma),
659                                dma_unmap_len(tx_buffer, len),
660                                DMA_TO_DEVICE);
661         }
662         tx_buffer->next_to_watch = NULL;
663         tx_buffer->skb = NULL;
664         dma_unmap_len_set(tx_buffer, len, 0);
665         /* tx_buffer must be completely set up in the transmit path */
666 }
667
668 static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)
669 {
670         struct ixgbe_hw *hw = &adapter->hw;
671         struct ixgbe_hw_stats *hwstats = &adapter->stats;
672         int i;
673         u32 data;
674
675         if ((hw->fc.current_mode != ixgbe_fc_full) &&
676             (hw->fc.current_mode != ixgbe_fc_rx_pause))
677                 return;
678
679         switch (hw->mac.type) {
680         case ixgbe_mac_82598EB:
681                 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
682                 break;
683         default:
684                 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
685         }
686         hwstats->lxoffrxc += data;
687
688         /* refill credits (no tx hang) if we received xoff */
689         if (!data)
690                 return;
691
692         for (i = 0; i < adapter->num_tx_queues; i++)
693                 clear_bit(__IXGBE_HANG_CHECK_ARMED,
694                           &adapter->tx_ring[i]->state);
695 }
696
697 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
698 {
699         struct ixgbe_hw *hw = &adapter->hw;
700         struct ixgbe_hw_stats *hwstats = &adapter->stats;
701         u32 xoff[8] = {0};
702         u8 tc;
703         int i;
704         bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
705
706         if (adapter->ixgbe_ieee_pfc)
707                 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
708
709         if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {
710                 ixgbe_update_xoff_rx_lfc(adapter);
711                 return;
712         }
713
714         /* update stats for each tc, only valid with PFC enabled */
715         for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
716                 u32 pxoffrxc;
717
718                 switch (hw->mac.type) {
719                 case ixgbe_mac_82598EB:
720                         pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
721                         break;
722                 default:
723                         pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
724                 }
725                 hwstats->pxoffrxc[i] += pxoffrxc;
726                 /* Get the TC for given UP */
727                 tc = netdev_get_prio_tc_map(adapter->netdev, i);
728                 xoff[tc] += pxoffrxc;
729         }
730
731         /* disarm tx queues that have received xoff frames */
732         for (i = 0; i < adapter->num_tx_queues; i++) {
733                 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
734
735                 tc = tx_ring->dcb_tc;
736                 if (xoff[tc])
737                         clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
738         }
739 }
740
741 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
742 {
743         return ring->stats.packets;
744 }
745
746 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
747 {
748         struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
749         struct ixgbe_hw *hw = &adapter->hw;
750
751         u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
752         u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
753
754         if (head != tail)
755                 return (head < tail) ?
756                         tail - head : (tail + ring->count - head);
757
758         return 0;
759 }
760
761 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
762 {
763         u32 tx_done = ixgbe_get_tx_completed(tx_ring);
764         u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
765         u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
766         bool ret = false;
767
768         clear_check_for_tx_hang(tx_ring);
769
770         /*
771          * Check for a hung queue, but be thorough. This verifies
772          * that a transmit has been completed since the previous
773          * check AND there is at least one packet pending. The
774          * ARMED bit is set to indicate a potential hang. The
775          * bit is cleared if a pause frame is received to remove
776          * false hang detection due to PFC or 802.3x frames. By
777          * requiring this to fail twice we avoid races with
778          * pfc clearing the ARMED bit and conditions where we
779          * run the check_tx_hang logic with a transmit completion
780          * pending but without time to complete it yet.
781          */
782         if ((tx_done_old == tx_done) && tx_pending) {
783                 /* make sure it is true for two checks in a row */
784                 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
785                                        &tx_ring->state);
786         } else {
787                 /* update completed stats and continue */
788                 tx_ring->tx_stats.tx_done_old = tx_done;
789                 /* reset the countdown */
790                 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
791         }
792
793         return ret;
794 }
795
796 /**
797  * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
798  * @adapter: driver private struct
799  **/
800 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
801 {
802
803         /* Do the reset outside of interrupt context */
804         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
805                 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
806                 ixgbe_service_event_schedule(adapter);
807         }
808 }
809
810 /**
811  * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
812  * @q_vector: structure containing interrupt and ring information
813  * @tx_ring: tx ring to clean
814  **/
815 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
816                                struct ixgbe_ring *tx_ring)
817 {
818         struct ixgbe_adapter *adapter = q_vector->adapter;
819         struct ixgbe_tx_buffer *tx_buffer;
820         union ixgbe_adv_tx_desc *tx_desc;
821         unsigned int total_bytes = 0, total_packets = 0;
822         unsigned int budget = q_vector->tx.work_limit;
823         unsigned int i = tx_ring->next_to_clean;
824
825         if (test_bit(__IXGBE_DOWN, &adapter->state))
826                 return true;
827
828         tx_buffer = &tx_ring->tx_buffer_info[i];
829         tx_desc = IXGBE_TX_DESC(tx_ring, i);
830         i -= tx_ring->count;
831
832         do {
833                 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
834
835                 /* if next_to_watch is not set then there is no work pending */
836                 if (!eop_desc)
837                         break;
838
839                 /* prevent any other reads prior to eop_desc */
840                 rmb();
841
842                 /* if DD is not set pending work has not been completed */
843                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
844                         break;
845
846                 /* clear next_to_watch to prevent false hangs */
847                 tx_buffer->next_to_watch = NULL;
848
849                 /* update the statistics for this packet */
850                 total_bytes += tx_buffer->bytecount;
851                 total_packets += tx_buffer->gso_segs;
852
853                 if (unlikely(tx_buffer->tx_flags & IXGBE_TX_FLAGS_TSTAMP))
854                         ixgbe_ptp_tx_hwtstamp(q_vector, tx_buffer->skb);
855
856                 /* free the skb */
857                 dev_kfree_skb_any(tx_buffer->skb);
858
859                 /* unmap skb header data */
860                 dma_unmap_single(tx_ring->dev,
861                                  dma_unmap_addr(tx_buffer, dma),
862                                  dma_unmap_len(tx_buffer, len),
863                                  DMA_TO_DEVICE);
864
865                 /* clear tx_buffer data */
866                 tx_buffer->skb = NULL;
867                 dma_unmap_len_set(tx_buffer, len, 0);
868
869                 /* unmap remaining buffers */
870                 while (tx_desc != eop_desc) {
871                         tx_buffer++;
872                         tx_desc++;
873                         i++;
874                         if (unlikely(!i)) {
875                                 i -= tx_ring->count;
876                                 tx_buffer = tx_ring->tx_buffer_info;
877                                 tx_desc = IXGBE_TX_DESC(tx_ring, 0);
878                         }
879
880                         /* unmap any remaining paged data */
881                         if (dma_unmap_len(tx_buffer, len)) {
882                                 dma_unmap_page(tx_ring->dev,
883                                                dma_unmap_addr(tx_buffer, dma),
884                                                dma_unmap_len(tx_buffer, len),
885                                                DMA_TO_DEVICE);
886                                 dma_unmap_len_set(tx_buffer, len, 0);
887                         }
888                 }
889
890                 /* move us one more past the eop_desc for start of next pkt */
891                 tx_buffer++;
892                 tx_desc++;
893                 i++;
894                 if (unlikely(!i)) {
895                         i -= tx_ring->count;
896                         tx_buffer = tx_ring->tx_buffer_info;
897                         tx_desc = IXGBE_TX_DESC(tx_ring, 0);
898                 }
899
900                 /* issue prefetch for next Tx descriptor */
901                 prefetch(tx_desc);
902
903                 /* update budget accounting */
904                 budget--;
905         } while (likely(budget));
906
907         i += tx_ring->count;
908         tx_ring->next_to_clean = i;
909         u64_stats_update_begin(&tx_ring->syncp);
910         tx_ring->stats.bytes += total_bytes;
911         tx_ring->stats.packets += total_packets;
912         u64_stats_update_end(&tx_ring->syncp);
913         q_vector->tx.total_bytes += total_bytes;
914         q_vector->tx.total_packets += total_packets;
915
916         if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
917                 /* schedule immediate reset if we believe we hung */
918                 struct ixgbe_hw *hw = &adapter->hw;
919                 e_err(drv, "Detected Tx Unit Hang\n"
920                         "  Tx Queue             <%d>\n"
921                         "  TDH, TDT             <%x>, <%x>\n"
922                         "  next_to_use          <%x>\n"
923                         "  next_to_clean        <%x>\n"
924                         "tx_buffer_info[next_to_clean]\n"
925                         "  time_stamp           <%lx>\n"
926                         "  jiffies              <%lx>\n",
927                         tx_ring->queue_index,
928                         IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
929                         IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
930                         tx_ring->next_to_use, i,
931                         tx_ring->tx_buffer_info[i].time_stamp, jiffies);
932
933                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
934
935                 e_info(probe,
936                        "tx hang %d detected on queue %d, resetting adapter\n",
937                         adapter->tx_timeout_count + 1, tx_ring->queue_index);
938
939                 /* schedule immediate reset if we believe we hung */
940                 ixgbe_tx_timeout_reset(adapter);
941
942                 /* the adapter is about to reset, no point in enabling stuff */
943                 return true;
944         }
945
946         netdev_tx_completed_queue(txring_txq(tx_ring),
947                                   total_packets, total_bytes);
948
949 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
950         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
951                      (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
952                 /* Make sure that anybody stopping the queue after this
953                  * sees the new next_to_clean.
954                  */
955                 smp_mb();
956                 if (__netif_subqueue_stopped(tx_ring->netdev,
957                                              tx_ring->queue_index)
958                     && !test_bit(__IXGBE_DOWN, &adapter->state)) {
959                         netif_wake_subqueue(tx_ring->netdev,
960                                             tx_ring->queue_index);
961                         ++tx_ring->tx_stats.restart_queue;
962                 }
963         }
964
965         return !!budget;
966 }
967
968 #ifdef CONFIG_IXGBE_DCA
969 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
970                                 struct ixgbe_ring *tx_ring,
971                                 int cpu)
972 {
973         struct ixgbe_hw *hw = &adapter->hw;
974         u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
975         u16 reg_offset;
976
977         switch (hw->mac.type) {
978         case ixgbe_mac_82598EB:
979                 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);
980                 break;
981         case ixgbe_mac_82599EB:
982         case ixgbe_mac_X540:
983                 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);
984                 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;
985                 break;
986         default:
987                 /* for unknown hardware do not write register */
988                 return;
989         }
990
991         /*
992          * We can enable relaxed ordering for reads, but not writes when
993          * DCA is enabled.  This is due to a known issue in some chipsets
994          * which will cause the DCA tag to be cleared.
995          */
996         txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |
997                   IXGBE_DCA_TXCTRL_DATA_RRO_EN |
998                   IXGBE_DCA_TXCTRL_DESC_DCA_EN;
999
1000         IXGBE_WRITE_REG(hw, reg_offset, txctrl);
1001 }
1002
1003 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
1004                                 struct ixgbe_ring *rx_ring,
1005                                 int cpu)
1006 {
1007         struct ixgbe_hw *hw = &adapter->hw;
1008         u32 rxctrl = dca3_get_tag(rx_ring->dev, cpu);
1009         u8 reg_idx = rx_ring->reg_idx;
1010
1011
1012         switch (hw->mac.type) {
1013         case ixgbe_mac_82599EB:
1014         case ixgbe_mac_X540:
1015                 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;
1016                 break;
1017         default:
1018                 break;
1019         }
1020
1021         /*
1022          * We can enable relaxed ordering for reads, but not writes when
1023          * DCA is enabled.  This is due to a known issue in some chipsets
1024          * which will cause the DCA tag to be cleared.
1025          */
1026         rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |
1027                   IXGBE_DCA_RXCTRL_DESC_DCA_EN;
1028
1029         IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
1030 }
1031
1032 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
1033 {
1034         struct ixgbe_adapter *adapter = q_vector->adapter;
1035         struct ixgbe_ring *ring;
1036         int cpu = get_cpu();
1037
1038         if (q_vector->cpu == cpu)
1039                 goto out_no_update;
1040
1041         ixgbe_for_each_ring(ring, q_vector->tx)
1042                 ixgbe_update_tx_dca(adapter, ring, cpu);
1043
1044         ixgbe_for_each_ring(ring, q_vector->rx)
1045                 ixgbe_update_rx_dca(adapter, ring, cpu);
1046
1047         q_vector->cpu = cpu;
1048 out_no_update:
1049         put_cpu();
1050 }
1051
1052 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
1053 {
1054         int i;
1055
1056         if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
1057                 return;
1058
1059         /* always use CB2 mode, difference is masked in the CB driver */
1060         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
1061
1062         for (i = 0; i < adapter->num_q_vectors; i++) {
1063                 adapter->q_vector[i]->cpu = -1;
1064                 ixgbe_update_dca(adapter->q_vector[i]);
1065         }
1066 }
1067
1068 static int __ixgbe_notify_dca(struct device *dev, void *data)
1069 {
1070         struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1071         unsigned long event = *(unsigned long *)data;
1072
1073         if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
1074                 return 0;
1075
1076         switch (event) {
1077         case DCA_PROVIDER_ADD:
1078                 /* if we're already enabled, don't do it again */
1079                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1080                         break;
1081                 if (dca_add_requester(dev) == 0) {
1082                         adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1083                         ixgbe_setup_dca(adapter);
1084                         break;
1085                 }
1086                 /* Fall Through since DCA is disabled. */
1087         case DCA_PROVIDER_REMOVE:
1088                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1089                         dca_remove_requester(dev);
1090                         adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
1091                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
1092                 }
1093                 break;
1094         }
1095
1096         return 0;
1097 }
1098
1099 #endif /* CONFIG_IXGBE_DCA */
1100 static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,
1101                                  union ixgbe_adv_rx_desc *rx_desc,
1102                                  struct sk_buff *skb)
1103 {
1104         if (ring->netdev->features & NETIF_F_RXHASH)
1105                 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
1106 }
1107
1108 #ifdef IXGBE_FCOE
1109 /**
1110  * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
1111  * @ring: structure containing ring specific data
1112  * @rx_desc: advanced rx descriptor
1113  *
1114  * Returns : true if it is FCoE pkt
1115  */
1116 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,
1117                                     union ixgbe_adv_rx_desc *rx_desc)
1118 {
1119         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1120
1121         return test_bit(__IXGBE_RX_FCOE, &ring->state) &&
1122                ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1123                 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1124                              IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1125 }
1126
1127 #endif /* IXGBE_FCOE */
1128 /**
1129  * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1130  * @ring: structure containing ring specific data
1131  * @rx_desc: current Rx descriptor being processed
1132  * @skb: skb currently being received and modified
1133  **/
1134 static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,
1135                                      union ixgbe_adv_rx_desc *rx_desc,
1136                                      struct sk_buff *skb)
1137 {
1138         skb_checksum_none_assert(skb);
1139
1140         /* Rx csum disabled */
1141         if (!(ring->netdev->features & NETIF_F_RXCSUM))
1142                 return;
1143
1144         /* if IP and error */
1145         if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
1146             ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
1147                 ring->rx_stats.csum_err++;
1148                 return;
1149         }
1150
1151         if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
1152                 return;
1153
1154         if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
1155                 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1156
1157                 /*
1158                  * 82599 errata, UDP frames with a 0 checksum can be marked as
1159                  * checksum errors.
1160                  */
1161                 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&
1162                     test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))
1163                         return;
1164
1165                 ring->rx_stats.csum_err++;
1166                 return;
1167         }
1168
1169         /* It must be a TCP or UDP packet with a valid checksum */
1170         skb->ip_summed = CHECKSUM_UNNECESSARY;
1171 }
1172
1173 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1174 {
1175         rx_ring->next_to_use = val;
1176
1177         /* update next to alloc since we have filled the ring */
1178         rx_ring->next_to_alloc = val;
1179         /*
1180          * Force memory writes to complete before letting h/w
1181          * know there are new descriptors to fetch.  (Only
1182          * applicable for weak-ordered memory model archs,
1183          * such as IA-64).
1184          */
1185         wmb();
1186         writel(val, rx_ring->tail);
1187 }
1188
1189 static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
1190                                     struct ixgbe_rx_buffer *bi)
1191 {
1192         struct page *page = bi->page;
1193         dma_addr_t dma = bi->dma;
1194
1195         /* since we are recycling buffers we should seldom need to alloc */
1196         if (likely(dma))
1197                 return true;
1198
1199         /* alloc new page for storage */
1200         if (likely(!page)) {
1201                 page = __skb_alloc_pages(GFP_ATOMIC | __GFP_COLD | __GFP_COMP,
1202                                          bi->skb, ixgbe_rx_pg_order(rx_ring));
1203                 if (unlikely(!page)) {
1204                         rx_ring->rx_stats.alloc_rx_page_failed++;
1205                         return false;
1206                 }
1207                 bi->page = page;
1208         }
1209
1210         /* map page for use */
1211         dma = dma_map_page(rx_ring->dev, page, 0,
1212                            ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
1213
1214         /*
1215          * if mapping failed free memory back to system since
1216          * there isn't much point in holding memory we can't use
1217          */
1218         if (dma_mapping_error(rx_ring->dev, dma)) {
1219                 __free_pages(page, ixgbe_rx_pg_order(rx_ring));
1220                 bi->page = NULL;
1221
1222                 rx_ring->rx_stats.alloc_rx_page_failed++;
1223                 return false;
1224         }
1225
1226         bi->dma = dma;
1227         bi->page_offset = 0;
1228
1229         return true;
1230 }
1231
1232 /**
1233  * ixgbe_alloc_rx_buffers - Replace used receive buffers
1234  * @rx_ring: ring to place buffers on
1235  * @cleaned_count: number of buffers to replace
1236  **/
1237 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1238 {
1239         union ixgbe_adv_rx_desc *rx_desc;
1240         struct ixgbe_rx_buffer *bi;
1241         u16 i = rx_ring->next_to_use;
1242
1243         /* nothing to do */
1244         if (!cleaned_count)
1245                 return;
1246
1247         rx_desc = IXGBE_RX_DESC(rx_ring, i);
1248         bi = &rx_ring->rx_buffer_info[i];
1249         i -= rx_ring->count;
1250
1251         do {
1252                 if (!ixgbe_alloc_mapped_page(rx_ring, bi))
1253                         break;
1254
1255                 /*
1256                  * Refresh the desc even if buffer_addrs didn't change
1257                  * because each write-back erases this info.
1258                  */
1259                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
1260
1261                 rx_desc++;
1262                 bi++;
1263                 i++;
1264                 if (unlikely(!i)) {
1265                         rx_desc = IXGBE_RX_DESC(rx_ring, 0);
1266                         bi = rx_ring->rx_buffer_info;
1267                         i -= rx_ring->count;
1268                 }
1269
1270                 /* clear the hdr_addr for the next_to_use descriptor */
1271                 rx_desc->read.hdr_addr = 0;
1272
1273                 cleaned_count--;
1274         } while (cleaned_count);
1275
1276         i += rx_ring->count;
1277
1278         if (rx_ring->next_to_use != i)
1279                 ixgbe_release_rx_desc(rx_ring, i);
1280 }
1281
1282 /**
1283  * ixgbe_get_headlen - determine size of header for RSC/LRO/GRO/FCOE
1284  * @data: pointer to the start of the headers
1285  * @max_len: total length of section to find headers in
1286  *
1287  * This function is meant to determine the length of headers that will
1288  * be recognized by hardware for LRO, GRO, and RSC offloads.  The main
1289  * motivation of doing this is to only perform one pull for IPv4 TCP
1290  * packets so that we can do basic things like calculating the gso_size
1291  * based on the average data per packet.
1292  **/
1293 static unsigned int ixgbe_get_headlen(unsigned char *data,
1294                                       unsigned int max_len)
1295 {
1296         union {
1297                 unsigned char *network;
1298                 /* l2 headers */
1299                 struct ethhdr *eth;
1300                 struct vlan_hdr *vlan;
1301                 /* l3 headers */
1302                 struct iphdr *ipv4;
1303                 struct ipv6hdr *ipv6;
1304         } hdr;
1305         __be16 protocol;
1306         u8 nexthdr = 0; /* default to not TCP */
1307         u8 hlen;
1308
1309         /* this should never happen, but better safe than sorry */
1310         if (max_len < ETH_HLEN)
1311                 return max_len;
1312
1313         /* initialize network frame pointer */
1314         hdr.network = data;
1315
1316         /* set first protocol and move network header forward */
1317         protocol = hdr.eth->h_proto;
1318         hdr.network += ETH_HLEN;
1319
1320         /* handle any vlan tag if present */
1321         if (protocol == __constant_htons(ETH_P_8021Q)) {
1322                 if ((hdr.network - data) > (max_len - VLAN_HLEN))
1323                         return max_len;
1324
1325                 protocol = hdr.vlan->h_vlan_encapsulated_proto;
1326                 hdr.network += VLAN_HLEN;
1327         }
1328
1329         /* handle L3 protocols */
1330         if (protocol == __constant_htons(ETH_P_IP)) {
1331                 if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
1332                         return max_len;
1333
1334                 /* access ihl as a u8 to avoid unaligned access on ia64 */
1335                 hlen = (hdr.network[0] & 0x0F) << 2;
1336
1337                 /* verify hlen meets minimum size requirements */
1338                 if (hlen < sizeof(struct iphdr))
1339                         return hdr.network - data;
1340
1341                 /* record next protocol if header is present */
1342                 if (!hdr.ipv4->frag_off)
1343                         nexthdr = hdr.ipv4->protocol;
1344         } else if (protocol == __constant_htons(ETH_P_IPV6)) {
1345                 if ((hdr.network - data) > (max_len - sizeof(struct ipv6hdr)))
1346                         return max_len;
1347
1348                 /* record next protocol */
1349                 nexthdr = hdr.ipv6->nexthdr;
1350                 hlen = sizeof(struct ipv6hdr);
1351 #ifdef IXGBE_FCOE
1352         } else if (protocol == __constant_htons(ETH_P_FCOE)) {
1353                 if ((hdr.network - data) > (max_len - FCOE_HEADER_LEN))
1354                         return max_len;
1355                 hlen = FCOE_HEADER_LEN;
1356 #endif
1357         } else {
1358                 return hdr.network - data;
1359         }
1360
1361         /* relocate pointer to start of L4 header */
1362         hdr.network += hlen;
1363
1364         /* finally sort out TCP/UDP */
1365         if (nexthdr == IPPROTO_TCP) {
1366                 if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
1367                         return max_len;
1368
1369                 /* access doff as a u8 to avoid unaligned access on ia64 */
1370                 hlen = (hdr.network[12] & 0xF0) >> 2;
1371
1372                 /* verify hlen meets minimum size requirements */
1373                 if (hlen < sizeof(struct tcphdr))
1374                         return hdr.network - data;
1375
1376                 hdr.network += hlen;
1377         } else if (nexthdr == IPPROTO_UDP) {
1378                 if ((hdr.network - data) > (max_len - sizeof(struct udphdr)))
1379                         return max_len;
1380
1381                 hdr.network += sizeof(struct udphdr);
1382         }
1383
1384         /*
1385          * If everything has gone correctly hdr.network should be the
1386          * data section of the packet and will be the end of the header.
1387          * If not then it probably represents the end of the last recognized
1388          * header.
1389          */
1390         if ((hdr.network - data) < max_len)
1391                 return hdr.network - data;
1392         else
1393                 return max_len;
1394 }
1395
1396 static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,
1397                                    struct sk_buff *skb)
1398 {
1399         u16 hdr_len = skb_headlen(skb);
1400
1401         /* set gso_size to avoid messing up TCP MSS */
1402         skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),
1403                                                  IXGBE_CB(skb)->append_cnt);
1404         skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1405 }
1406
1407 static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,
1408                                    struct sk_buff *skb)
1409 {
1410         /* if append_cnt is 0 then frame is not RSC */
1411         if (!IXGBE_CB(skb)->append_cnt)
1412                 return;
1413
1414         rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;
1415         rx_ring->rx_stats.rsc_flush++;
1416
1417         ixgbe_set_rsc_gso_size(rx_ring, skb);
1418
1419         /* gso_size is computed using append_cnt so always clear it last */
1420         IXGBE_CB(skb)->append_cnt = 0;
1421 }
1422
1423 /**
1424  * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor
1425  * @rx_ring: rx descriptor ring packet is being transacted on
1426  * @rx_desc: pointer to the EOP Rx descriptor
1427  * @skb: pointer to current skb being populated
1428  *
1429  * This function checks the ring, descriptor, and packet information in
1430  * order to populate the hash, checksum, VLAN, timestamp, protocol, and
1431  * other fields within the skb.
1432  **/
1433 static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
1434                                      union ixgbe_adv_rx_desc *rx_desc,
1435                                      struct sk_buff *skb)
1436 {
1437         struct net_device *dev = rx_ring->netdev;
1438
1439         ixgbe_update_rsc_stats(rx_ring, skb);
1440
1441         ixgbe_rx_hash(rx_ring, rx_desc, skb);
1442
1443         ixgbe_rx_checksum(rx_ring, rx_desc, skb);
1444
1445         ixgbe_ptp_rx_hwtstamp(rx_ring->q_vector, rx_desc, skb);
1446
1447         if ((dev->features & NETIF_F_HW_VLAN_RX) &&
1448             ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
1449                 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1450                 __vlan_hwaccel_put_tag(skb, vid);
1451         }
1452
1453         skb_record_rx_queue(skb, rx_ring->queue_index);
1454
1455         skb->protocol = eth_type_trans(skb, dev);
1456 }
1457
1458 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
1459                          struct sk_buff *skb)
1460 {
1461         struct ixgbe_adapter *adapter = q_vector->adapter;
1462
1463         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1464                 napi_gro_receive(&q_vector->napi, skb);
1465         else
1466                 netif_rx(skb);
1467 }
1468
1469 /**
1470  * ixgbe_is_non_eop - process handling of non-EOP buffers
1471  * @rx_ring: Rx ring being processed
1472  * @rx_desc: Rx descriptor for current buffer
1473  * @skb: Current socket buffer containing buffer in progress
1474  *
1475  * This function updates next to clean.  If the buffer is an EOP buffer
1476  * this function exits returning false, otherwise it will place the
1477  * sk_buff in the next buffer to be chained and return true indicating
1478  * that this is in fact a non-EOP buffer.
1479  **/
1480 static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,
1481                              union ixgbe_adv_rx_desc *rx_desc,
1482                              struct sk_buff *skb)
1483 {
1484         u32 ntc = rx_ring->next_to_clean + 1;
1485
1486         /* fetch, update, and store next to clean */
1487         ntc = (ntc < rx_ring->count) ? ntc : 0;
1488         rx_ring->next_to_clean = ntc;
1489
1490         prefetch(IXGBE_RX_DESC(rx_ring, ntc));
1491
1492         /* update RSC append count if present */
1493         if (ring_is_rsc_enabled(rx_ring)) {
1494                 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &
1495                                      cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);
1496
1497                 if (unlikely(rsc_enabled)) {
1498                         u32 rsc_cnt = le32_to_cpu(rsc_enabled);
1499
1500                         rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
1501                         IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;
1502
1503                         /* update ntc based on RSC value */
1504                         ntc = le32_to_cpu(rx_desc->wb.upper.status_error);
1505                         ntc &= IXGBE_RXDADV_NEXTP_MASK;
1506                         ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;
1507                 }
1508         }
1509
1510         /* if we are the last buffer then there is nothing else to do */
1511         if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1512                 return false;
1513
1514         /* place skb in next buffer to be received */
1515         rx_ring->rx_buffer_info[ntc].skb = skb;
1516         rx_ring->rx_stats.non_eop_descs++;
1517
1518         return true;
1519 }
1520
1521 /**
1522  * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail
1523  * @rx_ring: rx descriptor ring packet is being transacted on
1524  * @skb: pointer to current skb being adjusted
1525  *
1526  * This function is an ixgbe specific version of __pskb_pull_tail.  The
1527  * main difference between this version and the original function is that
1528  * this function can make several assumptions about the state of things
1529  * that allow for significant optimizations versus the standard function.
1530  * As a result we can do things like drop a frag and maintain an accurate
1531  * truesize for the skb.
1532  */
1533 static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
1534                             struct sk_buff *skb)
1535 {
1536         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1537         unsigned char *va;
1538         unsigned int pull_len;
1539
1540         /*
1541          * it is valid to use page_address instead of kmap since we are
1542          * working with pages allocated out of the lomem pool per
1543          * alloc_page(GFP_ATOMIC)
1544          */
1545         va = skb_frag_address(frag);
1546
1547         /*
1548          * we need the header to contain the greater of either ETH_HLEN or
1549          * 60 bytes if the skb->len is less than 60 for skb_pad.
1550          */
1551         pull_len = ixgbe_get_headlen(va, IXGBE_RX_HDR_SIZE);
1552
1553         /* align pull length to size of long to optimize memcpy performance */
1554         skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
1555
1556         /* update all of the pointers */
1557         skb_frag_size_sub(frag, pull_len);
1558         frag->page_offset += pull_len;
1559         skb->data_len -= pull_len;
1560         skb->tail += pull_len;
1561 }
1562
1563 /**
1564  * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB
1565  * @rx_ring: rx descriptor ring packet is being transacted on
1566  * @skb: pointer to current skb being updated
1567  *
1568  * This function provides a basic DMA sync up for the first fragment of an
1569  * skb.  The reason for doing this is that the first fragment cannot be
1570  * unmapped until we have reached the end of packet descriptor for a buffer
1571  * chain.
1572  */
1573 static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
1574                                 struct sk_buff *skb)
1575 {
1576         /* if the page was released unmap it, else just sync our portion */
1577         if (unlikely(IXGBE_CB(skb)->page_released)) {
1578                 dma_unmap_page(rx_ring->dev, IXGBE_CB(skb)->dma,
1579                                ixgbe_rx_pg_size(rx_ring), DMA_FROM_DEVICE);
1580                 IXGBE_CB(skb)->page_released = false;
1581         } else {
1582                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
1583
1584                 dma_sync_single_range_for_cpu(rx_ring->dev,
1585                                               IXGBE_CB(skb)->dma,
1586                                               frag->page_offset,
1587                                               ixgbe_rx_bufsz(rx_ring),
1588                                               DMA_FROM_DEVICE);
1589         }
1590         IXGBE_CB(skb)->dma = 0;
1591 }
1592
1593 /**
1594  * ixgbe_cleanup_headers - Correct corrupted or empty headers
1595  * @rx_ring: rx descriptor ring packet is being transacted on
1596  * @rx_desc: pointer to the EOP Rx descriptor
1597  * @skb: pointer to current skb being fixed
1598  *
1599  * Check for corrupted packet headers caused by senders on the local L2
1600  * embedded NIC switch not setting up their Tx Descriptors right.  These
1601  * should be very rare.
1602  *
1603  * Also address the case where we are pulling data in on pages only
1604  * and as such no data is present in the skb header.
1605  *
1606  * In addition if skb is not at least 60 bytes we need to pad it so that
1607  * it is large enough to qualify as a valid Ethernet frame.
1608  *
1609  * Returns true if an error was encountered and skb was freed.
1610  **/
1611 static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
1612                                   union ixgbe_adv_rx_desc *rx_desc,
1613                                   struct sk_buff *skb)
1614 {
1615         struct net_device *netdev = rx_ring->netdev;
1616
1617         /* verify that the packet does not have any known errors */
1618         if (unlikely(ixgbe_test_staterr(rx_desc,
1619                                         IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
1620             !(netdev->features & NETIF_F_RXALL))) {
1621                 dev_kfree_skb_any(skb);
1622                 return true;
1623         }
1624
1625         /* place header in linear portion of buffer */
1626         if (skb_is_nonlinear(skb))
1627                 ixgbe_pull_tail(rx_ring, skb);
1628
1629 #ifdef IXGBE_FCOE
1630         /* do not attempt to pad FCoE Frames as this will disrupt DDP */
1631         if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))
1632                 return false;
1633
1634 #endif
1635         /* if skb_pad returns an error the skb was freed */
1636         if (unlikely(skb->len < 60)) {
1637                 int pad_len = 60 - skb->len;
1638
1639                 if (skb_pad(skb, pad_len))
1640                         return true;
1641                 __skb_put(skb, pad_len);
1642         }
1643
1644         return false;
1645 }
1646
1647 /**
1648  * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring
1649  * @rx_ring: rx descriptor ring to store buffers on
1650  * @old_buff: donor buffer to have page reused
1651  *
1652  * Synchronizes page for reuse by the adapter
1653  **/
1654 static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,
1655                                 struct ixgbe_rx_buffer *old_buff)
1656 {
1657         struct ixgbe_rx_buffer *new_buff;
1658         u16 nta = rx_ring->next_to_alloc;
1659
1660         new_buff = &rx_ring->rx_buffer_info[nta];
1661
1662         /* update, and store next to alloc */
1663         nta++;
1664         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1665
1666         /* transfer page from old buffer to new buffer */
1667         new_buff->page = old_buff->page;
1668         new_buff->dma = old_buff->dma;
1669         new_buff->page_offset = old_buff->page_offset;
1670
1671         /* sync the buffer for use by the device */
1672         dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
1673                                          new_buff->page_offset,
1674                                          ixgbe_rx_bufsz(rx_ring),
1675                                          DMA_FROM_DEVICE);
1676 }
1677
1678 /**
1679  * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
1680  * @rx_ring: rx descriptor ring to transact packets on
1681  * @rx_buffer: buffer containing page to add
1682  * @rx_desc: descriptor containing length of buffer written by hardware
1683  * @skb: sk_buff to place the data into
1684  *
1685  * This function will add the data contained in rx_buffer->page to the skb.
1686  * This is done either through a direct copy if the data in the buffer is
1687  * less than the skb header size, otherwise it will just attach the page as
1688  * a frag to the skb.
1689  *
1690  * The function will then update the page offset if necessary and return
1691  * true if the buffer can be reused by the adapter.
1692  **/
1693 static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
1694                               struct ixgbe_rx_buffer *rx_buffer,
1695                               union ixgbe_adv_rx_desc *rx_desc,
1696                               struct sk_buff *skb)
1697 {
1698         struct page *page = rx_buffer->page;
1699         unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
1700 #if (PAGE_SIZE < 8192)
1701         unsigned int truesize = ixgbe_rx_bufsz(rx_ring);
1702 #else
1703         unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
1704         unsigned int last_offset = ixgbe_rx_pg_size(rx_ring) -
1705                                    ixgbe_rx_bufsz(rx_ring);
1706 #endif
1707
1708         if ((size <= IXGBE_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
1709                 unsigned char *va = page_address(page) + rx_buffer->page_offset;
1710
1711                 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
1712
1713                 /* we can reuse buffer as-is, just make sure it is local */
1714                 if (likely(page_to_nid(page) == numa_node_id()))
1715                         return true;
1716
1717                 /* this page cannot be reused so discard it */
1718                 put_page(page);
1719                 return false;
1720         }
1721
1722         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
1723                         rx_buffer->page_offset, size, truesize);
1724
1725         /* avoid re-using remote pages */
1726         if (unlikely(page_to_nid(page) != numa_node_id()))
1727                 return false;
1728
1729 #if (PAGE_SIZE < 8192)
1730         /* if we are only owner of page we can reuse it */
1731         if (unlikely(page_count(page) != 1))
1732                 return false;
1733
1734         /* flip page offset to other buffer */
1735         rx_buffer->page_offset ^= truesize;
1736
1737         /*
1738          * since we are the only owner of the page and we need to
1739          * increment it, just set the value to 2 in order to avoid
1740          * an unecessary locked operation
1741          */
1742         atomic_set(&page->_count, 2);
1743 #else
1744         /* move offset up to the next cache line */
1745         rx_buffer->page_offset += truesize;
1746
1747         if (rx_buffer->page_offset > last_offset)
1748                 return false;
1749
1750         /* bump ref count on page before it is given to the stack */
1751         get_page(page);
1752 #endif
1753
1754         return true;
1755 }
1756
1757 static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring,
1758                                              union ixgbe_adv_rx_desc *rx_desc)
1759 {
1760         struct ixgbe_rx_buffer *rx_buffer;
1761         struct sk_buff *skb;
1762         struct page *page;
1763
1764         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1765         page = rx_buffer->page;
1766         prefetchw(page);
1767
1768         skb = rx_buffer->skb;
1769
1770         if (likely(!skb)) {
1771                 void *page_addr = page_address(page) +
1772                                   rx_buffer->page_offset;
1773
1774                 /* prefetch first cache line of first page */
1775                 prefetch(page_addr);
1776 #if L1_CACHE_BYTES < 128
1777                 prefetch(page_addr + L1_CACHE_BYTES);
1778 #endif
1779
1780                 /* allocate a skb to store the frags */
1781                 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1782                                                 IXGBE_RX_HDR_SIZE);
1783                 if (unlikely(!skb)) {
1784                         rx_ring->rx_stats.alloc_rx_buff_failed++;
1785                         return NULL;
1786                 }
1787
1788                 /*
1789                  * we will be copying header into skb->data in
1790                  * pskb_may_pull so it is in our interest to prefetch
1791                  * it now to avoid a possible cache miss
1792                  */
1793                 prefetchw(skb->data);
1794
1795                 /*
1796                  * Delay unmapping of the first packet. It carries the
1797                  * header information, HW may still access the header
1798                  * after the writeback.  Only unmap it when EOP is
1799                  * reached
1800                  */
1801                 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
1802                         goto dma_sync;
1803
1804                 IXGBE_CB(skb)->dma = rx_buffer->dma;
1805         } else {
1806                 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))
1807                         ixgbe_dma_sync_frag(rx_ring, skb);
1808
1809 dma_sync:
1810                 /* we are reusing so sync this buffer for CPU use */
1811                 dma_sync_single_range_for_cpu(rx_ring->dev,
1812                                               rx_buffer->dma,
1813                                               rx_buffer->page_offset,
1814                                               ixgbe_rx_bufsz(rx_ring),
1815                                               DMA_FROM_DEVICE);
1816         }
1817
1818         /* pull page into skb */
1819         if (ixgbe_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
1820                 /* hand second half of page back to the ring */
1821                 ixgbe_reuse_rx_page(rx_ring, rx_buffer);
1822         } else if (IXGBE_CB(skb)->dma == rx_buffer->dma) {
1823                 /* the page has been released from the ring */
1824                 IXGBE_CB(skb)->page_released = true;
1825         } else {
1826                 /* we are not reusing the buffer so unmap it */
1827                 dma_unmap_page(rx_ring->dev, rx_buffer->dma,
1828                                ixgbe_rx_pg_size(rx_ring),
1829                                DMA_FROM_DEVICE);
1830         }
1831
1832         /* clear contents of buffer_info */
1833         rx_buffer->skb = NULL;
1834         rx_buffer->dma = 0;
1835         rx_buffer->page = NULL;
1836
1837         return skb;
1838 }
1839
1840 /**
1841  * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
1842  * @q_vector: structure containing interrupt and ring information
1843  * @rx_ring: rx descriptor ring to transact packets on
1844  * @budget: Total limit on number of packets to process
1845  *
1846  * This function provides a "bounce buffer" approach to Rx interrupt
1847  * processing.  The advantage to this is that on systems that have
1848  * expensive overhead for IOMMU access this provides a means of avoiding
1849  * it by maintaining the mapping of the page to the syste.
1850  *
1851  * Returns true if all work is completed without reaching budget
1852  **/
1853 static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1854                                struct ixgbe_ring *rx_ring,
1855                                const int budget)
1856 {
1857         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1858 #ifdef IXGBE_FCOE
1859         struct ixgbe_adapter *adapter = q_vector->adapter;
1860         int ddp_bytes;
1861         unsigned int mss = 0;
1862 #endif /* IXGBE_FCOE */
1863         u16 cleaned_count = ixgbe_desc_unused(rx_ring);
1864
1865         do {
1866                 union ixgbe_adv_rx_desc *rx_desc;
1867                 struct sk_buff *skb;
1868
1869                 /* return some buffers to hardware, one at a time is too slow */
1870                 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1871                         ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1872                         cleaned_count = 0;
1873                 }
1874
1875                 rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);
1876
1877                 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
1878                         break;
1879
1880                 /*
1881                  * This memory barrier is needed to keep us from reading
1882                  * any other fields out of the rx_desc until we know the
1883                  * RXD_STAT_DD bit is set
1884                  */
1885                 rmb();
1886
1887                 /* retrieve a buffer from the ring */
1888                 skb = ixgbe_fetch_rx_buffer(rx_ring, rx_desc);
1889
1890                 /* exit if we failed to retrieve a buffer */
1891                 if (!skb)
1892                         break;
1893
1894                 cleaned_count++;
1895
1896                 /* place incomplete frames back on ring for completion */
1897                 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))
1898                         continue;
1899
1900                 /* verify the packet layout is correct */
1901                 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))
1902                         continue;
1903
1904                 /* probably a little skewed due to removing CRC */
1905                 total_rx_bytes += skb->len;
1906
1907                 /* populate checksum, timestamp, VLAN, and protocol */
1908                 ixgbe_process_skb_fields(rx_ring, rx_desc, skb);
1909
1910 #ifdef IXGBE_FCOE
1911                 /* if ddp, not passing to ULD unless for FCP_RSP or error */
1912                 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {
1913                         ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
1914                         /* include DDPed FCoE data */
1915                         if (ddp_bytes > 0) {
1916                                 if (!mss) {
1917                                         mss = rx_ring->netdev->mtu -
1918                                                 sizeof(struct fcoe_hdr) -
1919                                                 sizeof(struct fc_frame_header) -
1920                                                 sizeof(struct fcoe_crc_eof);
1921                                         if (mss > 512)
1922                                                 mss &= ~511;
1923                                 }
1924                                 total_rx_bytes += ddp_bytes;
1925                                 total_rx_packets += DIV_ROUND_UP(ddp_bytes,
1926                                                                  mss);
1927                         }
1928                         if (!ddp_bytes) {
1929                                 dev_kfree_skb_any(skb);
1930                                 continue;
1931                         }
1932                 }
1933
1934 #endif /* IXGBE_FCOE */
1935                 ixgbe_rx_skb(q_vector, skb);
1936
1937                 /* update budget accounting */
1938                 total_rx_packets++;
1939         } while (likely(total_rx_packets < budget));
1940
1941         u64_stats_update_begin(&rx_ring->syncp);
1942         rx_ring->stats.packets += total_rx_packets;
1943         rx_ring->stats.bytes += total_rx_bytes;
1944         u64_stats_update_end(&rx_ring->syncp);
1945         q_vector->rx.total_packets += total_rx_packets;
1946         q_vector->rx.total_bytes += total_rx_bytes;
1947
1948         if (cleaned_count)
1949                 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1950
1951         return (total_rx_packets < budget);
1952 }
1953
1954 /**
1955  * ixgbe_configure_msix - Configure MSI-X hardware
1956  * @adapter: board private structure
1957  *
1958  * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
1959  * interrupts.
1960  **/
1961 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1962 {
1963         struct ixgbe_q_vector *q_vector;
1964         int v_idx;
1965         u32 mask;
1966
1967         /* Populate MSIX to EITR Select */
1968         if (adapter->num_vfs > 32) {
1969                 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
1970                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
1971         }
1972
1973         /*
1974          * Populate the IVAR table and set the ITR values to the
1975          * corresponding register.
1976          */
1977         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1978                 struct ixgbe_ring *ring;
1979                 q_vector = adapter->q_vector[v_idx];
1980
1981                 ixgbe_for_each_ring(ring, q_vector->rx)
1982                         ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1983
1984                 ixgbe_for_each_ring(ring, q_vector->tx)
1985                         ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1986
1987                 ixgbe_write_eitr(q_vector);
1988         }
1989
1990         switch (adapter->hw.mac.type) {
1991         case ixgbe_mac_82598EB:
1992                 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1993                                v_idx);
1994                 break;
1995         case ixgbe_mac_82599EB:
1996         case ixgbe_mac_X540:
1997                 ixgbe_set_ivar(adapter, -1, 1, v_idx);
1998                 break;
1999         default:
2000                 break;
2001         }
2002         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
2003
2004         /* set up to autoclear timer, and the vectors */
2005         mask = IXGBE_EIMS_ENABLE_MASK;
2006         mask &= ~(IXGBE_EIMS_OTHER |
2007                   IXGBE_EIMS_MAILBOX |
2008                   IXGBE_EIMS_LSC);
2009
2010         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
2011 }
2012
2013 enum latency_range {
2014         lowest_latency = 0,
2015         low_latency = 1,
2016         bulk_latency = 2,
2017         latency_invalid = 255
2018 };
2019
2020 /**
2021  * ixgbe_update_itr - update the dynamic ITR value based on statistics
2022  * @q_vector: structure containing interrupt and ring information
2023  * @ring_container: structure containing ring performance data
2024  *
2025  *      Stores a new ITR value based on packets and byte
2026  *      counts during the last interrupt.  The advantage of per interrupt
2027  *      computation is faster updates and more accurate ITR for the current
2028  *      traffic pattern.  Constants in this function were computed
2029  *      based on theoretical maximum wire speed and thresholds were set based
2030  *      on testing data as well as attempting to minimize response time
2031  *      while increasing bulk throughput.
2032  *      this functionality is controlled by the InterruptThrottleRate module
2033  *      parameter (see ixgbe_param.c)
2034  **/
2035 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
2036                              struct ixgbe_ring_container *ring_container)
2037 {
2038         int bytes = ring_container->total_bytes;
2039         int packets = ring_container->total_packets;
2040         u32 timepassed_us;
2041         u64 bytes_perint;
2042         u8 itr_setting = ring_container->itr;
2043
2044         if (packets == 0)
2045                 return;
2046
2047         /* simple throttlerate management
2048          *   0-10MB/s   lowest (100000 ints/s)
2049          *  10-20MB/s   low    (20000 ints/s)
2050          *  20-1249MB/s bulk   (8000 ints/s)
2051          */
2052         /* what was last interrupt timeslice? */
2053         timepassed_us = q_vector->itr >> 2;
2054         bytes_perint = bytes / timepassed_us; /* bytes/usec */
2055
2056         switch (itr_setting) {
2057         case lowest_latency:
2058                 if (bytes_perint > 10)
2059                         itr_setting = low_latency;
2060                 break;
2061         case low_latency:
2062                 if (bytes_perint > 20)
2063                         itr_setting = bulk_latency;
2064                 else if (bytes_perint <= 10)
2065                         itr_setting = lowest_latency;
2066                 break;
2067         case bulk_latency:
2068                 if (bytes_perint <= 20)
2069                         itr_setting = low_latency;
2070                 break;
2071         }
2072
2073         /* clear work counters since we have the values we need */
2074         ring_container->total_bytes = 0;
2075         ring_container->total_packets = 0;
2076
2077         /* write updated itr to ring container */
2078         ring_container->itr = itr_setting;
2079 }
2080
2081 /**
2082  * ixgbe_write_eitr - write EITR register in hardware specific way
2083  * @q_vector: structure containing interrupt and ring information
2084  *
2085  * This function is made to be called by ethtool and by the driver
2086  * when it needs to update EITR registers at runtime.  Hardware
2087  * specific quirks/differences are taken care of here.
2088  */
2089 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
2090 {
2091         struct ixgbe_adapter *adapter = q_vector->adapter;
2092         struct ixgbe_hw *hw = &adapter->hw;
2093         int v_idx = q_vector->v_idx;
2094         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
2095
2096         switch (adapter->hw.mac.type) {
2097         case ixgbe_mac_82598EB:
2098                 /* must write high and low 16 bits to reset counter */
2099                 itr_reg |= (itr_reg << 16);
2100                 break;
2101         case ixgbe_mac_82599EB:
2102         case ixgbe_mac_X540:
2103                 /*
2104                  * set the WDIS bit to not clear the timer bits and cause an
2105                  * immediate assertion of the interrupt
2106                  */
2107                 itr_reg |= IXGBE_EITR_CNT_WDIS;
2108                 break;
2109         default:
2110                 break;
2111         }
2112         IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
2113 }
2114
2115 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
2116 {
2117         u32 new_itr = q_vector->itr;
2118         u8 current_itr;
2119
2120         ixgbe_update_itr(q_vector, &q_vector->tx);
2121         ixgbe_update_itr(q_vector, &q_vector->rx);
2122
2123         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
2124
2125         switch (current_itr) {
2126         /* counts and packets in update_itr are dependent on these numbers */
2127         case lowest_latency:
2128                 new_itr = IXGBE_100K_ITR;
2129                 break;
2130         case low_latency:
2131                 new_itr = IXGBE_20K_ITR;
2132                 break;
2133         case bulk_latency:
2134                 new_itr = IXGBE_8K_ITR;
2135                 break;
2136         default:
2137                 break;
2138         }
2139
2140         if (new_itr != q_vector->itr) {
2141                 /* do an exponential smoothing */
2142                 new_itr = (10 * new_itr * q_vector->itr) /
2143                           ((9 * new_itr) + q_vector->itr);
2144
2145                 /* save the algorithm value here */
2146                 q_vector->itr = new_itr;
2147
2148                 ixgbe_write_eitr(q_vector);
2149         }
2150 }
2151
2152 /**
2153  * ixgbe_check_overtemp_subtask - check for over temperature
2154  * @adapter: pointer to adapter
2155  **/
2156 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
2157 {
2158         struct ixgbe_hw *hw = &adapter->hw;
2159         u32 eicr = adapter->interrupt_event;
2160
2161         if (test_bit(__IXGBE_DOWN, &adapter->state))
2162                 return;
2163
2164         if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
2165             !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
2166                 return;
2167
2168         adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2169
2170         switch (hw->device_id) {
2171         case IXGBE_DEV_ID_82599_T3_LOM:
2172                 /*
2173                  * Since the warning interrupt is for both ports
2174                  * we don't have to check if:
2175                  *  - This interrupt wasn't for our port.
2176                  *  - We may have missed the interrupt so always have to
2177                  *    check if we  got a LSC
2178                  */
2179                 if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
2180                     !(eicr & IXGBE_EICR_LSC))
2181                         return;
2182
2183                 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
2184                         u32 autoneg;
2185                         bool link_up = false;
2186
2187                         hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
2188
2189                         if (link_up)
2190                                 return;
2191                 }
2192
2193                 /* Check if this is not due to overtemp */
2194                 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
2195                         return;
2196
2197                 break;
2198         default:
2199                 if (!(eicr & IXGBE_EICR_GPI_SDP0))
2200                         return;
2201                 break;
2202         }
2203         e_crit(drv,
2204                "Network adapter has been stopped because it has over heated. "
2205                "Restart the computer. If the problem persists, "
2206                "power off the system and replace the adapter\n");
2207
2208         adapter->interrupt_event = 0;
2209 }
2210
2211 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
2212 {
2213         struct ixgbe_hw *hw = &adapter->hw;
2214
2215         if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
2216             (eicr & IXGBE_EICR_GPI_SDP1)) {
2217                 e_crit(probe, "Fan has stopped, replace the adapter\n");
2218                 /* write to clear the interrupt */
2219                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
2220         }
2221 }
2222
2223 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
2224 {
2225         if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
2226                 return;
2227
2228         switch (adapter->hw.mac.type) {
2229         case ixgbe_mac_82599EB:
2230                 /*
2231                  * Need to check link state so complete overtemp check
2232                  * on service task
2233                  */
2234                 if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
2235                     (!test_bit(__IXGBE_DOWN, &adapter->state))) {
2236                         adapter->interrupt_event = eicr;
2237                         adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2238                         ixgbe_service_event_schedule(adapter);
2239                         return;
2240                 }
2241                 return;
2242         case ixgbe_mac_X540:
2243                 if (!(eicr & IXGBE_EICR_TS))
2244                         return;
2245                 break;
2246         default:
2247                 return;
2248         }
2249
2250         e_crit(drv,
2251                "Network adapter has been stopped because it has over heated. "
2252                "Restart the computer. If the problem persists, "
2253                "power off the system and replace the adapter\n");
2254 }
2255
2256 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
2257 {
2258         struct ixgbe_hw *hw = &adapter->hw;
2259
2260         if (eicr & IXGBE_EICR_GPI_SDP2) {
2261                 /* Clear the interrupt */
2262                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
2263                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2264                         adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
2265                         ixgbe_service_event_schedule(adapter);
2266                 }
2267         }
2268
2269         if (eicr & IXGBE_EICR_GPI_SDP1) {
2270                 /* Clear the interrupt */
2271                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
2272                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2273                         adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
2274                         ixgbe_service_event_schedule(adapter);
2275                 }
2276         }
2277 }
2278
2279 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
2280 {
2281         struct ixgbe_hw *hw = &adapter->hw;
2282
2283         adapter->lsc_int++;
2284         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2285         adapter->link_check_timeout = jiffies;
2286         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2287                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
2288                 IXGBE_WRITE_FLUSH(hw);
2289                 ixgbe_service_event_schedule(adapter);
2290         }
2291 }
2292
2293 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
2294                                            u64 qmask)
2295 {
2296         u32 mask;
2297         struct ixgbe_hw *hw = &adapter->hw;
2298
2299         switch (hw->mac.type) {
2300         case ixgbe_mac_82598EB:
2301                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
2302                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
2303                 break;
2304         case ixgbe_mac_82599EB:
2305         case ixgbe_mac_X540:
2306                 mask = (qmask & 0xFFFFFFFF);
2307                 if (mask)
2308                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
2309                 mask = (qmask >> 32);
2310                 if (mask)
2311                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
2312                 break;
2313         default:
2314                 break;
2315         }
2316         /* skip the flush */
2317 }
2318
2319 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
2320                                             u64 qmask)
2321 {
2322         u32 mask;
2323         struct ixgbe_hw *hw = &adapter->hw;
2324
2325         switch (hw->mac.type) {
2326         case ixgbe_mac_82598EB:
2327                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
2328                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
2329                 break;
2330         case ixgbe_mac_82599EB:
2331         case ixgbe_mac_X540:
2332                 mask = (qmask & 0xFFFFFFFF);
2333                 if (mask)
2334                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
2335                 mask = (qmask >> 32);
2336                 if (mask)
2337                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
2338                 break;
2339         default:
2340                 break;
2341         }
2342         /* skip the flush */
2343 }
2344
2345 /**
2346  * ixgbe_irq_enable - Enable default interrupt generation settings
2347  * @adapter: board private structure
2348  **/
2349 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
2350                                     bool flush)
2351 {
2352         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
2353
2354         /* don't reenable LSC while waiting for link */
2355         if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
2356                 mask &= ~IXGBE_EIMS_LSC;
2357
2358         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
2359                 switch (adapter->hw.mac.type) {
2360                 case ixgbe_mac_82599EB:
2361                         mask |= IXGBE_EIMS_GPI_SDP0;
2362                         break;
2363                 case ixgbe_mac_X540:
2364                         mask |= IXGBE_EIMS_TS;
2365                         break;
2366                 default:
2367                         break;
2368                 }
2369         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
2370                 mask |= IXGBE_EIMS_GPI_SDP1;
2371         switch (adapter->hw.mac.type) {
2372         case ixgbe_mac_82599EB:
2373                 mask |= IXGBE_EIMS_GPI_SDP1;
2374                 mask |= IXGBE_EIMS_GPI_SDP2;
2375         case ixgbe_mac_X540:
2376                 mask |= IXGBE_EIMS_ECC;
2377                 mask |= IXGBE_EIMS_MAILBOX;
2378                 break;
2379         default:
2380                 break;
2381         }
2382
2383         if (adapter->hw.mac.type == ixgbe_mac_X540)
2384                 mask |= IXGBE_EIMS_TIMESYNC;
2385
2386         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2387             !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
2388                 mask |= IXGBE_EIMS_FLOW_DIR;
2389
2390         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2391         if (queues)
2392                 ixgbe_irq_enable_queues(adapter, ~0);
2393         if (flush)
2394                 IXGBE_WRITE_FLUSH(&adapter->hw);
2395 }
2396
2397 static irqreturn_t ixgbe_msix_other(int irq, void *data)
2398 {
2399         struct ixgbe_adapter *adapter = data;
2400         struct ixgbe_hw *hw = &adapter->hw;
2401         u32 eicr;
2402
2403         /*
2404          * Workaround for Silicon errata.  Use clear-by-write instead
2405          * of clear-by-read.  Reading with EICS will return the
2406          * interrupt causes without clearing, which later be done
2407          * with the write to EICR.
2408          */
2409         eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
2410         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
2411
2412         if (eicr & IXGBE_EICR_LSC)
2413                 ixgbe_check_lsc(adapter);
2414
2415         if (eicr & IXGBE_EICR_MAILBOX)
2416                 ixgbe_msg_task(adapter);
2417
2418         switch (hw->mac.type) {
2419         case ixgbe_mac_82599EB:
2420         case ixgbe_mac_X540:
2421                 if (eicr & IXGBE_EICR_ECC)
2422                         e_info(link, "Received unrecoverable ECC Err, please "
2423                                "reboot\n");
2424                 /* Handle Flow Director Full threshold interrupt */
2425                 if (eicr & IXGBE_EICR_FLOW_DIR) {
2426                         int reinit_count = 0;
2427                         int i;
2428                         for (i = 0; i < adapter->num_tx_queues; i++) {
2429                                 struct ixgbe_ring *ring = adapter->tx_ring[i];
2430                                 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
2431                                                        &ring->state))
2432                                         reinit_count++;
2433                         }
2434                         if (reinit_count) {
2435                                 /* no more flow director interrupts until after init */
2436                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
2437                                 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
2438                                 ixgbe_service_event_schedule(adapter);
2439                         }
2440                 }
2441                 ixgbe_check_sfp_event(adapter, eicr);
2442                 ixgbe_check_overtemp_event(adapter, eicr);
2443                 break;
2444         default:
2445                 break;
2446         }
2447
2448         ixgbe_check_fan_failure(adapter, eicr);
2449
2450         if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
2451                 ixgbe_ptp_check_pps_event(adapter, eicr);
2452
2453         /* re-enable the original interrupt state, no lsc, no queues */
2454         if (!test_bit(__IXGBE_DOWN, &adapter->state))
2455                 ixgbe_irq_enable(adapter, false, false);
2456
2457         return IRQ_HANDLED;
2458 }
2459
2460 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
2461 {
2462         struct ixgbe_q_vector *q_vector = data;
2463
2464         /* EIAM disabled interrupts (on this vector) for us */
2465
2466         if (q_vector->rx.ring || q_vector->tx.ring)
2467                 napi_schedule(&q_vector->napi);
2468
2469         return IRQ_HANDLED;
2470 }
2471
2472 /**
2473  * ixgbe_poll - NAPI Rx polling callback
2474  * @napi: structure for representing this polling device
2475  * @budget: how many packets driver is allowed to clean
2476  *
2477  * This function is used for legacy and MSI, NAPI mode
2478  **/
2479 int ixgbe_poll(struct napi_struct *napi, int budget)
2480 {
2481         struct ixgbe_q_vector *q_vector =
2482                                 container_of(napi, struct ixgbe_q_vector, napi);
2483         struct ixgbe_adapter *adapter = q_vector->adapter;
2484         struct ixgbe_ring *ring;
2485         int per_ring_budget;
2486         bool clean_complete = true;
2487
2488 #ifdef CONFIG_IXGBE_DCA
2489         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2490                 ixgbe_update_dca(q_vector);
2491 #endif
2492
2493         ixgbe_for_each_ring(ring, q_vector->tx)
2494                 clean_complete &= !!ixgbe_clean_tx_irq(q_vector, ring);
2495
2496         /* attempt to distribute budget to each queue fairly, but don't allow
2497          * the budget to go below 1 because we'll exit polling */
2498         if (q_vector->rx.count > 1)
2499                 per_ring_budget = max(budget/q_vector->rx.count, 1);
2500         else
2501                 per_ring_budget = budget;
2502
2503         ixgbe_for_each_ring(ring, q_vector->rx)
2504                 clean_complete &= ixgbe_clean_rx_irq(q_vector, ring,
2505                                                      per_ring_budget);
2506
2507         /* If all work not completed, return budget and keep polling */
2508         if (!clean_complete)
2509                 return budget;
2510
2511         /* all work done, exit the polling mode */
2512         napi_complete(napi);
2513         if (adapter->rx_itr_setting & 1)
2514                 ixgbe_set_itr(q_vector);
2515         if (!test_bit(__IXGBE_DOWN, &adapter->state))
2516                 ixgbe_irq_enable_queues(adapter, ((u64)1 << q_vector->v_idx));
2517
2518         return 0;
2519 }
2520
2521 /**
2522  * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2523  * @adapter: board private structure
2524  *
2525  * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2526  * interrupts from the kernel.
2527  **/
2528 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2529 {
2530         struct net_device *netdev = adapter->netdev;
2531         int vector, err;
2532         int ri = 0, ti = 0;
2533
2534         for (vector = 0; vector < adapter->num_q_vectors; vector++) {
2535                 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2536                 struct msix_entry *entry = &adapter->msix_entries[vector];
2537
2538                 if (q_vector->tx.ring && q_vector->rx.ring) {
2539                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2540                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
2541                         ti++;
2542                 } else if (q_vector->rx.ring) {
2543                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2544                                  "%s-%s-%d", netdev->name, "rx", ri++);
2545                 } else if (q_vector->tx.ring) {
2546                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2547                                  "%s-%s-%d", netdev->name, "tx", ti++);
2548                 } else {
2549                         /* skip this unused q_vector */
2550                         continue;
2551                 }
2552                 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2553                                   q_vector->name, q_vector);
2554                 if (err) {
2555                         e_err(probe, "request_irq failed for MSIX interrupt "
2556                               "Error: %d\n", err);
2557                         goto free_queue_irqs;
2558                 }
2559                 /* If Flow Director is enabled, set interrupt affinity */
2560                 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2561                         /* assign the mask for this irq */
2562                         irq_set_affinity_hint(entry->vector,
2563                                               &q_vector->affinity_mask);
2564                 }
2565         }
2566
2567         err = request_irq(adapter->msix_entries[vector].vector,
2568                           ixgbe_msix_other, 0, netdev->name, adapter);
2569         if (err) {
2570                 e_err(probe, "request_irq for msix_other failed: %d\n", err);
2571                 goto free_queue_irqs;
2572         }
2573
2574         return 0;
2575
2576 free_queue_irqs:
2577         while (vector) {
2578                 vector--;
2579                 irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2580                                       NULL);
2581                 free_irq(adapter->msix_entries[vector].vector,
2582                          adapter->q_vector[vector]);
2583         }
2584         adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2585         pci_disable_msix(adapter->pdev);
2586         kfree(adapter->msix_entries);
2587         adapter->msix_entries = NULL;
2588         return err;
2589 }
2590
2591 /**
2592  * ixgbe_intr - legacy mode Interrupt Handler
2593  * @irq: interrupt number
2594  * @data: pointer to a network interface device structure
2595  **/
2596 static irqreturn_t ixgbe_intr(int irq, void *data)
2597 {
2598         struct ixgbe_adapter *adapter = data;
2599         struct ixgbe_hw *hw = &adapter->hw;
2600         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2601         u32 eicr;
2602
2603         /*
2604          * Workaround for silicon errata #26 on 82598.  Mask the interrupt
2605          * before the read of EICR.
2606          */
2607         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2608
2609         /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2610          * therefore no explicit interrupt disable is necessary */
2611         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2612         if (!eicr) {
2613                 /*
2614                  * shared interrupt alert!
2615                  * make sure interrupts are enabled because the read will
2616                  * have disabled interrupts due to EIAM
2617                  * finish the workaround of silicon errata on 82598.  Unmask
2618                  * the interrupt that we masked before the EICR read.
2619                  */
2620                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2621                         ixgbe_irq_enable(adapter, true, true);
2622                 return IRQ_NONE;        /* Not our interrupt */
2623         }
2624
2625         if (eicr & IXGBE_EICR_LSC)
2626                 ixgbe_check_lsc(adapter);
2627
2628         switch (hw->mac.type) {
2629         case ixgbe_mac_82599EB:
2630                 ixgbe_check_sfp_event(adapter, eicr);
2631                 /* Fall through */
2632         case ixgbe_mac_X540:
2633                 if (eicr & IXGBE_EICR_ECC)
2634                         e_info(link, "Received unrecoverable ECC err, please "
2635                                      "reboot\n");
2636                 ixgbe_check_overtemp_event(adapter, eicr);
2637                 break;
2638         default:
2639                 break;
2640         }
2641
2642         ixgbe_check_fan_failure(adapter, eicr);
2643         if (unlikely(eicr & IXGBE_EICR_TIMESYNC))
2644                 ixgbe_ptp_check_pps_event(adapter, eicr);
2645
2646         /* would disable interrupts here but EIAM disabled it */
2647         napi_schedule(&q_vector->napi);
2648
2649         /*
2650          * re-enable link(maybe) and non-queue interrupts, no flush.
2651          * ixgbe_poll will re-enable the queue interrupts
2652          */
2653         if (!test_bit(__IXGBE_DOWN, &adapter->state))
2654                 ixgbe_irq_enable(adapter, false, false);
2655
2656         return IRQ_HANDLED;
2657 }
2658
2659 /**
2660  * ixgbe_request_irq - initialize interrupts
2661  * @adapter: board private structure
2662  *
2663  * Attempts to configure interrupts using the best available
2664  * capabilities of the hardware and kernel.
2665  **/
2666 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2667 {
2668         struct net_device *netdev = adapter->netdev;
2669         int err;
2670
2671         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
2672                 err = ixgbe_request_msix_irqs(adapter);
2673         else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
2674                 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2675                                   netdev->name, adapter);
2676         else
2677                 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2678                                   netdev->name, adapter);
2679
2680         if (err)
2681                 e_err(probe, "request_irq failed, Error %d\n", err);
2682
2683         return err;
2684 }
2685
2686 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2687 {
2688         int vector;
2689
2690         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
2691                 free_irq(adapter->pdev->irq, adapter);
2692                 return;
2693         }
2694
2695         for (vector = 0; vector < adapter->num_q_vectors; vector++) {
2696                 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2697                 struct msix_entry *entry = &adapter->msix_entries[vector];
2698
2699                 /* free only the irqs that were actually requested */
2700                 if (!q_vector->rx.ring && !q_vector->tx.ring)
2701                         continue;
2702
2703                 /* clear the affinity_mask in the IRQ descriptor */
2704                 irq_set_affinity_hint(entry->vector, NULL);
2705
2706                 free_irq(entry->vector, q_vector);
2707         }
2708
2709         free_irq(adapter->msix_entries[vector++].vector, adapter);
2710 }
2711
2712 /**
2713  * ixgbe_irq_disable - Mask off interrupt generation on the NIC
2714  * @adapter: board private structure
2715  **/
2716 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2717 {
2718         switch (adapter->hw.mac.type) {
2719         case ixgbe_mac_82598EB:
2720                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2721                 break;
2722         case ixgbe_mac_82599EB:
2723         case ixgbe_mac_X540:
2724                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2725                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2726                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2727                 break;
2728         default:
2729                 break;
2730         }
2731         IXGBE_WRITE_FLUSH(&adapter->hw);
2732         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2733                 int vector;
2734
2735                 for (vector = 0; vector < adapter->num_q_vectors; vector++)
2736                         synchronize_irq(adapter->msix_entries[vector].vector);
2737
2738                 synchronize_irq(adapter->msix_entries[vector++].vector);
2739         } else {
2740                 synchronize_irq(adapter->pdev->irq);
2741         }
2742 }
2743
2744 /**
2745  * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
2746  *
2747  **/
2748 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2749 {
2750         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2751
2752         ixgbe_write_eitr(q_vector);
2753
2754         ixgbe_set_ivar(adapter, 0, 0, 0);
2755         ixgbe_set_ivar(adapter, 1, 0, 0);
2756
2757         e_info(hw, "Legacy interrupt IVAR setup done\n");
2758 }
2759
2760 /**
2761  * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
2762  * @adapter: board private structure
2763  * @ring: structure containing ring specific data
2764  *
2765  * Configure the Tx descriptor ring after a reset.
2766  **/
2767 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
2768                              struct ixgbe_ring *ring)
2769 {
2770         struct ixgbe_hw *hw = &adapter->hw;
2771         u64 tdba = ring->dma;
2772         int wait_loop = 10;
2773         u32 txdctl = IXGBE_TXDCTL_ENABLE;
2774         u8 reg_idx = ring->reg_idx;
2775
2776         /* disable queue to avoid issues while updating state */
2777         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
2778         IXGBE_WRITE_FLUSH(hw);
2779
2780         IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2781                         (tdba & DMA_BIT_MASK(32)));
2782         IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2783         IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2784                         ring->count * sizeof(union ixgbe_adv_tx_desc));
2785         IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2786         IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2787         ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2788
2789         /*
2790          * set WTHRESH to encourage burst writeback, it should not be set
2791          * higher than 1 when ITR is 0 as it could cause false TX hangs
2792          *
2793          * In order to avoid issues WTHRESH + PTHRESH should always be equal
2794          * to or less than the number of on chip descriptors, which is
2795          * currently 40.
2796          */
2797         if (!ring->q_vector || (ring->q_vector->itr < 8))
2798                 txdctl |= (1 << 16);    /* WTHRESH = 1 */
2799         else
2800                 txdctl |= (8 << 16);    /* WTHRESH = 8 */
2801
2802         /*
2803          * Setting PTHRESH to 32 both improves performance
2804          * and avoids a TX hang with DFP enabled
2805          */
2806         txdctl |= (1 << 8) |    /* HTHRESH = 1 */
2807                    32;          /* PTHRESH = 32 */
2808
2809         /* reinitialize flowdirector state */
2810         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2811                 ring->atr_sample_rate = adapter->atr_sample_rate;
2812                 ring->atr_count = 0;
2813                 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
2814         } else {
2815                 ring->atr_sample_rate = 0;
2816         }
2817
2818         clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
2819
2820         /* enable queue */
2821         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2822
2823         /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2824         if (hw->mac.type == ixgbe_mac_82598EB &&
2825             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2826                 return;
2827
2828         /* poll to verify queue is enabled */
2829         do {
2830                 usleep_range(1000, 2000);
2831                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2832         } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2833         if (!wait_loop)
2834                 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2835 }
2836
2837 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2838 {
2839         struct ixgbe_hw *hw = &adapter->hw;
2840         u32 rttdcs, mtqc;
2841         u8 tcs = netdev_get_num_tc(adapter->netdev);
2842
2843         if (hw->mac.type == ixgbe_mac_82598EB)
2844                 return;
2845
2846         /* disable the arbiter while setting MTQC */
2847         rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2848         rttdcs |= IXGBE_RTTDCS_ARBDIS;
2849         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2850
2851         /* set transmit pool layout */
2852         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2853                 mtqc = IXGBE_MTQC_VT_ENA;
2854                 if (tcs > 4)
2855                         mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2856                 else if (tcs > 1)
2857                         mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2858                 else if (adapter->ring_feature[RING_F_RSS].indices == 4)
2859                         mtqc |= IXGBE_MTQC_32VF;
2860                 else
2861                         mtqc |= IXGBE_MTQC_64VF;
2862         } else {
2863                 if (tcs > 4)
2864                         mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2865                 else if (tcs > 1)
2866                         mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2867                 else
2868                         mtqc = IXGBE_MTQC_64Q_1PB;
2869         }
2870
2871         IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);
2872
2873         /* Enable Security TX Buffer IFG for multiple pb */
2874         if (tcs) {
2875                 u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2876                 sectx |= IXGBE_SECTX_DCB;
2877                 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);
2878         }
2879
2880         /* re-enable the arbiter */
2881         rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2882         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2883 }
2884
2885 /**
2886  * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
2887  * @adapter: board private structure
2888  *
2889  * Configure the Tx unit of the MAC after a reset.
2890  **/
2891 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2892 {
2893         struct ixgbe_hw *hw = &adapter->hw;
2894         u32 dmatxctl;
2895         u32 i;
2896
2897         ixgbe_setup_mtqc(adapter);
2898
2899         if (hw->mac.type != ixgbe_mac_82598EB) {
2900                 /* DMATXCTL.EN must be before Tx queues are enabled */
2901                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2902                 dmatxctl |= IXGBE_DMATXCTL_TE;
2903                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2904         }
2905
2906         /* Setup the HW Tx Head and Tail descriptor pointers */
2907         for (i = 0; i < adapter->num_tx_queues; i++)
2908                 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2909 }
2910
2911 static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,
2912                                  struct ixgbe_ring *ring)
2913 {
2914         struct ixgbe_hw *hw = &adapter->hw;
2915         u8 reg_idx = ring->reg_idx;
2916         u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
2917
2918         srrctl |= IXGBE_SRRCTL_DROP_EN;
2919
2920         IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
2921 }
2922
2923 static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,
2924                                   struct ixgbe_ring *ring)
2925 {
2926         struct ixgbe_hw *hw = &adapter->hw;
2927         u8 reg_idx = ring->reg_idx;
2928         u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));
2929
2930         srrctl &= ~IXGBE_SRRCTL_DROP_EN;
2931
2932         IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
2933 }
2934
2935 #ifdef CONFIG_IXGBE_DCB
2936 void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
2937 #else
2938 static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)
2939 #endif
2940 {
2941         int i;
2942         bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;
2943
2944         if (adapter->ixgbe_ieee_pfc)
2945                 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);
2946
2947         /*
2948          * We should set the drop enable bit if:
2949          *  SR-IOV is enabled
2950          *   or
2951          *  Number of Rx queues > 1 and flow control is disabled
2952          *
2953          *  This allows us to avoid head of line blocking for security
2954          *  and performance reasons.
2955          */
2956         if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&
2957             !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {
2958                 for (i = 0; i < adapter->num_rx_queues; i++)
2959                         ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);
2960         } else {
2961                 for (i = 0; i < adapter->num_rx_queues; i++)
2962                         ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);
2963         }
2964 }
2965
2966 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2967
2968 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2969                                    struct ixgbe_ring *rx_ring)
2970 {
2971         struct ixgbe_hw *hw = &adapter->hw;
2972         u32 srrctl;
2973         u8 reg_idx = rx_ring->reg_idx;
2974
2975         if (hw->mac.type == ixgbe_mac_82598EB) {
2976                 u16 mask = adapter->ring_feature[RING_F_RSS].mask;
2977
2978                 /*
2979                  * if VMDq is not active we must program one srrctl register
2980                  * per RSS queue since we have enabled RDRXCTL.MVMEN
2981                  */
2982                 reg_idx &= mask;
2983         }
2984
2985         /* configure header buffer length, needed for RSC */
2986         srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
2987
2988         /* configure the packet buffer length */
2989         srrctl |= ixgbe_rx_bufsz(rx_ring) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2990
2991         /* configure descriptor type */
2992         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2993
2994         IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);
2995 }
2996
2997 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2998 {
2999         struct ixgbe_hw *hw = &adapter->hw;
3000         static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
3001                           0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
3002                           0x6A3E67EA, 0x14364D17, 0x3BED200D};
3003         u32 mrqc = 0, reta = 0;
3004         u32 rxcsum;
3005         int i, j;
3006         u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
3007
3008         /*
3009          * Program table for at least 2 queues w/ SR-IOV so that VFs can
3010          * make full use of any rings they may have.  We will use the
3011          * PSRTYPE register to control how many rings we use within the PF.
3012          */
3013         if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 2))
3014                 rss_i = 2;
3015
3016         /* Fill out hash function seeds */
3017         for (i = 0; i < 10; i++)
3018                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
3019
3020         /* Fill out redirection table */
3021         for (i = 0, j = 0; i < 128; i++, j++) {
3022                 if (j == rss_i)
3023                         j = 0;
3024                 /* reta = 4-byte sliding window of
3025                  * 0x00..(indices-1)(indices-1)00..etc. */
3026                 reta = (reta << 8) | (j * 0x11);
3027                 if ((i & 3) == 3)
3028                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3029         }
3030
3031         /* Disable indicating checksum in descriptor, enables RSS hash */
3032         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);