bnx2x: set gso_type
[~shefty/rdma-dev.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2  *
3  * Copyright (c) 2007-2012 Broadcom Corporation
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation.
8  *
9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10  * Written by: Eliezer Tamir
11  * Based on code from Michael Chan's bnx2 driver
12  * UDP CSUM errata workaround by Arik Gendelman
13  * Slowpath and fastpath rework by Vladislav Zolotarov
14  * Statistics and Link management by Yitchak Gertner
15  *
16  */
17
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <net/ipv6.h>
25 #include <net/ip6_checksum.h>
26 #include <linux/prefetch.h>
27 #include "bnx2x_cmn.h"
28 #include "bnx2x_init.h"
29 #include "bnx2x_sp.h"
30
31
32
33 /**
34  * bnx2x_move_fp - move content of the fastpath structure.
35  *
36  * @bp:         driver handle
37  * @from:       source FP index
38  * @to:         destination FP index
39  *
40  * Makes sure the contents of the bp->fp[to].napi is kept
41  * intact. This is done by first copying the napi struct from
42  * the target to the source, and then mem copying the entire
43  * source onto the target. Update txdata pointers and related
44  * content.
45  */
46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
47 {
48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
54         int old_max_eth_txqs, new_max_eth_txqs;
55         int old_txdata_index = 0, new_txdata_index = 0;
56
57         /* Copy the NAPI object as it has been already initialized */
58         from_fp->napi = to_fp->napi;
59
60         /* Move bnx2x_fastpath contents */
61         memcpy(to_fp, from_fp, sizeof(*to_fp));
62         to_fp->index = to;
63
64         /* move sp_objs contents as well, as their indices match fp ones */
65         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
66
67         /* move fp_stats contents as well, as their indices match fp ones */
68         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
69
70         /* Update txdata pointers in fp and move txdata content accordingly:
71          * Each fp consumes 'max_cos' txdata structures, so the index should be
72          * decremented by max_cos x delta.
73          */
74
75         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
76         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
77                                 (bp)->max_cos;
78         if (from == FCOE_IDX(bp)) {
79                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
80                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
81         }
82
83         memcpy(&bp->bnx2x_txq[new_txdata_index],
84                &bp->bnx2x_txq[old_txdata_index],
85                sizeof(struct bnx2x_fp_txdata));
86         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
87 }
88
89 /**
90  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
91  *
92  * @bp: driver handle
93  * @delta:      number of eth queues which were not allocated
94  */
95 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
96 {
97         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
98
99         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
100          * backward along the array could cause memory to be overriden
101          */
102         for (cos = 1; cos < bp->max_cos; cos++) {
103                 for (i = 0; i < old_eth_num - delta; i++) {
104                         struct bnx2x_fastpath *fp = &bp->fp[i];
105                         int new_idx = cos * (old_eth_num - delta) + i;
106
107                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
108                                sizeof(struct bnx2x_fp_txdata));
109                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
110                 }
111         }
112 }
113
114 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
115
116 /* free skb in the packet ring at pos idx
117  * return idx of last bd freed
118  */
119 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
120                              u16 idx, unsigned int *pkts_compl,
121                              unsigned int *bytes_compl)
122 {
123         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
124         struct eth_tx_start_bd *tx_start_bd;
125         struct eth_tx_bd *tx_data_bd;
126         struct sk_buff *skb = tx_buf->skb;
127         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
128         int nbd;
129
130         /* prefetch skb end pointer to speedup dev_kfree_skb() */
131         prefetch(&skb->end);
132
133         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
134            txdata->txq_index, idx, tx_buf, skb);
135
136         /* unmap first bd */
137         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
138         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
139                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
140
141
142         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
143 #ifdef BNX2X_STOP_ON_ERROR
144         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
145                 BNX2X_ERR("BAD nbd!\n");
146                 bnx2x_panic();
147         }
148 #endif
149         new_cons = nbd + tx_buf->first_bd;
150
151         /* Get the next bd */
152         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
153
154         /* Skip a parse bd... */
155         --nbd;
156         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
157
158         /* ...and the TSO split header bd since they have no mapping */
159         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
160                 --nbd;
161                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
162         }
163
164         /* now free frags */
165         while (nbd > 0) {
166
167                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
168                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
169                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
170                 if (--nbd)
171                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
172         }
173
174         /* release skb */
175         WARN_ON(!skb);
176         if (likely(skb)) {
177                 (*pkts_compl)++;
178                 (*bytes_compl) += skb->len;
179         }
180
181         dev_kfree_skb_any(skb);
182         tx_buf->first_bd = 0;
183         tx_buf->skb = NULL;
184
185         return new_cons;
186 }
187
188 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
189 {
190         struct netdev_queue *txq;
191         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
192         unsigned int pkts_compl = 0, bytes_compl = 0;
193
194 #ifdef BNX2X_STOP_ON_ERROR
195         if (unlikely(bp->panic))
196                 return -1;
197 #endif
198
199         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
200         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
201         sw_cons = txdata->tx_pkt_cons;
202
203         while (sw_cons != hw_cons) {
204                 u16 pkt_cons;
205
206                 pkt_cons = TX_BD(sw_cons);
207
208                 DP(NETIF_MSG_TX_DONE,
209                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
210                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
211
212                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
213                     &pkts_compl, &bytes_compl);
214
215                 sw_cons++;
216         }
217
218         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
219
220         txdata->tx_pkt_cons = sw_cons;
221         txdata->tx_bd_cons = bd_cons;
222
223         /* Need to make the tx_bd_cons update visible to start_xmit()
224          * before checking for netif_tx_queue_stopped().  Without the
225          * memory barrier, there is a small possibility that
226          * start_xmit() will miss it and cause the queue to be stopped
227          * forever.
228          * On the other hand we need an rmb() here to ensure the proper
229          * ordering of bit testing in the following
230          * netif_tx_queue_stopped(txq) call.
231          */
232         smp_mb();
233
234         if (unlikely(netif_tx_queue_stopped(txq))) {
235                 /* Taking tx_lock() is needed to prevent reenabling the queue
236                  * while it's empty. This could have happen if rx_action() gets
237                  * suspended in bnx2x_tx_int() after the condition before
238                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
239                  *
240                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
241                  * sends some packets consuming the whole queue again->
242                  * stops the queue
243                  */
244
245                 __netif_tx_lock(txq, smp_processor_id());
246
247                 if ((netif_tx_queue_stopped(txq)) &&
248                     (bp->state == BNX2X_STATE_OPEN) &&
249                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
250                         netif_tx_wake_queue(txq);
251
252                 __netif_tx_unlock(txq);
253         }
254         return 0;
255 }
256
257 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
258                                              u16 idx)
259 {
260         u16 last_max = fp->last_max_sge;
261
262         if (SUB_S16(idx, last_max) > 0)
263                 fp->last_max_sge = idx;
264 }
265
266 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
267                                          u16 sge_len,
268                                          struct eth_end_agg_rx_cqe *cqe)
269 {
270         struct bnx2x *bp = fp->bp;
271         u16 last_max, last_elem, first_elem;
272         u16 delta = 0;
273         u16 i;
274
275         if (!sge_len)
276                 return;
277
278         /* First mark all used pages */
279         for (i = 0; i < sge_len; i++)
280                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
281                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
282
283         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
284            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
285
286         /* Here we assume that the last SGE index is the biggest */
287         prefetch((void *)(fp->sge_mask));
288         bnx2x_update_last_max_sge(fp,
289                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
290
291         last_max = RX_SGE(fp->last_max_sge);
292         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
293         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
294
295         /* If ring is not full */
296         if (last_elem + 1 != first_elem)
297                 last_elem++;
298
299         /* Now update the prod */
300         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
301                 if (likely(fp->sge_mask[i]))
302                         break;
303
304                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
305                 delta += BIT_VEC64_ELEM_SZ;
306         }
307
308         if (delta > 0) {
309                 fp->rx_sge_prod += delta;
310                 /* clear page-end entries */
311                 bnx2x_clear_sge_mask_next_elems(fp);
312         }
313
314         DP(NETIF_MSG_RX_STATUS,
315            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
316            fp->last_max_sge, fp->rx_sge_prod);
317 }
318
319 /* Set Toeplitz hash value in the skb using the value from the
320  * CQE (calculated by HW).
321  */
322 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
323                             const struct eth_fast_path_rx_cqe *cqe,
324                             bool *l4_rxhash)
325 {
326         /* Set Toeplitz hash from CQE */
327         if ((bp->dev->features & NETIF_F_RXHASH) &&
328             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
329                 enum eth_rss_hash_type htype;
330
331                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
332                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
333                              (htype == TCP_IPV6_HASH_TYPE);
334                 return le32_to_cpu(cqe->rss_hash_result);
335         }
336         *l4_rxhash = false;
337         return 0;
338 }
339
340 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
341                             u16 cons, u16 prod,
342                             struct eth_fast_path_rx_cqe *cqe)
343 {
344         struct bnx2x *bp = fp->bp;
345         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
346         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
347         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
348         dma_addr_t mapping;
349         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
350         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
351
352         /* print error if current state != stop */
353         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
354                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
355
356         /* Try to map an empty data buffer from the aggregation info  */
357         mapping = dma_map_single(&bp->pdev->dev,
358                                  first_buf->data + NET_SKB_PAD,
359                                  fp->rx_buf_size, DMA_FROM_DEVICE);
360         /*
361          *  ...if it fails - move the skb from the consumer to the producer
362          *  and set the current aggregation state as ERROR to drop it
363          *  when TPA_STOP arrives.
364          */
365
366         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
367                 /* Move the BD from the consumer to the producer */
368                 bnx2x_reuse_rx_data(fp, cons, prod);
369                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
370                 return;
371         }
372
373         /* move empty data from pool to prod */
374         prod_rx_buf->data = first_buf->data;
375         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
376         /* point prod_bd to new data */
377         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
378         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
379
380         /* move partial skb from cons to pool (don't unmap yet) */
381         *first_buf = *cons_rx_buf;
382
383         /* mark bin state as START */
384         tpa_info->parsing_flags =
385                 le16_to_cpu(cqe->pars_flags.flags);
386         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
387         tpa_info->tpa_state = BNX2X_TPA_START;
388         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
389         tpa_info->placement_offset = cqe->placement_offset;
390         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
391         if (fp->mode == TPA_MODE_GRO) {
392                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
393                 tpa_info->full_page =
394                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
395                 tpa_info->gro_size = gro_size;
396         }
397
398 #ifdef BNX2X_STOP_ON_ERROR
399         fp->tpa_queue_used |= (1 << queue);
400 #ifdef _ASM_GENERIC_INT_L64_H
401         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
402 #else
403         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
404 #endif
405            fp->tpa_queue_used);
406 #endif
407 }
408
409 /* Timestamp option length allowed for TPA aggregation:
410  *
411  *              nop nop kind length echo val
412  */
413 #define TPA_TSTAMP_OPT_LEN      12
414 /**
415  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
416  *
417  * @bp:                 driver handle
418  * @parsing_flags:      parsing flags from the START CQE
419  * @len_on_bd:          total length of the first packet for the
420  *                      aggregation.
421  *
422  * Approximate value of the MSS for this aggregation calculated using
423  * the first packet of it.
424  */
425 static u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
426                              u16 len_on_bd)
427 {
428         /*
429          * TPA arrgregation won't have either IP options or TCP options
430          * other than timestamp or IPv6 extension headers.
431          */
432         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
433
434         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
435             PRS_FLAG_OVERETH_IPV6)
436                 hdrs_len += sizeof(struct ipv6hdr);
437         else /* IPv4 */
438                 hdrs_len += sizeof(struct iphdr);
439
440
441         /* Check if there was a TCP timestamp, if there is it's will
442          * always be 12 bytes length: nop nop kind length echo val.
443          *
444          * Otherwise FW would close the aggregation.
445          */
446         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
447                 hdrs_len += TPA_TSTAMP_OPT_LEN;
448
449         return len_on_bd - hdrs_len;
450 }
451
452 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
453                               struct bnx2x_fastpath *fp, u16 index)
454 {
455         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
456         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
457         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
458         dma_addr_t mapping;
459
460         if (unlikely(page == NULL)) {
461                 BNX2X_ERR("Can't alloc sge\n");
462                 return -ENOMEM;
463         }
464
465         mapping = dma_map_page(&bp->pdev->dev, page, 0,
466                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
467         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
468                 __free_pages(page, PAGES_PER_SGE_SHIFT);
469                 BNX2X_ERR("Can't map sge\n");
470                 return -ENOMEM;
471         }
472
473         sw_buf->page = page;
474         dma_unmap_addr_set(sw_buf, mapping, mapping);
475
476         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
477         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
478
479         return 0;
480 }
481
482 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
483                                struct bnx2x_agg_info *tpa_info,
484                                u16 pages,
485                                struct sk_buff *skb,
486                                struct eth_end_agg_rx_cqe *cqe,
487                                u16 cqe_idx)
488 {
489         struct sw_rx_page *rx_pg, old_rx_pg;
490         u32 i, frag_len, frag_size;
491         int err, j, frag_id = 0;
492         u16 len_on_bd = tpa_info->len_on_bd;
493         u16 full_page = 0, gro_size = 0;
494
495         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
496
497         if (fp->mode == TPA_MODE_GRO) {
498                 gro_size = tpa_info->gro_size;
499                 full_page = tpa_info->full_page;
500         }
501
502         /* This is needed in order to enable forwarding support */
503         if (frag_size) {
504                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
505                                         tpa_info->parsing_flags, len_on_bd);
506
507                 skb_shinfo(skb)->gso_type =
508                         (GET_FLAG(tpa_info->parsing_flags,
509                                   PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
510                          PRS_FLAG_OVERETH_IPV6) ?
511                         SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
512         }
513
514
515 #ifdef BNX2X_STOP_ON_ERROR
516         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
517                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
518                           pages, cqe_idx);
519                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
520                 bnx2x_panic();
521                 return -EINVAL;
522         }
523 #endif
524
525         /* Run through the SGL and compose the fragmented skb */
526         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
527                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
528
529                 /* FW gives the indices of the SGE as if the ring is an array
530                    (meaning that "next" element will consume 2 indices) */
531                 if (fp->mode == TPA_MODE_GRO)
532                         frag_len = min_t(u32, frag_size, (u32)full_page);
533                 else /* LRO */
534                         frag_len = min_t(u32, frag_size,
535                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
536
537                 rx_pg = &fp->rx_page_ring[sge_idx];
538                 old_rx_pg = *rx_pg;
539
540                 /* If we fail to allocate a substitute page, we simply stop
541                    where we are and drop the whole packet */
542                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
543                 if (unlikely(err)) {
544                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
545                         return err;
546                 }
547
548                 /* Unmap the page as we r going to pass it to the stack */
549                 dma_unmap_page(&bp->pdev->dev,
550                                dma_unmap_addr(&old_rx_pg, mapping),
551                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
552                 /* Add one frag and update the appropriate fields in the skb */
553                 if (fp->mode == TPA_MODE_LRO)
554                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
555                 else { /* GRO */
556                         int rem;
557                         int offset = 0;
558                         for (rem = frag_len; rem > 0; rem -= gro_size) {
559                                 int len = rem > gro_size ? gro_size : rem;
560                                 skb_fill_page_desc(skb, frag_id++,
561                                                    old_rx_pg.page, offset, len);
562                                 if (offset)
563                                         get_page(old_rx_pg.page);
564                                 offset += len;
565                         }
566                 }
567
568                 skb->data_len += frag_len;
569                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
570                 skb->len += frag_len;
571
572                 frag_size -= frag_len;
573         }
574
575         return 0;
576 }
577
578 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
579 {
580         if (fp->rx_frag_size)
581                 put_page(virt_to_head_page(data));
582         else
583                 kfree(data);
584 }
585
586 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
587 {
588         if (fp->rx_frag_size)
589                 return netdev_alloc_frag(fp->rx_frag_size);
590
591         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
592 }
593
594
595 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
596                            struct bnx2x_agg_info *tpa_info,
597                            u16 pages,
598                            struct eth_end_agg_rx_cqe *cqe,
599                            u16 cqe_idx)
600 {
601         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
602         u8 pad = tpa_info->placement_offset;
603         u16 len = tpa_info->len_on_bd;
604         struct sk_buff *skb = NULL;
605         u8 *new_data, *data = rx_buf->data;
606         u8 old_tpa_state = tpa_info->tpa_state;
607
608         tpa_info->tpa_state = BNX2X_TPA_STOP;
609
610         /* If we there was an error during the handling of the TPA_START -
611          * drop this aggregation.
612          */
613         if (old_tpa_state == BNX2X_TPA_ERROR)
614                 goto drop;
615
616         /* Try to allocate the new data */
617         new_data = bnx2x_frag_alloc(fp);
618         /* Unmap skb in the pool anyway, as we are going to change
619            pool entry status to BNX2X_TPA_STOP even if new skb allocation
620            fails. */
621         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
622                          fp->rx_buf_size, DMA_FROM_DEVICE);
623         if (likely(new_data))
624                 skb = build_skb(data, fp->rx_frag_size);
625
626         if (likely(skb)) {
627 #ifdef BNX2X_STOP_ON_ERROR
628                 if (pad + len > fp->rx_buf_size) {
629                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
630                                   pad, len, fp->rx_buf_size);
631                         bnx2x_panic();
632                         return;
633                 }
634 #endif
635
636                 skb_reserve(skb, pad + NET_SKB_PAD);
637                 skb_put(skb, len);
638                 skb->rxhash = tpa_info->rxhash;
639                 skb->l4_rxhash = tpa_info->l4_rxhash;
640
641                 skb->protocol = eth_type_trans(skb, bp->dev);
642                 skb->ip_summed = CHECKSUM_UNNECESSARY;
643
644                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
645                                          skb, cqe, cqe_idx)) {
646                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
647                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
648                         napi_gro_receive(&fp->napi, skb);
649                 } else {
650                         DP(NETIF_MSG_RX_STATUS,
651                            "Failed to allocate new pages - dropping packet!\n");
652                         dev_kfree_skb_any(skb);
653                 }
654
655
656                 /* put new data in bin */
657                 rx_buf->data = new_data;
658
659                 return;
660         }
661         bnx2x_frag_free(fp, new_data);
662 drop:
663         /* drop the packet and keep the buffer in the bin */
664         DP(NETIF_MSG_RX_STATUS,
665            "Failed to allocate or map a new skb - dropping packet!\n");
666         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
667 }
668
669 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
670                                struct bnx2x_fastpath *fp, u16 index)
671 {
672         u8 *data;
673         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
674         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
675         dma_addr_t mapping;
676
677         data = bnx2x_frag_alloc(fp);
678         if (unlikely(data == NULL))
679                 return -ENOMEM;
680
681         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
682                                  fp->rx_buf_size,
683                                  DMA_FROM_DEVICE);
684         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
685                 bnx2x_frag_free(fp, data);
686                 BNX2X_ERR("Can't map rx data\n");
687                 return -ENOMEM;
688         }
689
690         rx_buf->data = data;
691         dma_unmap_addr_set(rx_buf, mapping, mapping);
692
693         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
694         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
695
696         return 0;
697 }
698
699 static
700 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
701                                  struct bnx2x_fastpath *fp,
702                                  struct bnx2x_eth_q_stats *qstats)
703 {
704         /* Do nothing if no L4 csum validation was done.
705          * We do not check whether IP csum was validated. For IPv4 we assume
706          * that if the card got as far as validating the L4 csum, it also
707          * validated the IP csum. IPv6 has no IP csum.
708          */
709         if (cqe->fast_path_cqe.status_flags &
710             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
711                 return;
712
713         /* If L4 validation was done, check if an error was found. */
714
715         if (cqe->fast_path_cqe.type_error_flags &
716             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
717              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
718                 qstats->hw_csum_err++;
719         else
720                 skb->ip_summed = CHECKSUM_UNNECESSARY;
721 }
722
723 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
724 {
725         struct bnx2x *bp = fp->bp;
726         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
727         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
728         int rx_pkt = 0;
729
730 #ifdef BNX2X_STOP_ON_ERROR
731         if (unlikely(bp->panic))
732                 return 0;
733 #endif
734
735         /* CQ "next element" is of the size of the regular element,
736            that's why it's ok here */
737         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
738         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
739                 hw_comp_cons++;
740
741         bd_cons = fp->rx_bd_cons;
742         bd_prod = fp->rx_bd_prod;
743         bd_prod_fw = bd_prod;
744         sw_comp_cons = fp->rx_comp_cons;
745         sw_comp_prod = fp->rx_comp_prod;
746
747         /* Memory barrier necessary as speculative reads of the rx
748          * buffer can be ahead of the index in the status block
749          */
750         rmb();
751
752         DP(NETIF_MSG_RX_STATUS,
753            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
754            fp->index, hw_comp_cons, sw_comp_cons);
755
756         while (sw_comp_cons != hw_comp_cons) {
757                 struct sw_rx_bd *rx_buf = NULL;
758                 struct sk_buff *skb;
759                 union eth_rx_cqe *cqe;
760                 struct eth_fast_path_rx_cqe *cqe_fp;
761                 u8 cqe_fp_flags;
762                 enum eth_rx_cqe_type cqe_fp_type;
763                 u16 len, pad, queue;
764                 u8 *data;
765                 bool l4_rxhash;
766
767 #ifdef BNX2X_STOP_ON_ERROR
768                 if (unlikely(bp->panic))
769                         return 0;
770 #endif
771
772                 comp_ring_cons = RCQ_BD(sw_comp_cons);
773                 bd_prod = RX_BD(bd_prod);
774                 bd_cons = RX_BD(bd_cons);
775
776                 cqe = &fp->rx_comp_ring[comp_ring_cons];
777                 cqe_fp = &cqe->fast_path_cqe;
778                 cqe_fp_flags = cqe_fp->type_error_flags;
779                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
780
781                 DP(NETIF_MSG_RX_STATUS,
782                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
783                    CQE_TYPE(cqe_fp_flags),
784                    cqe_fp_flags, cqe_fp->status_flags,
785                    le32_to_cpu(cqe_fp->rss_hash_result),
786                    le16_to_cpu(cqe_fp->vlan_tag),
787                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
788
789                 /* is this a slowpath msg? */
790                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
791                         bnx2x_sp_event(fp, cqe);
792                         goto next_cqe;
793                 }
794
795                 rx_buf = &fp->rx_buf_ring[bd_cons];
796                 data = rx_buf->data;
797
798                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
799                         struct bnx2x_agg_info *tpa_info;
800                         u16 frag_size, pages;
801 #ifdef BNX2X_STOP_ON_ERROR
802                         /* sanity check */
803                         if (fp->disable_tpa &&
804                             (CQE_TYPE_START(cqe_fp_type) ||
805                              CQE_TYPE_STOP(cqe_fp_type)))
806                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
807                                           CQE_TYPE(cqe_fp_type));
808 #endif
809
810                         if (CQE_TYPE_START(cqe_fp_type)) {
811                                 u16 queue = cqe_fp->queue_index;
812                                 DP(NETIF_MSG_RX_STATUS,
813                                    "calling tpa_start on queue %d\n",
814                                    queue);
815
816                                 bnx2x_tpa_start(fp, queue,
817                                                 bd_cons, bd_prod,
818                                                 cqe_fp);
819
820                                 goto next_rx;
821
822                         }
823                         queue = cqe->end_agg_cqe.queue_index;
824                         tpa_info = &fp->tpa_info[queue];
825                         DP(NETIF_MSG_RX_STATUS,
826                            "calling tpa_stop on queue %d\n",
827                            queue);
828
829                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
830                                     tpa_info->len_on_bd;
831
832                         if (fp->mode == TPA_MODE_GRO)
833                                 pages = (frag_size + tpa_info->full_page - 1) /
834                                          tpa_info->full_page;
835                         else
836                                 pages = SGE_PAGE_ALIGN(frag_size) >>
837                                         SGE_PAGE_SHIFT;
838
839                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
840                                        &cqe->end_agg_cqe, comp_ring_cons);
841 #ifdef BNX2X_STOP_ON_ERROR
842                         if (bp->panic)
843                                 return 0;
844 #endif
845
846                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
847                         goto next_cqe;
848                 }
849                 /* non TPA */
850                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
851                 pad = cqe_fp->placement_offset;
852                 dma_sync_single_for_cpu(&bp->pdev->dev,
853                                         dma_unmap_addr(rx_buf, mapping),
854                                         pad + RX_COPY_THRESH,
855                                         DMA_FROM_DEVICE);
856                 pad += NET_SKB_PAD;
857                 prefetch(data + pad); /* speedup eth_type_trans() */
858                 /* is this an error packet? */
859                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
860                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
861                            "ERROR  flags %x  rx packet %u\n",
862                            cqe_fp_flags, sw_comp_cons);
863                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
864                         goto reuse_rx;
865                 }
866
867                 /* Since we don't have a jumbo ring
868                  * copy small packets if mtu > 1500
869                  */
870                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
871                     (len <= RX_COPY_THRESH)) {
872                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
873                         if (skb == NULL) {
874                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
875                                    "ERROR  packet dropped because of alloc failure\n");
876                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
877                                 goto reuse_rx;
878                         }
879                         memcpy(skb->data, data + pad, len);
880                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
881                 } else {
882                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
883                                 dma_unmap_single(&bp->pdev->dev,
884                                                  dma_unmap_addr(rx_buf, mapping),
885                                                  fp->rx_buf_size,
886                                                  DMA_FROM_DEVICE);
887                                 skb = build_skb(data, fp->rx_frag_size);
888                                 if (unlikely(!skb)) {
889                                         bnx2x_frag_free(fp, data);
890                                         bnx2x_fp_qstats(bp, fp)->
891                                                         rx_skb_alloc_failed++;
892                                         goto next_rx;
893                                 }
894                                 skb_reserve(skb, pad);
895                         } else {
896                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
897                                    "ERROR  packet dropped because of alloc failure\n");
898                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
899 reuse_rx:
900                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
901                                 goto next_rx;
902                         }
903                 }
904
905                 skb_put(skb, len);
906                 skb->protocol = eth_type_trans(skb, bp->dev);
907
908                 /* Set Toeplitz hash for a none-LRO skb */
909                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
910                 skb->l4_rxhash = l4_rxhash;
911
912                 skb_checksum_none_assert(skb);
913
914                 if (bp->dev->features & NETIF_F_RXCSUM)
915                         bnx2x_csum_validate(skb, cqe, fp,
916                                             bnx2x_fp_qstats(bp, fp));
917
918                 skb_record_rx_queue(skb, fp->rx_queue);
919
920                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
921                     PARSING_FLAGS_VLAN)
922                         __vlan_hwaccel_put_tag(skb,
923                                                le16_to_cpu(cqe_fp->vlan_tag));
924                 napi_gro_receive(&fp->napi, skb);
925
926
927 next_rx:
928                 rx_buf->data = NULL;
929
930                 bd_cons = NEXT_RX_IDX(bd_cons);
931                 bd_prod = NEXT_RX_IDX(bd_prod);
932                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
933                 rx_pkt++;
934 next_cqe:
935                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
936                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
937
938                 if (rx_pkt == budget)
939                         break;
940         } /* while */
941
942         fp->rx_bd_cons = bd_cons;
943         fp->rx_bd_prod = bd_prod_fw;
944         fp->rx_comp_cons = sw_comp_cons;
945         fp->rx_comp_prod = sw_comp_prod;
946
947         /* Update producers */
948         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
949                              fp->rx_sge_prod);
950
951         fp->rx_pkt += rx_pkt;
952         fp->rx_calls++;
953
954         return rx_pkt;
955 }
956
957 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
958 {
959         struct bnx2x_fastpath *fp = fp_cookie;
960         struct bnx2x *bp = fp->bp;
961         u8 cos;
962
963         DP(NETIF_MSG_INTR,
964            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
965            fp->index, fp->fw_sb_id, fp->igu_sb_id);
966         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
967
968 #ifdef BNX2X_STOP_ON_ERROR
969         if (unlikely(bp->panic))
970                 return IRQ_HANDLED;
971 #endif
972
973         /* Handle Rx and Tx according to MSI-X vector */
974         prefetch(fp->rx_cons_sb);
975
976         for_each_cos_in_tx_queue(fp, cos)
977                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
978
979         prefetch(&fp->sb_running_index[SM_RX_ID]);
980         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
981
982         return IRQ_HANDLED;
983 }
984
985 /* HW Lock for shared dual port PHYs */
986 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
987 {
988         mutex_lock(&bp->port.phy_mutex);
989
990         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
991 }
992
993 void bnx2x_release_phy_lock(struct bnx2x *bp)
994 {
995         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
996
997         mutex_unlock(&bp->port.phy_mutex);
998 }
999
1000 /* calculates MF speed according to current linespeed and MF configuration */
1001 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1002 {
1003         u16 line_speed = bp->link_vars.line_speed;
1004         if (IS_MF(bp)) {
1005                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1006                                                    bp->mf_config[BP_VN(bp)]);
1007
1008                 /* Calculate the current MAX line speed limit for the MF
1009                  * devices
1010                  */
1011                 if (IS_MF_SI(bp))
1012                         line_speed = (line_speed * maxCfg) / 100;
1013                 else { /* SD mode */
1014                         u16 vn_max_rate = maxCfg * 100;
1015
1016                         if (vn_max_rate < line_speed)
1017                                 line_speed = vn_max_rate;
1018                 }
1019         }
1020
1021         return line_speed;
1022 }
1023
1024 /**
1025  * bnx2x_fill_report_data - fill link report data to report
1026  *
1027  * @bp:         driver handle
1028  * @data:       link state to update
1029  *
1030  * It uses a none-atomic bit operations because is called under the mutex.
1031  */
1032 static void bnx2x_fill_report_data(struct bnx2x *bp,
1033                                    struct bnx2x_link_report_data *data)
1034 {
1035         u16 line_speed = bnx2x_get_mf_speed(bp);
1036
1037         memset(data, 0, sizeof(*data));
1038
1039         /* Fill the report data: efective line speed */
1040         data->line_speed = line_speed;
1041
1042         /* Link is down */
1043         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1044                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1045                           &data->link_report_flags);
1046
1047         /* Full DUPLEX */
1048         if (bp->link_vars.duplex == DUPLEX_FULL)
1049                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1050
1051         /* Rx Flow Control is ON */
1052         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1053                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1054
1055         /* Tx Flow Control is ON */
1056         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1057                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1058 }
1059
1060 /**
1061  * bnx2x_link_report - report link status to OS.
1062  *
1063  * @bp:         driver handle
1064  *
1065  * Calls the __bnx2x_link_report() under the same locking scheme
1066  * as a link/PHY state managing code to ensure a consistent link
1067  * reporting.
1068  */
1069
1070 void bnx2x_link_report(struct bnx2x *bp)
1071 {
1072         bnx2x_acquire_phy_lock(bp);
1073         __bnx2x_link_report(bp);
1074         bnx2x_release_phy_lock(bp);
1075 }
1076
1077 /**
1078  * __bnx2x_link_report - report link status to OS.
1079  *
1080  * @bp:         driver handle
1081  *
1082  * None atomic inmlementation.
1083  * Should be called under the phy_lock.
1084  */
1085 void __bnx2x_link_report(struct bnx2x *bp)
1086 {
1087         struct bnx2x_link_report_data cur_data;
1088
1089         /* reread mf_cfg */
1090         if (!CHIP_IS_E1(bp))
1091                 bnx2x_read_mf_cfg(bp);
1092
1093         /* Read the current link report info */
1094         bnx2x_fill_report_data(bp, &cur_data);
1095
1096         /* Don't report link down or exactly the same link status twice */
1097         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1098             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1099                       &bp->last_reported_link.link_report_flags) &&
1100              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1101                       &cur_data.link_report_flags)))
1102                 return;
1103
1104         bp->link_cnt++;
1105
1106         /* We are going to report a new link parameters now -
1107          * remember the current data for the next time.
1108          */
1109         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1110
1111         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1112                      &cur_data.link_report_flags)) {
1113                 netif_carrier_off(bp->dev);
1114                 netdev_err(bp->dev, "NIC Link is Down\n");
1115                 return;
1116         } else {
1117                 const char *duplex;
1118                 const char *flow;
1119
1120                 netif_carrier_on(bp->dev);
1121
1122                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1123                                        &cur_data.link_report_flags))
1124                         duplex = "full";
1125                 else
1126                         duplex = "half";
1127
1128                 /* Handle the FC at the end so that only these flags would be
1129                  * possibly set. This way we may easily check if there is no FC
1130                  * enabled.
1131                  */
1132                 if (cur_data.link_report_flags) {
1133                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1134                                      &cur_data.link_report_flags)) {
1135                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1136                                      &cur_data.link_report_flags))
1137                                         flow = "ON - receive & transmit";
1138                                 else
1139                                         flow = "ON - receive";
1140                         } else {
1141                                 flow = "ON - transmit";
1142                         }
1143                 } else {
1144                         flow = "none";
1145                 }
1146                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1147                             cur_data.line_speed, duplex, flow);
1148         }
1149 }
1150
1151 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1152 {
1153         int i;
1154
1155         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1156                 struct eth_rx_sge *sge;
1157
1158                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1159                 sge->addr_hi =
1160                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1161                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1162
1163                 sge->addr_lo =
1164                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1165                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1166         }
1167 }
1168
1169 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1170                                 struct bnx2x_fastpath *fp, int last)
1171 {
1172         int i;
1173
1174         for (i = 0; i < last; i++) {
1175                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1176                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1177                 u8 *data = first_buf->data;
1178
1179                 if (data == NULL) {
1180                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1181                         continue;
1182                 }
1183                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1184                         dma_unmap_single(&bp->pdev->dev,
1185                                          dma_unmap_addr(first_buf, mapping),
1186                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1187                 bnx2x_frag_free(fp, data);
1188                 first_buf->data = NULL;
1189         }
1190 }
1191
1192 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1193 {
1194         int j;
1195
1196         for_each_rx_queue_cnic(bp, j) {
1197                 struct bnx2x_fastpath *fp = &bp->fp[j];
1198
1199                 fp->rx_bd_cons = 0;
1200
1201                 /* Activate BD ring */
1202                 /* Warning!
1203                  * this will generate an interrupt (to the TSTORM)
1204                  * must only be done after chip is initialized
1205                  */
1206                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1207                                      fp->rx_sge_prod);
1208         }
1209 }
1210
1211 void bnx2x_init_rx_rings(struct bnx2x *bp)
1212 {
1213         int func = BP_FUNC(bp);
1214         u16 ring_prod;
1215         int i, j;
1216
1217         /* Allocate TPA resources */
1218         for_each_eth_queue(bp, j) {
1219                 struct bnx2x_fastpath *fp = &bp->fp[j];
1220
1221                 DP(NETIF_MSG_IFUP,
1222                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1223
1224                 if (!fp->disable_tpa) {
1225                         /* Fill the per-aggregtion pool */
1226                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1227                                 struct bnx2x_agg_info *tpa_info =
1228                                         &fp->tpa_info[i];
1229                                 struct sw_rx_bd *first_buf =
1230                                         &tpa_info->first_buf;
1231
1232                                 first_buf->data = bnx2x_frag_alloc(fp);
1233                                 if (!first_buf->data) {
1234                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1235                                                   j);
1236                                         bnx2x_free_tpa_pool(bp, fp, i);
1237                                         fp->disable_tpa = 1;
1238                                         break;
1239                                 }
1240                                 dma_unmap_addr_set(first_buf, mapping, 0);
1241                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1242                         }
1243
1244                         /* "next page" elements initialization */
1245                         bnx2x_set_next_page_sgl(fp);
1246
1247                         /* set SGEs bit mask */
1248                         bnx2x_init_sge_ring_bit_mask(fp);
1249
1250                         /* Allocate SGEs and initialize the ring elements */
1251                         for (i = 0, ring_prod = 0;
1252                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1253
1254                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1255                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1256                                                   i);
1257                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1258                                                   j);
1259                                         /* Cleanup already allocated elements */
1260                                         bnx2x_free_rx_sge_range(bp, fp,
1261                                                                 ring_prod);
1262                                         bnx2x_free_tpa_pool(bp, fp,
1263                                                             MAX_AGG_QS(bp));
1264                                         fp->disable_tpa = 1;
1265                                         ring_prod = 0;
1266                                         break;
1267                                 }
1268                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1269                         }
1270
1271                         fp->rx_sge_prod = ring_prod;
1272                 }
1273         }
1274
1275         for_each_eth_queue(bp, j) {
1276                 struct bnx2x_fastpath *fp = &bp->fp[j];
1277
1278                 fp->rx_bd_cons = 0;
1279
1280                 /* Activate BD ring */
1281                 /* Warning!
1282                  * this will generate an interrupt (to the TSTORM)
1283                  * must only be done after chip is initialized
1284                  */
1285                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1286                                      fp->rx_sge_prod);
1287
1288                 if (j != 0)
1289                         continue;
1290
1291                 if (CHIP_IS_E1(bp)) {
1292                         REG_WR(bp, BAR_USTRORM_INTMEM +
1293                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1294                                U64_LO(fp->rx_comp_mapping));
1295                         REG_WR(bp, BAR_USTRORM_INTMEM +
1296                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1297                                U64_HI(fp->rx_comp_mapping));
1298                 }
1299         }
1300 }
1301
1302 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1303 {
1304         u8 cos;
1305         struct bnx2x *bp = fp->bp;
1306
1307         for_each_cos_in_tx_queue(fp, cos) {
1308                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1309                 unsigned pkts_compl = 0, bytes_compl = 0;
1310
1311                 u16 sw_prod = txdata->tx_pkt_prod;
1312                 u16 sw_cons = txdata->tx_pkt_cons;
1313
1314                 while (sw_cons != sw_prod) {
1315                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1316                                           &pkts_compl, &bytes_compl);
1317                         sw_cons++;
1318                 }
1319
1320                 netdev_tx_reset_queue(
1321                         netdev_get_tx_queue(bp->dev,
1322                                             txdata->txq_index));
1323         }
1324 }
1325
1326 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1327 {
1328         int i;
1329
1330         for_each_tx_queue_cnic(bp, i) {
1331                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1332         }
1333 }
1334
1335 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1336 {
1337         int i;
1338
1339         for_each_eth_queue(bp, i) {
1340                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1341         }
1342 }
1343
1344 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1345 {
1346         struct bnx2x *bp = fp->bp;
1347         int i;
1348
1349         /* ring wasn't allocated */
1350         if (fp->rx_buf_ring == NULL)
1351                 return;
1352
1353         for (i = 0; i < NUM_RX_BD; i++) {
1354                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1355                 u8 *data = rx_buf->data;
1356
1357                 if (data == NULL)
1358                         continue;
1359                 dma_unmap_single(&bp->pdev->dev,
1360                                  dma_unmap_addr(rx_buf, mapping),
1361                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1362
1363                 rx_buf->data = NULL;
1364                 bnx2x_frag_free(fp, data);
1365         }
1366 }
1367
1368 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1369 {
1370         int j;
1371
1372         for_each_rx_queue_cnic(bp, j) {
1373                 bnx2x_free_rx_bds(&bp->fp[j]);
1374         }
1375 }
1376
1377 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1378 {
1379         int j;
1380
1381         for_each_eth_queue(bp, j) {
1382                 struct bnx2x_fastpath *fp = &bp->fp[j];
1383
1384                 bnx2x_free_rx_bds(fp);
1385
1386                 if (!fp->disable_tpa)
1387                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1388         }
1389 }
1390
1391 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1392 {
1393         bnx2x_free_tx_skbs_cnic(bp);
1394         bnx2x_free_rx_skbs_cnic(bp);
1395 }
1396
1397 void bnx2x_free_skbs(struct bnx2x *bp)
1398 {
1399         bnx2x_free_tx_skbs(bp);
1400         bnx2x_free_rx_skbs(bp);
1401 }
1402
1403 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1404 {
1405         /* load old values */
1406         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1407
1408         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1409                 /* leave all but MAX value */
1410                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1411
1412                 /* set new MAX value */
1413                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1414                                 & FUNC_MF_CFG_MAX_BW_MASK;
1415
1416                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1417         }
1418 }
1419
1420 /**
1421  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1422  *
1423  * @bp:         driver handle
1424  * @nvecs:      number of vectors to be released
1425  */
1426 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1427 {
1428         int i, offset = 0;
1429
1430         if (nvecs == offset)
1431                 return;
1432         free_irq(bp->msix_table[offset].vector, bp->dev);
1433         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1434            bp->msix_table[offset].vector);
1435         offset++;
1436
1437         if (CNIC_SUPPORT(bp)) {
1438                 if (nvecs == offset)
1439                         return;
1440                 offset++;
1441         }
1442
1443         for_each_eth_queue(bp, i) {
1444                 if (nvecs == offset)
1445                         return;
1446                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1447                    i, bp->msix_table[offset].vector);
1448
1449                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1450         }
1451 }
1452
1453 void bnx2x_free_irq(struct bnx2x *bp)
1454 {
1455         if (bp->flags & USING_MSIX_FLAG &&
1456             !(bp->flags & USING_SINGLE_MSIX_FLAG))
1457                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1458                                      CNIC_SUPPORT(bp) + 1);
1459         else
1460                 free_irq(bp->dev->irq, bp->dev);
1461 }
1462
1463 int bnx2x_enable_msix(struct bnx2x *bp)
1464 {
1465         int msix_vec = 0, i, rc, req_cnt;
1466
1467         bp->msix_table[msix_vec].entry = msix_vec;
1468         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1469            bp->msix_table[0].entry);
1470         msix_vec++;
1471
1472         /* Cnic requires an msix vector for itself */
1473         if (CNIC_SUPPORT(bp)) {
1474                 bp->msix_table[msix_vec].entry = msix_vec;
1475                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1476                                msix_vec, bp->msix_table[msix_vec].entry);
1477                 msix_vec++;
1478         }
1479
1480         /* We need separate vectors for ETH queues only (not FCoE) */
1481         for_each_eth_queue(bp, i) {
1482                 bp->msix_table[msix_vec].entry = msix_vec;
1483                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1484                                msix_vec, msix_vec, i);
1485                 msix_vec++;
1486         }
1487
1488         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp) + 1;
1489
1490         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1491
1492         /*
1493          * reconfigure number of tx/rx queues according to available
1494          * MSI-X vectors
1495          */
1496         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1497                 /* how less vectors we will have? */
1498                 int diff = req_cnt - rc;
1499
1500                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1501
1502                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1503
1504                 if (rc) {
1505                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1506                         goto no_msix;
1507                 }
1508                 /*
1509                  * decrease number of queues by number of unallocated entries
1510                  */
1511                 bp->num_ethernet_queues -= diff;
1512                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1513
1514                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1515                                bp->num_queues);
1516         } else if (rc > 0) {
1517                 /* Get by with single vector */
1518                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1519                 if (rc) {
1520                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1521                                        rc);
1522                         goto no_msix;
1523                 }
1524
1525                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1526                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1527
1528                 BNX2X_DEV_INFO("set number of queues to 1\n");
1529                 bp->num_ethernet_queues = 1;
1530                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1531         } else if (rc < 0) {
1532                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1533                 goto no_msix;
1534         }
1535
1536         bp->flags |= USING_MSIX_FLAG;
1537
1538         return 0;
1539
1540 no_msix:
1541         /* fall to INTx if not enough memory */
1542         if (rc == -ENOMEM)
1543                 bp->flags |= DISABLE_MSI_FLAG;
1544
1545         return rc;
1546 }
1547
1548 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1549 {
1550         int i, rc, offset = 0;
1551
1552         rc = request_irq(bp->msix_table[offset++].vector,
1553                          bnx2x_msix_sp_int, 0,
1554                          bp->dev->name, bp->dev);
1555         if (rc) {
1556                 BNX2X_ERR("request sp irq failed\n");
1557                 return -EBUSY;
1558         }
1559
1560         if (CNIC_SUPPORT(bp))
1561                 offset++;
1562
1563         for_each_eth_queue(bp, i) {
1564                 struct bnx2x_fastpath *fp = &bp->fp[i];
1565                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1566                          bp->dev->name, i);
1567
1568                 rc = request_irq(bp->msix_table[offset].vector,
1569                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1570                 if (rc) {
1571                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1572                               bp->msix_table[offset].vector, rc);
1573                         bnx2x_free_msix_irqs(bp, offset);
1574                         return -EBUSY;
1575                 }
1576
1577                 offset++;
1578         }
1579
1580         i = BNX2X_NUM_ETH_QUEUES(bp);
1581         offset = 1 + CNIC_SUPPORT(bp);
1582         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1583                bp->msix_table[0].vector,
1584                0, bp->msix_table[offset].vector,
1585                i - 1, bp->msix_table[offset + i - 1].vector);
1586
1587         return 0;
1588 }
1589
1590 int bnx2x_enable_msi(struct bnx2x *bp)
1591 {
1592         int rc;
1593
1594         rc = pci_enable_msi(bp->pdev);
1595         if (rc) {
1596                 BNX2X_DEV_INFO("MSI is not attainable\n");
1597                 return -1;
1598         }
1599         bp->flags |= USING_MSI_FLAG;
1600
1601         return 0;
1602 }
1603
1604 static int bnx2x_req_irq(struct bnx2x *bp)
1605 {
1606         unsigned long flags;
1607         unsigned int irq;
1608
1609         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1610                 flags = 0;
1611         else
1612                 flags = IRQF_SHARED;
1613
1614         if (bp->flags & USING_MSIX_FLAG)
1615                 irq = bp->msix_table[0].vector;
1616         else
1617                 irq = bp->pdev->irq;
1618
1619         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1620 }
1621
1622 static int bnx2x_setup_irqs(struct bnx2x *bp)
1623 {
1624         int rc = 0;
1625         if (bp->flags & USING_MSIX_FLAG &&
1626             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1627                 rc = bnx2x_req_msix_irqs(bp);
1628                 if (rc)
1629                         return rc;
1630         } else {
1631                 bnx2x_ack_int(bp);
1632                 rc = bnx2x_req_irq(bp);
1633                 if (rc) {
1634                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1635                         return rc;
1636                 }
1637                 if (bp->flags & USING_MSI_FLAG) {
1638                         bp->dev->irq = bp->pdev->irq;
1639                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1640                                     bp->dev->irq);
1641                 }
1642                 if (bp->flags & USING_MSIX_FLAG) {
1643                         bp->dev->irq = bp->msix_table[0].vector;
1644                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1645                                     bp->dev->irq);
1646                 }
1647         }
1648
1649         return 0;
1650 }
1651
1652 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1653 {
1654         int i;
1655
1656         for_each_rx_queue_cnic(bp, i)
1657                 napi_enable(&bnx2x_fp(bp, i, napi));
1658 }
1659
1660 static void bnx2x_napi_enable(struct bnx2x *bp)
1661 {
1662         int i;
1663
1664         for_each_eth_queue(bp, i)
1665                 napi_enable(&bnx2x_fp(bp, i, napi));
1666 }
1667
1668 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1669 {
1670         int i;
1671
1672         for_each_rx_queue_cnic(bp, i)
1673                 napi_disable(&bnx2x_fp(bp, i, napi));
1674 }
1675
1676 static void bnx2x_napi_disable(struct bnx2x *bp)
1677 {
1678         int i;
1679
1680         for_each_eth_queue(bp, i)
1681                 napi_disable(&bnx2x_fp(bp, i, napi));
1682 }
1683
1684 void bnx2x_netif_start(struct bnx2x *bp)
1685 {
1686         if (netif_running(bp->dev)) {
1687                 bnx2x_napi_enable(bp);
1688                 if (CNIC_LOADED(bp))
1689                         bnx2x_napi_enable_cnic(bp);
1690                 bnx2x_int_enable(bp);
1691                 if (bp->state == BNX2X_STATE_OPEN)
1692                         netif_tx_wake_all_queues(bp->dev);
1693         }
1694 }
1695
1696 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1697 {
1698         bnx2x_int_disable_sync(bp, disable_hw);
1699         bnx2x_napi_disable(bp);
1700         if (CNIC_LOADED(bp))
1701                 bnx2x_napi_disable_cnic(bp);
1702 }
1703
1704 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1705 {
1706         struct bnx2x *bp = netdev_priv(dev);
1707
1708         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1709                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1710                 u16 ether_type = ntohs(hdr->h_proto);
1711
1712                 /* Skip VLAN tag if present */
1713                 if (ether_type == ETH_P_8021Q) {
1714                         struct vlan_ethhdr *vhdr =
1715                                 (struct vlan_ethhdr *)skb->data;
1716
1717                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1718                 }
1719
1720                 /* If ethertype is FCoE or FIP - use FCoE ring */
1721                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1722                         return bnx2x_fcoe_tx(bp, txq_index);
1723         }
1724
1725         /* select a non-FCoE queue */
1726         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1727 }
1728
1729
1730 void bnx2x_set_num_queues(struct bnx2x *bp)
1731 {
1732         /* RSS queues */
1733         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1734
1735         /* override in STORAGE SD modes */
1736         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1737                 bp->num_ethernet_queues = 1;
1738
1739         /* Add special queues */
1740         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1741         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1742
1743         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1744 }
1745
1746 /**
1747  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1748  *
1749  * @bp:         Driver handle
1750  *
1751  * We currently support for at most 16 Tx queues for each CoS thus we will
1752  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1753  * bp->max_cos.
1754  *
1755  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1756  * index after all ETH L2 indices.
1757  *
1758  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1759  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1760  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1761  *
1762  * The proper configuration of skb->queue_mapping is handled by
1763  * bnx2x_select_queue() and __skb_tx_hash().
1764  *
1765  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1766  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1767  */
1768 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1769 {
1770         int rc, tx, rx;
1771
1772         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1773         rx = BNX2X_NUM_ETH_QUEUES(bp);
1774
1775 /* account for fcoe queue */
1776         if (include_cnic && !NO_FCOE(bp)) {
1777                 rx++;
1778                 tx++;
1779         }
1780
1781         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1782         if (rc) {
1783                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1784                 return rc;
1785         }
1786         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1787         if (rc) {
1788                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1789                 return rc;
1790         }
1791
1792         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1793                           tx, rx);
1794
1795         return rc;
1796 }
1797
1798 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1799 {
1800         int i;
1801
1802         for_each_queue(bp, i) {
1803                 struct bnx2x_fastpath *fp = &bp->fp[i];
1804                 u32 mtu;
1805
1806                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1807                 if (IS_FCOE_IDX(i))
1808                         /*
1809                          * Although there are no IP frames expected to arrive to
1810                          * this ring we still want to add an
1811                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1812                          * overrun attack.
1813                          */
1814                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1815                 else
1816                         mtu = bp->dev->mtu;
1817                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1818                                   IP_HEADER_ALIGNMENT_PADDING +
1819                                   ETH_OVREHEAD +
1820                                   mtu +
1821                                   BNX2X_FW_RX_ALIGN_END;
1822                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1823                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1824                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1825                 else
1826                         fp->rx_frag_size = 0;
1827         }
1828 }
1829
1830 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1831 {
1832         int i;
1833         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1834
1835         /* Prepare the initial contents fo the indirection table if RSS is
1836          * enabled
1837          */
1838         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1839                 bp->rss_conf_obj.ind_table[i] =
1840                         bp->fp->cl_id +
1841                         ethtool_rxfh_indir_default(i, num_eth_queues);
1842
1843         /*
1844          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1845          * per-port, so if explicit configuration is needed , do it only
1846          * for a PMF.
1847          *
1848          * For 57712 and newer on the other hand it's a per-function
1849          * configuration.
1850          */
1851         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1852 }
1853
1854 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1855                         bool config_hash)
1856 {
1857         struct bnx2x_config_rss_params params = {NULL};
1858
1859         /* Although RSS is meaningless when there is a single HW queue we
1860          * still need it enabled in order to have HW Rx hash generated.
1861          *
1862          * if (!is_eth_multi(bp))
1863          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1864          */
1865
1866         params.rss_obj = rss_obj;
1867
1868         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1869
1870         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1871
1872         /* RSS configuration */
1873         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1874         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1875         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1876         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1877         if (rss_obj->udp_rss_v4)
1878                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1879         if (rss_obj->udp_rss_v6)
1880                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1881
1882         /* Hash bits */
1883         params.rss_result_mask = MULTI_MASK;
1884
1885         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1886
1887         if (config_hash) {
1888                 /* RSS keys */
1889                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
1890                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1891         }
1892
1893         return bnx2x_config_rss(bp, &params);
1894 }
1895
1896 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1897 {
1898         struct bnx2x_func_state_params func_params = {NULL};
1899
1900         /* Prepare parameters for function state transitions */
1901         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1902
1903         func_params.f_obj = &bp->func_obj;
1904         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1905
1906         func_params.params.hw_init.load_phase = load_code;
1907
1908         return bnx2x_func_state_change(bp, &func_params);
1909 }
1910
1911 /*
1912  * Cleans the object that have internal lists without sending
1913  * ramrods. Should be run when interrutps are disabled.
1914  */
1915 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1916 {
1917         int rc;
1918         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1919         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1920         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
1921
1922         /***************** Cleanup MACs' object first *************************/
1923
1924         /* Wait for completion of requested */
1925         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1926         /* Perform a dry cleanup */
1927         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1928
1929         /* Clean ETH primary MAC */
1930         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1931         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
1932                                  &ramrod_flags);
1933         if (rc != 0)
1934                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1935
1936         /* Cleanup UC list */
1937         vlan_mac_flags = 0;
1938         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1939         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1940                                  &ramrod_flags);
1941         if (rc != 0)
1942                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1943
1944         /***************** Now clean mcast object *****************************/
1945         rparam.mcast_obj = &bp->mcast_obj;
1946         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1947
1948         /* Add a DEL command... */
1949         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1950         if (rc < 0)
1951                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1952                           rc);
1953
1954         /* ...and wait until all pending commands are cleared */
1955         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1956         while (rc != 0) {
1957                 if (rc < 0) {
1958                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1959                                   rc);
1960                         return;
1961                 }
1962
1963                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1964         }
1965 }
1966
1967 #ifndef BNX2X_STOP_ON_ERROR
1968 #define LOAD_ERROR_EXIT(bp, label) \
1969         do { \
1970                 (bp)->state = BNX2X_STATE_ERROR; \
1971                 goto label; \
1972         } while (0)
1973
1974 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1975         do { \
1976                 bp->cnic_loaded = false; \
1977                 goto label; \
1978         } while (0)
1979 #else /*BNX2X_STOP_ON_ERROR*/
1980 #define LOAD_ERROR_EXIT(bp, label) \
1981         do { \
1982                 (bp)->state = BNX2X_STATE_ERROR; \
1983                 (bp)->panic = 1; \
1984                 return -EBUSY; \
1985         } while (0)
1986 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1987         do { \
1988                 bp->cnic_loaded = false; \
1989                 (bp)->panic = 1; \
1990                 return -EBUSY; \
1991         } while (0)
1992 #endif /*BNX2X_STOP_ON_ERROR*/
1993
1994 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1995 {
1996         /* build FW version dword */
1997         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1998                     (BCM_5710_FW_MINOR_VERSION << 8) +
1999                     (BCM_5710_FW_REVISION_VERSION << 16) +
2000                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
2001
2002         /* read loaded FW from chip */
2003         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2004
2005         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
2006
2007         if (loaded_fw != my_fw) {
2008                 if (is_err)
2009                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
2010                                   loaded_fw, my_fw);
2011                 return false;
2012         }
2013
2014         return true;
2015 }
2016
2017 /**
2018  * bnx2x_bz_fp - zero content of the fastpath structure.
2019  *
2020  * @bp:         driver handle
2021  * @index:      fastpath index to be zeroed
2022  *
2023  * Makes sure the contents of the bp->fp[index].napi is kept
2024  * intact.
2025  */
2026 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2027 {
2028         struct bnx2x_fastpath *fp = &bp->fp[index];
2029         struct bnx2x_fp_stats *fp_stats = &bp->fp_stats[index];
2030
2031         int cos;
2032         struct napi_struct orig_napi = fp->napi;
2033         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2034         /* bzero bnx2x_fastpath contents */
2035         if (bp->stats_init) {
2036                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2037                 memset(fp, 0, sizeof(*fp));
2038         } else {
2039                 /* Keep Queue statistics */
2040                 struct bnx2x_eth_q_stats *tmp_eth_q_stats;
2041                 struct bnx2x_eth_q_stats_old *tmp_eth_q_stats_old;
2042
2043                 tmp_eth_q_stats = kzalloc(sizeof(struct bnx2x_eth_q_stats),
2044                                           GFP_KERNEL);
2045                 if (tmp_eth_q_stats)
2046                         memcpy(tmp_eth_q_stats, &fp_stats->eth_q_stats,
2047                                sizeof(struct bnx2x_eth_q_stats));
2048
2049                 tmp_eth_q_stats_old =
2050                         kzalloc(sizeof(struct bnx2x_eth_q_stats_old),
2051                                 GFP_KERNEL);
2052                 if (tmp_eth_q_stats_old)
2053                         memcpy(tmp_eth_q_stats_old, &fp_stats->eth_q_stats_old,
2054                                sizeof(struct bnx2x_eth_q_stats_old));
2055
2056                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2057                 memset(fp, 0, sizeof(*fp));
2058
2059                 if (tmp_eth_q_stats) {
2060                         memcpy(&fp_stats->eth_q_stats, tmp_eth_q_stats,
2061                                sizeof(struct bnx2x_eth_q_stats));
2062                         kfree(tmp_eth_q_stats);
2063                 }
2064
2065                 if (tmp_eth_q_stats_old) {
2066                         memcpy(&fp_stats->eth_q_stats_old, tmp_eth_q_stats_old,
2067                                sizeof(struct bnx2x_eth_q_stats_old));
2068                         kfree(tmp_eth_q_stats_old);
2069                 }
2070
2071         }
2072
2073         /* Restore the NAPI object as it has been already initialized */
2074         fp->napi = orig_napi;
2075         fp->tpa_info = orig_tpa_info;
2076         fp->bp = bp;
2077         fp->index = index;
2078         if (IS_ETH_FP(fp))
2079                 fp->max_cos = bp->max_cos;
2080         else
2081                 /* Special queues support only one CoS */
2082                 fp->max_cos = 1;
2083
2084         /* Init txdata pointers */
2085         if (IS_FCOE_FP(fp))
2086                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2087         if (IS_ETH_FP(fp))
2088                 for_each_cos_in_tx_queue(fp, cos)
2089                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2090                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2091
2092         /*
2093          * set the tpa flag for each queue. The tpa flag determines the queue
2094          * minimal size so it must be set prior to queue memory allocation
2095          */
2096         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2097                                   (bp->flags & GRO_ENABLE_FLAG &&
2098                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2099         if (bp->flags & TPA_ENABLE_FLAG)
2100                 fp->mode = TPA_MODE_LRO;
2101         else if (bp->flags & GRO_ENABLE_FLAG)
2102                 fp->mode = TPA_MODE_GRO;
2103
2104         /* We don't want TPA on an FCoE L2 ring */
2105         if (IS_FCOE_FP(fp))
2106                 fp->disable_tpa = 1;
2107 }
2108
2109 int bnx2x_load_cnic(struct bnx2x *bp)
2110 {
2111         int i, rc, port = BP_PORT(bp);
2112
2113         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2114
2115         mutex_init(&bp->cnic_mutex);
2116
2117         rc = bnx2x_alloc_mem_cnic(bp);
2118         if (rc) {
2119                 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2120                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2121         }
2122
2123         rc = bnx2x_alloc_fp_mem_cnic(bp);
2124         if (rc) {
2125                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2126                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2127         }
2128
2129         /* Update the number of queues with the cnic queues */
2130         rc = bnx2x_set_real_num_queues(bp, 1);
2131         if (rc) {
2132                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2133                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2134         }
2135
2136         /* Add all CNIC NAPI objects */
2137         bnx2x_add_all_napi_cnic(bp);
2138         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2139         bnx2x_napi_enable_cnic(bp);
2140
2141         rc = bnx2x_init_hw_func_cnic(bp);
2142         if (rc)
2143                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2144
2145         bnx2x_nic_init_cnic(bp);
2146
2147         /* Enable Timer scan */
2148         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2149
2150         for_each_cnic_queue(bp, i) {
2151                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2152                 if (rc) {
2153                         BNX2X_ERR("Queue setup failed\n");
2154                         LOAD_ERROR_EXIT(bp, load_error_cnic2);
2155                 }
2156         }
2157
2158         /* Initialize Rx filter. */
2159         netif_addr_lock_bh(bp->dev);
2160         bnx2x_set_rx_mode(bp->dev);
2161         netif_addr_unlock_bh(bp->dev);
2162
2163         /* re-read iscsi info */
2164         bnx2x_get_iscsi_info(bp);
2165         bnx2x_setup_cnic_irq_info(bp);
2166         bnx2x_setup_cnic_info(bp);
2167         bp->cnic_loaded = true;
2168         if (bp->state == BNX2X_STATE_OPEN)
2169                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2170
2171
2172         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2173
2174         return 0;
2175
2176 #ifndef BNX2X_STOP_ON_ERROR
2177 load_error_cnic2:
2178         /* Disable Timer scan */
2179         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2180
2181 load_error_cnic1:
2182         bnx2x_napi_disable_cnic(bp);
2183         /* Update the number of queues without the cnic queues */
2184         rc = bnx2x_set_real_num_queues(bp, 0);
2185         if (rc)
2186                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2187 load_error_cnic0:
2188         BNX2X_ERR("CNIC-related load failed\n");
2189         bnx2x_free_fp_mem_cnic(bp);
2190         bnx2x_free_mem_cnic(bp);
2191         return rc;
2192 #endif /* ! BNX2X_STOP_ON_ERROR */
2193 }
2194
2195
2196 /* must be called with rtnl_lock */
2197 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2198 {
2199         int port = BP_PORT(bp);
2200         u32 load_code;
2201         int i, rc;
2202
2203         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2204         DP(NETIF_MSG_IFUP,
2205            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2206
2207 #ifdef BNX2X_STOP_ON_ERROR
2208         if (unlikely(bp->panic)) {
2209                 BNX2X_ERR("Can't load NIC when there is panic\n");
2210                 return -EPERM;
2211         }
2212 #endif
2213
2214         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2215
2216         /* Set the initial link reported state to link down */
2217         bnx2x_acquire_phy_lock(bp);
2218         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2219         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2220                 &bp->last_reported_link.link_report_flags);
2221         bnx2x_release_phy_lock(bp);
2222
2223         /* must be called before memory allocation and HW init */
2224         bnx2x_ilt_set_info(bp);
2225
2226         /*
2227          * Zero fastpath structures preserving invariants like napi, which are
2228          * allocated only once, fp index, max_cos, bp pointer.
2229          * Also set fp->disable_tpa and txdata_ptr.
2230          */
2231         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2232         for_each_queue(bp, i)
2233                 bnx2x_bz_fp(bp, i);
2234         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2235                                   bp->num_cnic_queues) *
2236                                   sizeof(struct bnx2x_fp_txdata));
2237
2238         bp->fcoe_init = false;
2239
2240         /* Set the receive queues buffer size */
2241         bnx2x_set_rx_buf_size(bp);
2242
2243         if (bnx2x_alloc_mem(bp))
2244                 return -ENOMEM;
2245
2246         /* As long as bnx2x_alloc_mem() may possibly update
2247          * bp->num_queues, bnx2x_set_real_num_queues() should always
2248          * come after it. At this stage cnic queues are not counted.
2249          */
2250         rc = bnx2x_set_real_num_queues(bp, 0);
2251         if (rc) {
2252                 BNX2X_ERR("Unable to set real_num_queues\n");
2253                 LOAD_ERROR_EXIT(bp, load_error0);
2254         }
2255
2256         /* configure multi cos mappings in kernel.
2257          * this configuration may be overriden by a multi class queue discipline
2258          * or by a dcbx negotiation result.
2259          */
2260         bnx2x_setup_tc(bp->dev, bp->max_cos);
2261
2262         /* Add all NAPI objects */
2263         bnx2x_add_all_napi(bp);
2264         DP(NETIF_MSG_IFUP, "napi added\n");
2265         bnx2x_napi_enable(bp);
2266
2267         /* set pf load just before approaching the MCP */
2268         bnx2x_set_pf_load(bp);
2269
2270         /* Send LOAD_REQUEST command to MCP
2271          * Returns the type of LOAD command:
2272          * if it is the first port to be initialized
2273          * common blocks should be initialized, otherwise - not
2274          */
2275         if (!BP_NOMCP(bp)) {
2276                 /* init fw_seq */
2277                 bp->fw_seq =
2278                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2279                          DRV_MSG_SEQ_NUMBER_MASK);
2280                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2281
2282                 /* Get current FW pulse sequence */
2283                 bp->fw_drv_pulse_wr_seq =
2284                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2285                          DRV_PULSE_SEQ_MASK);
2286                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2287
2288                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2289                                              DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2290                 if (!load_code) {
2291                         BNX2X_ERR("MCP response failure, aborting\n");
2292                         rc = -EBUSY;
2293                         LOAD_ERROR_EXIT(bp, load_error1);
2294                 }
2295                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2296                         BNX2X_ERR("Driver load refused\n");
2297                         rc = -EBUSY; /* other port in diagnostic mode */
2298                         LOAD_ERROR_EXIT(bp, load_error1);
2299                 }
2300                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2301                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2302                         /* abort nic load if version mismatch */
2303                         if (!bnx2x_test_firmware_version(bp, true)) {
2304                                 rc = -EBUSY;
2305                                 LOAD_ERROR_EXIT(bp, load_error2);
2306                         }
2307                 }
2308
2309         } else {
2310                 int path = BP_PATH(bp);
2311
2312                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2313                    path, load_count[path][0], load_count[path][1],
2314                    load_count[path][2]);
2315                 load_count[path][0]++;
2316                 load_count[path][1 + port]++;
2317                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2318                    path, load_count[path][0], load_count[path][1],
2319                    load_count[path][2]);
2320                 if (load_count[path][0] == 1)
2321                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
2322                 else if (load_count[path][1 + port] == 1)
2323                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
2324                 else
2325                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
2326         }
2327
2328         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2329             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2330             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2331                 bp->port.pmf = 1;
2332                 /*
2333                  * We need the barrier to ensure the ordering between the
2334                  * writing to bp->port.pmf here and reading it from the
2335                  * bnx2x_periodic_task().
2336                  */
2337                 smp_mb();
2338         } else
2339                 bp->port.pmf = 0;
2340
2341         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
2342
2343         /* Init Function state controlling object */
2344         bnx2x__init_func_obj(bp);
2345
2346         /* Initialize HW */
2347         rc = bnx2x_init_hw(bp, load_code);
2348         if (rc) {
2349                 BNX2X_ERR("HW init failed, aborting\n");
2350                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2351                 LOAD_ERROR_EXIT(bp, load_error2);
2352         }
2353
2354         /* Connect to IRQs */
2355         rc = bnx2x_setup_irqs(bp);
2356         if (rc) {
2357                 BNX2X_ERR("IRQs setup failed\n");
2358                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2359                 LOAD_ERROR_EXIT(bp, load_error2);
2360         }
2361
2362         /* Setup NIC internals and enable interrupts */
2363         bnx2x_nic_init(bp, load_code);
2364
2365         /* Init per-function objects */
2366         bnx2x_init_bp_objs(bp);
2367
2368         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2369             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2370             (bp->common.shmem2_base)) {
2371                 if (SHMEM2_HAS(bp, dcc_support))
2372                         SHMEM2_WR(bp, dcc_support,
2373                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2374                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2375                 if (SHMEM2_HAS(bp, afex_driver_support))
2376                         SHMEM2_WR(bp, afex_driver_support,
2377                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2378         }
2379
2380         /* Set AFEX default VLAN tag to an invalid value */
2381         bp->afex_def_vlan_tag = -1;
2382
2383         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2384         rc = bnx2x_func_start(bp);
2385         if (rc) {
2386                 BNX2X_ERR("Function start failed!\n");
2387                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2388                 LOAD_ERROR_EXIT(bp, load_error3);
2389         }
2390
2391         /* Send LOAD_DONE command to MCP */
2392         if (!BP_NOMCP(bp)) {
2393                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2394                 if (!load_code) {
2395                         BNX2X_ERR("MCP response failure, aborting\n");
2396                         rc = -EBUSY;
2397                         LOAD_ERROR_EXIT(bp, load_error3);
2398                 }
2399         }
2400
2401         rc = bnx2x_setup_leading(bp);
2402         if (rc) {
2403                 BNX2X_ERR("Setup leading failed!\n");
2404                 LOAD_ERROR_EXIT(bp, load_error3);
2405         }
2406
2407         for_each_nondefault_eth_queue(bp, i) {
2408                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2409                 if (rc) {
2410                         BNX2X_ERR("Queue setup failed\n");
2411                         LOAD_ERROR_EXIT(bp, load_error3);
2412                 }
2413         }
2414
2415         rc = bnx2x_init_rss_pf(bp);
2416         if (rc) {
2417                 BNX2X_ERR("PF RSS init failed\n");
2418                 LOAD_ERROR_EXIT(bp, load_error3);
2419         }
2420
2421         /* Now when Clients are configured we are ready to work */
2422         bp->state = BNX2X_STATE_OPEN;
2423
2424         /* Configure a ucast MAC */
2425         rc = bnx2x_set_eth_mac(bp, true);
2426         if (rc) {
2427                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2428                 LOAD_ERROR_EXIT(bp, load_error3);
2429         }
2430
2431         if (bp->pending_max) {
2432                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2433                 bp->pending_max = 0;
2434         }
2435
2436         if (bp->port.pmf)
2437                 bnx2x_initial_phy_init(bp, load_mode);
2438         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2439
2440         /* Start fast path */
2441
2442         /* Initialize Rx filter. */
2443         netif_addr_lock_bh(bp->dev);
2444         bnx2x_set_rx_mode(bp->dev);
2445         netif_addr_unlock_bh(bp->dev);
2446
2447         /* Start the Tx */
2448         switch (load_mode) {
2449         case LOAD_NORMAL:
2450                 /* Tx queue should be only reenabled */
2451                 netif_tx_wake_all_queues(bp->dev);
2452                 break;
2453
2454         case LOAD_OPEN:
2455                 netif_tx_start_all_queues(bp->dev);
2456                 smp_mb__after_clear_bit();
2457                 break;
2458
2459         case LOAD_DIAG:
2460         case LOAD_LOOPBACK_EXT:
2461                 bp->state = BNX2X_STATE_DIAG;
2462                 break;
2463
2464         default:
2465                 break;
2466         }
2467
2468         if (bp->port.pmf)
2469                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2470         else
2471                 bnx2x__link_status_update(bp);
2472
2473         /* start the timer */
2474         mod_timer(&bp->timer, jiffies + bp->current_interval);
2475
2476         if (CNIC_ENABLED(bp))
2477                 bnx2x_load_cnic(bp);
2478
2479         /* mark driver is loaded in shmem2 */
2480         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2481                 u32 val;
2482                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2483                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2484                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2485                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2486         }
2487
2488         /* Wait for all pending SP commands to complete */
2489         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2490                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2491                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2492                 return -EBUSY;
2493         }
2494
2495         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2496         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2497                 bnx2x_dcbx_init(bp, false);
2498
2499         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2500
2501         return 0;
2502
2503 #ifndef BNX2X_STOP_ON_ERROR
2504 load_error3:
2505         bnx2x_int_disable_sync(bp, 1);
2506
2507         /* Clean queueable objects */
2508         bnx2x_squeeze_objects(bp);
2509
2510         /* Free SKBs, SGEs, TPA pool and driver internals */
2511         bnx2x_free_skbs(bp);
2512         for_each_rx_queue(bp, i)
2513                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2514
2515         /* Release IRQs */
2516         bnx2x_free_irq(bp);
2517 load_error2:
2518         if (!BP_NOMCP(bp)) {
2519                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2520                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2521         }
2522
2523         bp->port.pmf = 0;
2524 load_error1:
2525         bnx2x_napi_disable(bp);
2526         /* clear pf_load status, as it was already set */
2527         bnx2x_clear_pf_load(bp);
2528 load_error0:
2529         bnx2x_free_mem(bp);
2530
2531         return rc;
2532 #endif /* ! BNX2X_STOP_ON_ERROR */
2533 }
2534
2535 /* must be called with rtnl_lock */
2536 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2537 {
2538         int i;
2539         bool global = false;
2540
2541         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2542
2543         /* mark driver is unloaded in shmem2 */
2544         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2545                 u32 val;
2546                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2547                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2548                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2549         }
2550
2551         if ((bp->state == BNX2X_STATE_CLOSED) ||
2552             (bp->state == BNX2X_STATE_ERROR)) {
2553                 /* We can get here if the driver has been unloaded
2554                  * during parity error recovery and is either waiting for a
2555                  * leader to complete or for other functions to unload and
2556                  * then ifdown has been issued. In this case we want to
2557                  * unload and let other functions to complete a recovery
2558                  * process.
2559                  */
2560                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2561                 bp->is_leader = 0;
2562                 bnx2x_release_leader_lock(bp);
2563                 smp_mb();
2564
2565                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2566                 BNX2X_ERR("Can't unload in closed or error state\n");
2567                 return -EINVAL;
2568         }
2569
2570         /*
2571          * It's important to set the bp->state to the value different from
2572          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2573          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2574          */
2575         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2576         smp_mb();
2577
2578         if (CNIC_LOADED(bp))
2579                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2580
2581         /* Stop Tx */
2582         bnx2x_tx_disable(bp);
2583         netdev_reset_tc(bp->dev);
2584
2585         bp->rx_mode = BNX2X_RX_MODE_NONE;
2586
2587         del_timer_sync(&bp->timer);
2588
2589         /* Set ALWAYS_ALIVE bit in shmem */
2590         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2591
2592         bnx2x_drv_pulse(bp);
2593
2594         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2595         bnx2x_save_statistics(bp);
2596
2597         /* Cleanup the chip if needed */
2598         if (unload_mode != UNLOAD_RECOVERY)
2599                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2600         else {
2601                 /* Send the UNLOAD_REQUEST to the MCP */
2602                 bnx2x_send_unload_req(bp, unload_mode);
2603
2604                 /*
2605                  * Prevent transactions to host from the functions on the
2606                  * engine that doesn't reset global blocks in case of global
2607                  * attention once gloabl blocks are reset and gates are opened
2608                  * (the engine which leader will perform the recovery
2609                  * last).
2610                  */
2611                 if (!CHIP_IS_E1x(bp))
2612                         bnx2x_pf_disable(bp);
2613
2614                 /* Disable HW interrupts, NAPI */
2615                 bnx2x_netif_stop(bp, 1);
2616                 /* Delete all NAPI objects */
2617                 bnx2x_del_all_napi(bp);
2618                 if (CNIC_LOADED(bp))
2619                         bnx2x_del_all_napi_cnic(bp);
2620                 /* Release IRQs */
2621                 bnx2x_free_irq(bp);
2622
2623                 /* Report UNLOAD_DONE to MCP */
2624                 bnx2x_send_unload_done(bp, false);
2625         }
2626
2627         /*
2628          * At this stage no more interrupts will arrive so we may safly clean
2629          * the queueable objects here in case they failed to get cleaned so far.
2630          */
2631         bnx2x_squeeze_objects(bp);
2632
2633         /* There should be no more pending SP commands at this stage */
2634         bp->sp_state = 0;
2635
2636         bp->port.pmf = 0;
2637
2638         /* Free SKBs, SGEs, TPA pool and driver internals */
2639         bnx2x_free_skbs(bp);
2640         if (CNIC_LOADED(bp))
2641                 bnx2x_free_skbs_cnic(bp);
2642         for_each_rx_queue(bp, i)
2643                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2644
2645         if (CNIC_LOADED(bp)) {
2646                 bnx2x_free_fp_mem_cnic(bp);
2647                 bnx2x_free_mem_cnic(bp);
2648         }
2649         bnx2x_free_mem(bp);
2650
2651         bp->state = BNX2X_STATE_CLOSED;
2652         bp->cnic_loaded = false;
2653
2654         /* Check if there are pending parity attentions. If there are - set
2655          * RECOVERY_IN_PROGRESS.
2656          */
2657         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2658                 bnx2x_set_reset_in_progress(bp);
2659
2660                 /* Set RESET_IS_GLOBAL if needed */
2661                 if (global)
2662                         bnx2x_set_reset_global(bp);
2663         }
2664
2665
2666         /* The last driver must disable a "close the gate" if there is no
2667          * parity attention or "process kill" pending.
2668          */
2669         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2670                 bnx2x_disable_close_the_gate(bp);
2671
2672         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2673
2674         return 0;
2675 }
2676
2677 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2678 {
2679         u16 pmcsr;
2680
2681         /* If there is no power capability, silently succeed */
2682         if (!bp->pm_cap) {
2683                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2684                 return 0;
2685         }
2686
2687         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2688
2689         switch (state) {
2690         case PCI_D0:
2691                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2692                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2693                                        PCI_PM_CTRL_PME_STATUS));
2694
2695                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2696                         /* delay required during transition out of D3hot */
2697                         msleep(20);
2698                 break;
2699
2700         case PCI_D3hot:
2701                 /* If there are other clients above don't
2702                    shut down the power */
2703                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2704                         return 0;
2705                 /* Don't shut down the power for emulation and FPGA */
2706                 if (CHIP_REV_IS_SLOW(bp))
2707                         return 0;
2708
2709                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2710                 pmcsr |= 3;
2711
2712                 if (bp->wol)
2713                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2714
2715                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2716                                       pmcsr);
2717
2718                 /* No more memory access after this point until
2719                 * device is brought back to D0.
2720                 */
2721                 break;
2722
2723         default:
2724                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2725                 return -EINVAL;
2726         }
2727         return 0;
2728 }
2729
2730 /*
2731  * net_device service functions
2732  */
2733 int bnx2x_poll(struct napi_struct *napi, int budget)
2734 {
2735         int work_done = 0;
2736         u8 cos;
2737         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2738                                                  napi);
2739         struct bnx2x *bp = fp->bp;
2740
2741         while (1) {
2742 #ifdef BNX2X_STOP_ON_ERROR
2743                 if (unlikely(bp->panic)) {
2744                         napi_complete(napi);
2745                         return 0;
2746                 }
2747 #endif
2748
2749                 for_each_cos_in_tx_queue(fp, cos)
2750                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
2751                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
2752
2753
2754                 if (bnx2x_has_rx_work(fp)) {
2755                         work_done += bnx2x_rx_int(fp, budget - work_done);
2756
2757                         /* must not complete if we consumed full budget */
2758                         if (work_done >= budget)
2759                                 break;
2760                 }
2761
2762                 /* Fall out from the NAPI loop if needed */
2763                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2764
2765                         /* No need to update SB for FCoE L2 ring as long as
2766                          * it's connected to the default SB and the SB
2767                          * has been updated when NAPI was scheduled.
2768                          */
2769                         if (IS_FCOE_FP(fp)) {
2770                                 napi_complete(napi);
2771                                 break;
2772                         }
2773                         bnx2x_update_fpsb_idx(fp);
2774                         /* bnx2x_has_rx_work() reads the status block,
2775                          * thus we need to ensure that status block indices
2776                          * have been actually read (bnx2x_update_fpsb_idx)
2777                          * prior to this check (bnx2x_has_rx_work) so that
2778                          * we won't write the "newer" value of the status block
2779                          * to IGU (if there was a DMA right after
2780                          * bnx2x_has_rx_work and if there is no rmb, the memory
2781                          * reading (bnx2x_update_fpsb_idx) may be postponed
2782                          * to right before bnx2x_ack_sb). In this case there
2783                          * will never be another interrupt until there is
2784                          * another update of the status block, while there
2785                          * is still unhandled work.
2786                          */
2787                         rmb();
2788
2789                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2790                                 napi_complete(napi);
2791                                 /* Re-enable interrupts */
2792                                 DP(NETIF_MSG_RX_STATUS,
2793                                    "Update index to %d\n", fp->fp_hc_idx);
2794                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2795                                              le16_to_cpu(fp->fp_hc_idx),
2796                                              IGU_INT_ENABLE, 1);
2797                                 break;
2798                         }
2799                 }
2800         }
2801
2802         return work_done;
2803 }
2804
2805 /* we split the first BD into headers and data BDs
2806  * to ease the pain of our fellow microcode engineers
2807  * we use one mapping for both BDs
2808  */
2809 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2810                                    struct bnx2x_fp_txdata *txdata,
2811                                    struct sw_tx_bd *tx_buf,
2812                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2813                                    u16 bd_prod, int nbd)
2814 {
2815         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2816         struct eth_tx_bd *d_tx_bd;
2817         dma_addr_t mapping;
2818         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2819
2820         /* first fix first BD */
2821         h_tx_bd->nbd = cpu_to_le16(nbd);
2822         h_tx_bd->nbytes = cpu_to_le16(hlen);
2823
2824         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2825            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2826
2827         /* now get a new data BD
2828          * (after the pbd) and fill it */
2829         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2830         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2831
2832         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2833                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2834
2835         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2836         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2837         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2838
2839         /* this marks the BD as one that has no individual mapping */
2840         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2841
2842         DP(NETIF_MSG_TX_QUEUED,
2843            "TSO split data size is %d (%x:%x)\n",
2844            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2845
2846         /* update tx_bd */
2847         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2848
2849         return bd_prod;
2850 }
2851
2852 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2853 {
2854         if (fix > 0)
2855                 csum = (u16) ~csum_fold(csum_sub(csum,
2856                                 csum_partial(t_header - fix, fix, 0)));
2857
2858         else if (fix < 0)
2859                 csum = (u16) ~csum_fold(csum_add(csum,
2860                                 csum_partial(t_header, -fix, 0)));
2861
2862         return swab16(csum);
2863 }
2864
2865 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2866 {
2867         u32 rc;
2868
2869         if (skb->ip_summed != CHECKSUM_PARTIAL)
2870                 rc = XMIT_PLAIN;
2871
2872         else {
2873                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2874                         rc = XMIT_CSUM_V6;
2875                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2876                                 rc |= XMIT_CSUM_TCP;
2877
2878                 } else {
2879                         rc = XMIT_CSUM_V4;
2880                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2881                                 rc |= XMIT_CSUM_TCP;
2882                 }
2883         }
2884
2885         if (skb_is_gso_v6(skb))
2886                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2887         else if (skb_is_gso(skb))
2888                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2889
2890         return rc;
2891 }
2892
2893 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2894 /* check if packet requires linearization (packet is too fragmented)
2895    no need to check fragmentation if page size > 8K (there will be no
2896    violation to FW restrictions) */
2897 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2898                              u32 xmit_type)
2899 {
2900         int to_copy = 0;
2901         int hlen = 0;
2902         int first_bd_sz = 0;
2903
2904         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2905         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2906
2907                 if (xmit_type & XMIT_GSO) {
2908                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2909                         /* Check if LSO packet needs to be copied:
2910                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2911                         int wnd_size = MAX_FETCH_BD - 3;
2912                         /* Number of windows to check */
2913                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2914                         int wnd_idx = 0;
2915                         int frag_idx = 0;
2916                         u32 wnd_sum = 0;
2917
2918                         /* Headers length */
2919                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2920                                 tcp_hdrlen(skb);
2921
2922                         /* Amount of data (w/o headers) on linear part of SKB*/
2923                         first_bd_sz = skb_headlen(skb) - hlen;
2924
2925                         wnd_sum  = first_bd_sz;
2926
2927                         /* Calculate the first sum - it's special */
2928                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2929                                 wnd_sum +=
2930                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2931
2932                         /* If there was data on linear skb data - check it */
2933                         if (first_bd_sz > 0) {
2934                                 if (unlikely(wnd_sum < lso_mss)) {
2935                                         to_copy = 1;
2936                                         goto exit_lbl;
2937                                 }
2938
2939                                 wnd_sum -= first_bd_sz;
2940                         }
2941
2942                         /* Others are easier: run through the frag list and
2943                            check all windows */
2944                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2945                                 wnd_sum +=
2946                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2947
2948                                 if (unlikely(wnd_sum < lso_mss)) {
2949                                         to_copy = 1;
2950                                         break;
2951                                 }
2952                                 wnd_sum -=
2953                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2954                         }
2955                 } else {
2956                         /* in non-LSO too fragmented packet should always
2957                            be linearized */
2958                         to_copy = 1;
2959                 }
2960         }
2961
2962 exit_lbl:
2963         if (unlikely(to_copy))
2964                 DP(NETIF_MSG_TX_QUEUED,
2965                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2966                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2967                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2968
2969         return to_copy;
2970 }
2971 #endif
2972
2973 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2974                                         u32 xmit_type)
2975 {
2976         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2977                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2978                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2979         if ((xmit_type & XMIT_GSO_V6) &&
2980             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2981                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2982 }
2983
2984 /**
2985  * bnx2x_set_pbd_gso - update PBD in GSO case.
2986  *
2987  * @skb:        packet skb
2988  * @pbd:        parse BD
2989  * @xmit_type:  xmit flags
2990  */
2991 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2992                                      struct eth_tx_parse_bd_e1x *pbd,
2993                                      u32 xmit_type)
2994 {
2995         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2996         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2997         pbd->tcp_flags = pbd_tcp_flags(skb);
2998
2999         if (xmit_type & XMIT_GSO_V4) {
3000                 pbd->ip_id = swab16(ip_hdr(skb)->id);
3001                 pbd->tcp_pseudo_csum =
3002                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3003                                                   ip_hdr(skb)->daddr,
3004                                                   0, IPPROTO_TCP, 0));
3005
3006         } else
3007                 pbd->tcp_pseudo_csum =
3008                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3009                                                 &ipv6_hdr(skb)->daddr,
3010                                                 0, IPPROTO_TCP, 0));
3011
3012         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
3013 }
3014
3015 /**
3016  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3017  *
3018  * @bp:                 driver handle
3019  * @skb:                packet skb
3020  * @parsing_data:       data to be updated
3021  * @xmit_type:          xmit flags
3022  *
3023  * 57712 related
3024  */
3025 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3026         u32 *parsing_data, u32 xmit_type)
3027 {