f771ddfba646b0a773e89f3365b8add151087836
[~shefty/rdma-dev.git] / drivers / net / ethernet / broadcom / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2  *
3  * Copyright (c) 2007-2012 Broadcom Corporation
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation.
8  *
9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10  * Written by: Eliezer Tamir
11  * Based on code from Michael Chan's bnx2 driver
12  * UDP CSUM errata workaround by Arik Gendelman
13  * Slowpath and fastpath rework by Vladislav Zolotarov
14  * Statistics and Link management by Yitchak Gertner
15  *
16  */
17
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/etherdevice.h>
21 #include <linux/if_vlan.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <net/ipv6.h>
25 #include <net/ip6_checksum.h>
26 #include <linux/prefetch.h>
27 #include "bnx2x_cmn.h"
28 #include "bnx2x_init.h"
29 #include "bnx2x_sp.h"
30
31
32
33 /**
34  * bnx2x_move_fp - move content of the fastpath structure.
35  *
36  * @bp:         driver handle
37  * @from:       source FP index
38  * @to:         destination FP index
39  *
40  * Makes sure the contents of the bp->fp[to].napi is kept
41  * intact. This is done by first copying the napi struct from
42  * the target to the source, and then mem copying the entire
43  * source onto the target. Update txdata pointers and related
44  * content.
45  */
46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
47 {
48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
54         int old_max_eth_txqs, new_max_eth_txqs;
55         int old_txdata_index = 0, new_txdata_index = 0;
56
57         /* Copy the NAPI object as it has been already initialized */
58         from_fp->napi = to_fp->napi;
59
60         /* Move bnx2x_fastpath contents */
61         memcpy(to_fp, from_fp, sizeof(*to_fp));
62         to_fp->index = to;
63
64         /* move sp_objs contents as well, as their indices match fp ones */
65         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
66
67         /* move fp_stats contents as well, as their indices match fp ones */
68         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
69
70         /* Update txdata pointers in fp and move txdata content accordingly:
71          * Each fp consumes 'max_cos' txdata structures, so the index should be
72          * decremented by max_cos x delta.
73          */
74
75         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
76         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
77                                 (bp)->max_cos;
78         if (from == FCOE_IDX(bp)) {
79                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
80                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
81         }
82
83         memcpy(&bp->bnx2x_txq[new_txdata_index],
84                &bp->bnx2x_txq[old_txdata_index],
85                sizeof(struct bnx2x_fp_txdata));
86         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
87 }
88
89 /**
90  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
91  *
92  * @bp: driver handle
93  * @delta:      number of eth queues which were not allocated
94  */
95 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
96 {
97         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
98
99         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
100          * backward along the array could cause memory to be overriden
101          */
102         for (cos = 1; cos < bp->max_cos; cos++) {
103                 for (i = 0; i < old_eth_num - delta; i++) {
104                         struct bnx2x_fastpath *fp = &bp->fp[i];
105                         int new_idx = cos * (old_eth_num - delta) + i;
106
107                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
108                                sizeof(struct bnx2x_fp_txdata));
109                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
110                 }
111         }
112 }
113
114 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
115
116 /* free skb in the packet ring at pos idx
117  * return idx of last bd freed
118  */
119 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
120                              u16 idx, unsigned int *pkts_compl,
121                              unsigned int *bytes_compl)
122 {
123         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
124         struct eth_tx_start_bd *tx_start_bd;
125         struct eth_tx_bd *tx_data_bd;
126         struct sk_buff *skb = tx_buf->skb;
127         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
128         int nbd;
129
130         /* prefetch skb end pointer to speedup dev_kfree_skb() */
131         prefetch(&skb->end);
132
133         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
134            txdata->txq_index, idx, tx_buf, skb);
135
136         /* unmap first bd */
137         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
138         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
139                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
140
141
142         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
143 #ifdef BNX2X_STOP_ON_ERROR
144         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
145                 BNX2X_ERR("BAD nbd!\n");
146                 bnx2x_panic();
147         }
148 #endif
149         new_cons = nbd + tx_buf->first_bd;
150
151         /* Get the next bd */
152         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
153
154         /* Skip a parse bd... */
155         --nbd;
156         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
157
158         /* ...and the TSO split header bd since they have no mapping */
159         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
160                 --nbd;
161                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
162         }
163
164         /* now free frags */
165         while (nbd > 0) {
166
167                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
168                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
169                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
170                 if (--nbd)
171                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
172         }
173
174         /* release skb */
175         WARN_ON(!skb);
176         if (likely(skb)) {
177                 (*pkts_compl)++;
178                 (*bytes_compl) += skb->len;
179         }
180
181         dev_kfree_skb_any(skb);
182         tx_buf->first_bd = 0;
183         tx_buf->skb = NULL;
184
185         return new_cons;
186 }
187
188 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
189 {
190         struct netdev_queue *txq;
191         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
192         unsigned int pkts_compl = 0, bytes_compl = 0;
193
194 #ifdef BNX2X_STOP_ON_ERROR
195         if (unlikely(bp->panic))
196                 return -1;
197 #endif
198
199         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
200         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
201         sw_cons = txdata->tx_pkt_cons;
202
203         while (sw_cons != hw_cons) {
204                 u16 pkt_cons;
205
206                 pkt_cons = TX_BD(sw_cons);
207
208                 DP(NETIF_MSG_TX_DONE,
209                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
210                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
211
212                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
213                     &pkts_compl, &bytes_compl);
214
215                 sw_cons++;
216         }
217
218         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
219
220         txdata->tx_pkt_cons = sw_cons;
221         txdata->tx_bd_cons = bd_cons;
222
223         /* Need to make the tx_bd_cons update visible to start_xmit()
224          * before checking for netif_tx_queue_stopped().  Without the
225          * memory barrier, there is a small possibility that
226          * start_xmit() will miss it and cause the queue to be stopped
227          * forever.
228          * On the other hand we need an rmb() here to ensure the proper
229          * ordering of bit testing in the following
230          * netif_tx_queue_stopped(txq) call.
231          */
232         smp_mb();
233
234         if (unlikely(netif_tx_queue_stopped(txq))) {
235                 /* Taking tx_lock() is needed to prevent reenabling the queue
236                  * while it's empty. This could have happen if rx_action() gets
237                  * suspended in bnx2x_tx_int() after the condition before
238                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
239                  *
240                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
241                  * sends some packets consuming the whole queue again->
242                  * stops the queue
243                  */
244
245                 __netif_tx_lock(txq, smp_processor_id());
246
247                 if ((netif_tx_queue_stopped(txq)) &&
248                     (bp->state == BNX2X_STATE_OPEN) &&
249                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
250                         netif_tx_wake_queue(txq);
251
252                 __netif_tx_unlock(txq);
253         }
254         return 0;
255 }
256
257 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
258                                              u16 idx)
259 {
260         u16 last_max = fp->last_max_sge;
261
262         if (SUB_S16(idx, last_max) > 0)
263                 fp->last_max_sge = idx;
264 }
265
266 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
267                                          u16 sge_len,
268                                          struct eth_end_agg_rx_cqe *cqe)
269 {
270         struct bnx2x *bp = fp->bp;
271         u16 last_max, last_elem, first_elem;
272         u16 delta = 0;
273         u16 i;
274
275         if (!sge_len)
276                 return;
277
278         /* First mark all used pages */
279         for (i = 0; i < sge_len; i++)
280                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
281                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
282
283         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
284            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
285
286         /* Here we assume that the last SGE index is the biggest */
287         prefetch((void *)(fp->sge_mask));
288         bnx2x_update_last_max_sge(fp,
289                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
290
291         last_max = RX_SGE(fp->last_max_sge);
292         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
293         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
294
295         /* If ring is not full */
296         if (last_elem + 1 != first_elem)
297                 last_elem++;
298
299         /* Now update the prod */
300         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
301                 if (likely(fp->sge_mask[i]))
302                         break;
303
304                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
305                 delta += BIT_VEC64_ELEM_SZ;
306         }
307
308         if (delta > 0) {
309                 fp->rx_sge_prod += delta;
310                 /* clear page-end entries */
311                 bnx2x_clear_sge_mask_next_elems(fp);
312         }
313
314         DP(NETIF_MSG_RX_STATUS,
315            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
316            fp->last_max_sge, fp->rx_sge_prod);
317 }
318
319 /* Set Toeplitz hash value in the skb using the value from the
320  * CQE (calculated by HW).
321  */
322 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
323                             const struct eth_fast_path_rx_cqe *cqe,
324                             bool *l4_rxhash)
325 {
326         /* Set Toeplitz hash from CQE */
327         if ((bp->dev->features & NETIF_F_RXHASH) &&
328             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
329                 enum eth_rss_hash_type htype;
330
331                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
332                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
333                              (htype == TCP_IPV6_HASH_TYPE);
334                 return le32_to_cpu(cqe->rss_hash_result);
335         }
336         *l4_rxhash = false;
337         return 0;
338 }
339
340 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
341                             u16 cons, u16 prod,
342                             struct eth_fast_path_rx_cqe *cqe)
343 {
344         struct bnx2x *bp = fp->bp;
345         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
346         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
347         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
348         dma_addr_t mapping;
349         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
350         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
351
352         /* print error if current state != stop */
353         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
354                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
355
356         /* Try to map an empty data buffer from the aggregation info  */
357         mapping = dma_map_single(&bp->pdev->dev,
358                                  first_buf->data + NET_SKB_PAD,
359                                  fp->rx_buf_size, DMA_FROM_DEVICE);
360         /*
361          *  ...if it fails - move the skb from the consumer to the producer
362          *  and set the current aggregation state as ERROR to drop it
363          *  when TPA_STOP arrives.
364          */
365
366         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
367                 /* Move the BD from the consumer to the producer */
368                 bnx2x_reuse_rx_data(fp, cons, prod);
369                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
370                 return;
371         }
372
373         /* move empty data from pool to prod */
374         prod_rx_buf->data = first_buf->data;
375         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
376         /* point prod_bd to new data */
377         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
378         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
379
380         /* move partial skb from cons to pool (don't unmap yet) */
381         *first_buf = *cons_rx_buf;
382
383         /* mark bin state as START */
384         tpa_info->parsing_flags =
385                 le16_to_cpu(cqe->pars_flags.flags);
386         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
387         tpa_info->tpa_state = BNX2X_TPA_START;
388         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
389         tpa_info->placement_offset = cqe->placement_offset;
390         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
391         if (fp->mode == TPA_MODE_GRO) {
392                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
393                 tpa_info->full_page =
394                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
395                 tpa_info->gro_size = gro_size;
396         }
397
398 #ifdef BNX2X_STOP_ON_ERROR
399         fp->tpa_queue_used |= (1 << queue);
400 #ifdef _ASM_GENERIC_INT_L64_H
401         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
402 #else
403         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
404 #endif
405            fp->tpa_queue_used);
406 #endif
407 }
408
409 /* Timestamp option length allowed for TPA aggregation:
410  *
411  *              nop nop kind length echo val
412  */
413 #define TPA_TSTAMP_OPT_LEN      12
414 /**
415  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
416  *
417  * @bp:                 driver handle
418  * @parsing_flags:      parsing flags from the START CQE
419  * @len_on_bd:          total length of the first packet for the
420  *                      aggregation.
421  *
422  * Approximate value of the MSS for this aggregation calculated using
423  * the first packet of it.
424  */
425 static u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
426                              u16 len_on_bd)
427 {
428         /*
429          * TPA arrgregation won't have either IP options or TCP options
430          * other than timestamp or IPv6 extension headers.
431          */
432         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
433
434         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
435             PRS_FLAG_OVERETH_IPV6)
436                 hdrs_len += sizeof(struct ipv6hdr);
437         else /* IPv4 */
438                 hdrs_len += sizeof(struct iphdr);
439
440
441         /* Check if there was a TCP timestamp, if there is it's will
442          * always be 12 bytes length: nop nop kind length echo val.
443          *
444          * Otherwise FW would close the aggregation.
445          */
446         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
447                 hdrs_len += TPA_TSTAMP_OPT_LEN;
448
449         return len_on_bd - hdrs_len;
450 }
451
452 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
453                               struct bnx2x_fastpath *fp, u16 index)
454 {
455         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
456         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
457         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
458         dma_addr_t mapping;
459
460         if (unlikely(page == NULL)) {
461                 BNX2X_ERR("Can't alloc sge\n");
462                 return -ENOMEM;
463         }
464
465         mapping = dma_map_page(&bp->pdev->dev, page, 0,
466                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
467         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
468                 __free_pages(page, PAGES_PER_SGE_SHIFT);
469                 BNX2X_ERR("Can't map sge\n");
470                 return -ENOMEM;
471         }
472
473         sw_buf->page = page;
474         dma_unmap_addr_set(sw_buf, mapping, mapping);
475
476         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
477         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
478
479         return 0;
480 }
481
482 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
483                                struct bnx2x_agg_info *tpa_info,
484                                u16 pages,
485                                struct sk_buff *skb,
486                                struct eth_end_agg_rx_cqe *cqe,
487                                u16 cqe_idx)
488 {
489         struct sw_rx_page *rx_pg, old_rx_pg;
490         u32 i, frag_len, frag_size;
491         int err, j, frag_id = 0;
492         u16 len_on_bd = tpa_info->len_on_bd;
493         u16 full_page = 0, gro_size = 0;
494
495         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
496
497         if (fp->mode == TPA_MODE_GRO) {
498                 gro_size = tpa_info->gro_size;
499                 full_page = tpa_info->full_page;
500         }
501
502         /* This is needed in order to enable forwarding support */
503         if (frag_size) {
504                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
505                                         tpa_info->parsing_flags, len_on_bd);
506
507                 /* set for GRO */
508                 if (fp->mode == TPA_MODE_GRO)
509                         skb_shinfo(skb)->gso_type =
510                             (GET_FLAG(tpa_info->parsing_flags,
511                                       PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
512                                                 PRS_FLAG_OVERETH_IPV6) ?
513                                 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
514         }
515
516
517 #ifdef BNX2X_STOP_ON_ERROR
518         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
519                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
520                           pages, cqe_idx);
521                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
522                 bnx2x_panic();
523                 return -EINVAL;
524         }
525 #endif
526
527         /* Run through the SGL and compose the fragmented skb */
528         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
529                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
530
531                 /* FW gives the indices of the SGE as if the ring is an array
532                    (meaning that "next" element will consume 2 indices) */
533                 if (fp->mode == TPA_MODE_GRO)
534                         frag_len = min_t(u32, frag_size, (u32)full_page);
535                 else /* LRO */
536                         frag_len = min_t(u32, frag_size,
537                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
538
539                 rx_pg = &fp->rx_page_ring[sge_idx];
540                 old_rx_pg = *rx_pg;
541
542                 /* If we fail to allocate a substitute page, we simply stop
543                    where we are and drop the whole packet */
544                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
545                 if (unlikely(err)) {
546                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
547                         return err;
548                 }
549
550                 /* Unmap the page as we r going to pass it to the stack */
551                 dma_unmap_page(&bp->pdev->dev,
552                                dma_unmap_addr(&old_rx_pg, mapping),
553                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
554                 /* Add one frag and update the appropriate fields in the skb */
555                 if (fp->mode == TPA_MODE_LRO)
556                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
557                 else { /* GRO */
558                         int rem;
559                         int offset = 0;
560                         for (rem = frag_len; rem > 0; rem -= gro_size) {
561                                 int len = rem > gro_size ? gro_size : rem;
562                                 skb_fill_page_desc(skb, frag_id++,
563                                                    old_rx_pg.page, offset, len);
564                                 if (offset)
565                                         get_page(old_rx_pg.page);
566                                 offset += len;
567                         }
568                 }
569
570                 skb->data_len += frag_len;
571                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
572                 skb->len += frag_len;
573
574                 frag_size -= frag_len;
575         }
576
577         return 0;
578 }
579
580 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
581 {
582         if (fp->rx_frag_size)
583                 put_page(virt_to_head_page(data));
584         else
585                 kfree(data);
586 }
587
588 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
589 {
590         if (fp->rx_frag_size)
591                 return netdev_alloc_frag(fp->rx_frag_size);
592
593         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
594 }
595
596
597 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
598                            struct bnx2x_agg_info *tpa_info,
599                            u16 pages,
600                            struct eth_end_agg_rx_cqe *cqe,
601                            u16 cqe_idx)
602 {
603         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
604         u8 pad = tpa_info->placement_offset;
605         u16 len = tpa_info->len_on_bd;
606         struct sk_buff *skb = NULL;
607         u8 *new_data, *data = rx_buf->data;
608         u8 old_tpa_state = tpa_info->tpa_state;
609
610         tpa_info->tpa_state = BNX2X_TPA_STOP;
611
612         /* If we there was an error during the handling of the TPA_START -
613          * drop this aggregation.
614          */
615         if (old_tpa_state == BNX2X_TPA_ERROR)
616                 goto drop;
617
618         /* Try to allocate the new data */
619         new_data = bnx2x_frag_alloc(fp);
620         /* Unmap skb in the pool anyway, as we are going to change
621            pool entry status to BNX2X_TPA_STOP even if new skb allocation
622            fails. */
623         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
624                          fp->rx_buf_size, DMA_FROM_DEVICE);
625         if (likely(new_data))
626                 skb = build_skb(data, fp->rx_frag_size);
627
628         if (likely(skb)) {
629 #ifdef BNX2X_STOP_ON_ERROR
630                 if (pad + len > fp->rx_buf_size) {
631                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
632                                   pad, len, fp->rx_buf_size);
633                         bnx2x_panic();
634                         return;
635                 }
636 #endif
637
638                 skb_reserve(skb, pad + NET_SKB_PAD);
639                 skb_put(skb, len);
640                 skb->rxhash = tpa_info->rxhash;
641                 skb->l4_rxhash = tpa_info->l4_rxhash;
642
643                 skb->protocol = eth_type_trans(skb, bp->dev);
644                 skb->ip_summed = CHECKSUM_UNNECESSARY;
645
646                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
647                                          skb, cqe, cqe_idx)) {
648                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
649                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
650                         napi_gro_receive(&fp->napi, skb);
651                 } else {
652                         DP(NETIF_MSG_RX_STATUS,
653                            "Failed to allocate new pages - dropping packet!\n");
654                         dev_kfree_skb_any(skb);
655                 }
656
657
658                 /* put new data in bin */
659                 rx_buf->data = new_data;
660
661                 return;
662         }
663         bnx2x_frag_free(fp, new_data);
664 drop:
665         /* drop the packet and keep the buffer in the bin */
666         DP(NETIF_MSG_RX_STATUS,
667            "Failed to allocate or map a new skb - dropping packet!\n");
668         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
669 }
670
671 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
672                                struct bnx2x_fastpath *fp, u16 index)
673 {
674         u8 *data;
675         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
676         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
677         dma_addr_t mapping;
678
679         data = bnx2x_frag_alloc(fp);
680         if (unlikely(data == NULL))
681                 return -ENOMEM;
682
683         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
684                                  fp->rx_buf_size,
685                                  DMA_FROM_DEVICE);
686         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
687                 bnx2x_frag_free(fp, data);
688                 BNX2X_ERR("Can't map rx data\n");
689                 return -ENOMEM;
690         }
691
692         rx_buf->data = data;
693         dma_unmap_addr_set(rx_buf, mapping, mapping);
694
695         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
696         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
697
698         return 0;
699 }
700
701 static
702 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
703                                  struct bnx2x_fastpath *fp,
704                                  struct bnx2x_eth_q_stats *qstats)
705 {
706         /* Do nothing if no L4 csum validation was done.
707          * We do not check whether IP csum was validated. For IPv4 we assume
708          * that if the card got as far as validating the L4 csum, it also
709          * validated the IP csum. IPv6 has no IP csum.
710          */
711         if (cqe->fast_path_cqe.status_flags &
712             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
713                 return;
714
715         /* If L4 validation was done, check if an error was found. */
716
717         if (cqe->fast_path_cqe.type_error_flags &
718             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
719              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
720                 qstats->hw_csum_err++;
721         else
722                 skb->ip_summed = CHECKSUM_UNNECESSARY;
723 }
724
725 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
726 {
727         struct bnx2x *bp = fp->bp;
728         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
729         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
730         int rx_pkt = 0;
731
732 #ifdef BNX2X_STOP_ON_ERROR
733         if (unlikely(bp->panic))
734                 return 0;
735 #endif
736
737         /* CQ "next element" is of the size of the regular element,
738            that's why it's ok here */
739         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
740         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
741                 hw_comp_cons++;
742
743         bd_cons = fp->rx_bd_cons;
744         bd_prod = fp->rx_bd_prod;
745         bd_prod_fw = bd_prod;
746         sw_comp_cons = fp->rx_comp_cons;
747         sw_comp_prod = fp->rx_comp_prod;
748
749         /* Memory barrier necessary as speculative reads of the rx
750          * buffer can be ahead of the index in the status block
751          */
752         rmb();
753
754         DP(NETIF_MSG_RX_STATUS,
755            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
756            fp->index, hw_comp_cons, sw_comp_cons);
757
758         while (sw_comp_cons != hw_comp_cons) {
759                 struct sw_rx_bd *rx_buf = NULL;
760                 struct sk_buff *skb;
761                 union eth_rx_cqe *cqe;
762                 struct eth_fast_path_rx_cqe *cqe_fp;
763                 u8 cqe_fp_flags;
764                 enum eth_rx_cqe_type cqe_fp_type;
765                 u16 len, pad, queue;
766                 u8 *data;
767                 bool l4_rxhash;
768
769 #ifdef BNX2X_STOP_ON_ERROR
770                 if (unlikely(bp->panic))
771                         return 0;
772 #endif
773
774                 comp_ring_cons = RCQ_BD(sw_comp_cons);
775                 bd_prod = RX_BD(bd_prod);
776                 bd_cons = RX_BD(bd_cons);
777
778                 cqe = &fp->rx_comp_ring[comp_ring_cons];
779                 cqe_fp = &cqe->fast_path_cqe;
780                 cqe_fp_flags = cqe_fp->type_error_flags;
781                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
782
783                 DP(NETIF_MSG_RX_STATUS,
784                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
785                    CQE_TYPE(cqe_fp_flags),
786                    cqe_fp_flags, cqe_fp->status_flags,
787                    le32_to_cpu(cqe_fp->rss_hash_result),
788                    le16_to_cpu(cqe_fp->vlan_tag),
789                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
790
791                 /* is this a slowpath msg? */
792                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
793                         bnx2x_sp_event(fp, cqe);
794                         goto next_cqe;
795                 }
796
797                 rx_buf = &fp->rx_buf_ring[bd_cons];
798                 data = rx_buf->data;
799
800                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
801                         struct bnx2x_agg_info *tpa_info;
802                         u16 frag_size, pages;
803 #ifdef BNX2X_STOP_ON_ERROR
804                         /* sanity check */
805                         if (fp->disable_tpa &&
806                             (CQE_TYPE_START(cqe_fp_type) ||
807                              CQE_TYPE_STOP(cqe_fp_type)))
808                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
809                                           CQE_TYPE(cqe_fp_type));
810 #endif
811
812                         if (CQE_TYPE_START(cqe_fp_type)) {
813                                 u16 queue = cqe_fp->queue_index;
814                                 DP(NETIF_MSG_RX_STATUS,
815                                    "calling tpa_start on queue %d\n",
816                                    queue);
817
818                                 bnx2x_tpa_start(fp, queue,
819                                                 bd_cons, bd_prod,
820                                                 cqe_fp);
821
822                                 goto next_rx;
823
824                         }
825                         queue = cqe->end_agg_cqe.queue_index;
826                         tpa_info = &fp->tpa_info[queue];
827                         DP(NETIF_MSG_RX_STATUS,
828                            "calling tpa_stop on queue %d\n",
829                            queue);
830
831                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
832                                     tpa_info->len_on_bd;
833
834                         if (fp->mode == TPA_MODE_GRO)
835                                 pages = (frag_size + tpa_info->full_page - 1) /
836                                          tpa_info->full_page;
837                         else
838                                 pages = SGE_PAGE_ALIGN(frag_size) >>
839                                         SGE_PAGE_SHIFT;
840
841                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
842                                        &cqe->end_agg_cqe, comp_ring_cons);
843 #ifdef BNX2X_STOP_ON_ERROR
844                         if (bp->panic)
845                                 return 0;
846 #endif
847
848                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
849                         goto next_cqe;
850                 }
851                 /* non TPA */
852                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
853                 pad = cqe_fp->placement_offset;
854                 dma_sync_single_for_cpu(&bp->pdev->dev,
855                                         dma_unmap_addr(rx_buf, mapping),
856                                         pad + RX_COPY_THRESH,
857                                         DMA_FROM_DEVICE);
858                 pad += NET_SKB_PAD;
859                 prefetch(data + pad); /* speedup eth_type_trans() */
860                 /* is this an error packet? */
861                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
862                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
863                            "ERROR  flags %x  rx packet %u\n",
864                            cqe_fp_flags, sw_comp_cons);
865                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
866                         goto reuse_rx;
867                 }
868
869                 /* Since we don't have a jumbo ring
870                  * copy small packets if mtu > 1500
871                  */
872                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
873                     (len <= RX_COPY_THRESH)) {
874                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
875                         if (skb == NULL) {
876                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
877                                    "ERROR  packet dropped because of alloc failure\n");
878                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
879                                 goto reuse_rx;
880                         }
881                         memcpy(skb->data, data + pad, len);
882                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
883                 } else {
884                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
885                                 dma_unmap_single(&bp->pdev->dev,
886                                                  dma_unmap_addr(rx_buf, mapping),
887                                                  fp->rx_buf_size,
888                                                  DMA_FROM_DEVICE);
889                                 skb = build_skb(data, fp->rx_frag_size);
890                                 if (unlikely(!skb)) {
891                                         bnx2x_frag_free(fp, data);
892                                         bnx2x_fp_qstats(bp, fp)->
893                                                         rx_skb_alloc_failed++;
894                                         goto next_rx;
895                                 }
896                                 skb_reserve(skb, pad);
897                         } else {
898                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
899                                    "ERROR  packet dropped because of alloc failure\n");
900                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
901 reuse_rx:
902                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
903                                 goto next_rx;
904                         }
905                 }
906
907                 skb_put(skb, len);
908                 skb->protocol = eth_type_trans(skb, bp->dev);
909
910                 /* Set Toeplitz hash for a none-LRO skb */
911                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
912                 skb->l4_rxhash = l4_rxhash;
913
914                 skb_checksum_none_assert(skb);
915
916                 if (bp->dev->features & NETIF_F_RXCSUM)
917                         bnx2x_csum_validate(skb, cqe, fp,
918                                             bnx2x_fp_qstats(bp, fp));
919
920                 skb_record_rx_queue(skb, fp->rx_queue);
921
922                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
923                     PARSING_FLAGS_VLAN)
924                         __vlan_hwaccel_put_tag(skb,
925                                                le16_to_cpu(cqe_fp->vlan_tag));
926                 napi_gro_receive(&fp->napi, skb);
927
928
929 next_rx:
930                 rx_buf->data = NULL;
931
932                 bd_cons = NEXT_RX_IDX(bd_cons);
933                 bd_prod = NEXT_RX_IDX(bd_prod);
934                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
935                 rx_pkt++;
936 next_cqe:
937                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
938                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
939
940                 if (rx_pkt == budget)
941                         break;
942         } /* while */
943
944         fp->rx_bd_cons = bd_cons;
945         fp->rx_bd_prod = bd_prod_fw;
946         fp->rx_comp_cons = sw_comp_cons;
947         fp->rx_comp_prod = sw_comp_prod;
948
949         /* Update producers */
950         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
951                              fp->rx_sge_prod);
952
953         fp->rx_pkt += rx_pkt;
954         fp->rx_calls++;
955
956         return rx_pkt;
957 }
958
959 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
960 {
961         struct bnx2x_fastpath *fp = fp_cookie;
962         struct bnx2x *bp = fp->bp;
963         u8 cos;
964
965         DP(NETIF_MSG_INTR,
966            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
967            fp->index, fp->fw_sb_id, fp->igu_sb_id);
968         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
969
970 #ifdef BNX2X_STOP_ON_ERROR
971         if (unlikely(bp->panic))
972                 return IRQ_HANDLED;
973 #endif
974
975         /* Handle Rx and Tx according to MSI-X vector */
976         prefetch(fp->rx_cons_sb);
977
978         for_each_cos_in_tx_queue(fp, cos)
979                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
980
981         prefetch(&fp->sb_running_index[SM_RX_ID]);
982         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
983
984         return IRQ_HANDLED;
985 }
986
987 /* HW Lock for shared dual port PHYs */
988 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
989 {
990         mutex_lock(&bp->port.phy_mutex);
991
992         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
993 }
994
995 void bnx2x_release_phy_lock(struct bnx2x *bp)
996 {
997         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
998
999         mutex_unlock(&bp->port.phy_mutex);
1000 }
1001
1002 /* calculates MF speed according to current linespeed and MF configuration */
1003 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1004 {
1005         u16 line_speed = bp->link_vars.line_speed;
1006         if (IS_MF(bp)) {
1007                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1008                                                    bp->mf_config[BP_VN(bp)]);
1009
1010                 /* Calculate the current MAX line speed limit for the MF
1011                  * devices
1012                  */
1013                 if (IS_MF_SI(bp))
1014                         line_speed = (line_speed * maxCfg) / 100;
1015                 else { /* SD mode */
1016                         u16 vn_max_rate = maxCfg * 100;
1017
1018                         if (vn_max_rate < line_speed)
1019                                 line_speed = vn_max_rate;
1020                 }
1021         }
1022
1023         return line_speed;
1024 }
1025
1026 /**
1027  * bnx2x_fill_report_data - fill link report data to report
1028  *
1029  * @bp:         driver handle
1030  * @data:       link state to update
1031  *
1032  * It uses a none-atomic bit operations because is called under the mutex.
1033  */
1034 static void bnx2x_fill_report_data(struct bnx2x *bp,
1035                                    struct bnx2x_link_report_data *data)
1036 {
1037         u16 line_speed = bnx2x_get_mf_speed(bp);
1038
1039         memset(data, 0, sizeof(*data));
1040
1041         /* Fill the report data: efective line speed */
1042         data->line_speed = line_speed;
1043
1044         /* Link is down */
1045         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1046                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1047                           &data->link_report_flags);
1048
1049         /* Full DUPLEX */
1050         if (bp->link_vars.duplex == DUPLEX_FULL)
1051                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1052
1053         /* Rx Flow Control is ON */
1054         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1055                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1056
1057         /* Tx Flow Control is ON */
1058         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1059                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1060 }
1061
1062 /**
1063  * bnx2x_link_report - report link status to OS.
1064  *
1065  * @bp:         driver handle
1066  *
1067  * Calls the __bnx2x_link_report() under the same locking scheme
1068  * as a link/PHY state managing code to ensure a consistent link
1069  * reporting.
1070  */
1071
1072 void bnx2x_link_report(struct bnx2x *bp)
1073 {
1074         bnx2x_acquire_phy_lock(bp);
1075         __bnx2x_link_report(bp);
1076         bnx2x_release_phy_lock(bp);
1077 }
1078
1079 /**
1080  * __bnx2x_link_report - report link status to OS.
1081  *
1082  * @bp:         driver handle
1083  *
1084  * None atomic inmlementation.
1085  * Should be called under the phy_lock.
1086  */
1087 void __bnx2x_link_report(struct bnx2x *bp)
1088 {
1089         struct bnx2x_link_report_data cur_data;
1090
1091         /* reread mf_cfg */
1092         if (!CHIP_IS_E1(bp))
1093                 bnx2x_read_mf_cfg(bp);
1094
1095         /* Read the current link report info */
1096         bnx2x_fill_report_data(bp, &cur_data);
1097
1098         /* Don't report link down or exactly the same link status twice */
1099         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1100             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1101                       &bp->last_reported_link.link_report_flags) &&
1102              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1103                       &cur_data.link_report_flags)))
1104                 return;
1105
1106         bp->link_cnt++;
1107
1108         /* We are going to report a new link parameters now -
1109          * remember the current data for the next time.
1110          */
1111         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1112
1113         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1114                      &cur_data.link_report_flags)) {
1115                 netif_carrier_off(bp->dev);
1116                 netdev_err(bp->dev, "NIC Link is Down\n");
1117                 return;
1118         } else {
1119                 const char *duplex;
1120                 const char *flow;
1121
1122                 netif_carrier_on(bp->dev);
1123
1124                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1125                                        &cur_data.link_report_flags))
1126                         duplex = "full";
1127                 else
1128                         duplex = "half";
1129
1130                 /* Handle the FC at the end so that only these flags would be
1131                  * possibly set. This way we may easily check if there is no FC
1132                  * enabled.
1133                  */
1134                 if (cur_data.link_report_flags) {
1135                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1136                                      &cur_data.link_report_flags)) {
1137                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1138                                      &cur_data.link_report_flags))
1139                                         flow = "ON - receive & transmit";
1140                                 else
1141                                         flow = "ON - receive";
1142                         } else {
1143                                 flow = "ON - transmit";
1144                         }
1145                 } else {
1146                         flow = "none";
1147                 }
1148                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1149                             cur_data.line_speed, duplex, flow);
1150         }
1151 }
1152
1153 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1154 {
1155         int i;
1156
1157         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1158                 struct eth_rx_sge *sge;
1159
1160                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1161                 sge->addr_hi =
1162                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1163                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1164
1165                 sge->addr_lo =
1166                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1167                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1168         }
1169 }
1170
1171 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1172                                 struct bnx2x_fastpath *fp, int last)
1173 {
1174         int i;
1175
1176         for (i = 0; i < last; i++) {
1177                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1178                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1179                 u8 *data = first_buf->data;
1180
1181                 if (data == NULL) {
1182                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1183                         continue;
1184                 }
1185                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1186                         dma_unmap_single(&bp->pdev->dev,
1187                                          dma_unmap_addr(first_buf, mapping),
1188                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1189                 bnx2x_frag_free(fp, data);
1190                 first_buf->data = NULL;
1191         }
1192 }
1193
1194 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1195 {
1196         int j;
1197
1198         for_each_rx_queue_cnic(bp, j) {
1199                 struct bnx2x_fastpath *fp = &bp->fp[j];
1200
1201                 fp->rx_bd_cons = 0;
1202
1203                 /* Activate BD ring */
1204                 /* Warning!
1205                  * this will generate an interrupt (to the TSTORM)
1206                  * must only be done after chip is initialized
1207                  */
1208                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1209                                      fp->rx_sge_prod);
1210         }
1211 }
1212
1213 void bnx2x_init_rx_rings(struct bnx2x *bp)
1214 {
1215         int func = BP_FUNC(bp);
1216         u16 ring_prod;
1217         int i, j;
1218
1219         /* Allocate TPA resources */
1220         for_each_eth_queue(bp, j) {
1221                 struct bnx2x_fastpath *fp = &bp->fp[j];
1222
1223                 DP(NETIF_MSG_IFUP,
1224                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1225
1226                 if (!fp->disable_tpa) {
1227                         /* Fill the per-aggregtion pool */
1228                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1229                                 struct bnx2x_agg_info *tpa_info =
1230                                         &fp->tpa_info[i];
1231                                 struct sw_rx_bd *first_buf =
1232                                         &tpa_info->first_buf;
1233
1234                                 first_buf->data = bnx2x_frag_alloc(fp);
1235                                 if (!first_buf->data) {
1236                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1237                                                   j);
1238                                         bnx2x_free_tpa_pool(bp, fp, i);
1239                                         fp->disable_tpa = 1;
1240                                         break;
1241                                 }
1242                                 dma_unmap_addr_set(first_buf, mapping, 0);
1243                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1244                         }
1245
1246                         /* "next page" elements initialization */
1247                         bnx2x_set_next_page_sgl(fp);
1248
1249                         /* set SGEs bit mask */
1250                         bnx2x_init_sge_ring_bit_mask(fp);
1251
1252                         /* Allocate SGEs and initialize the ring elements */
1253                         for (i = 0, ring_prod = 0;
1254                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1255
1256                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1257                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1258                                                   i);
1259                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1260                                                   j);
1261                                         /* Cleanup already allocated elements */
1262                                         bnx2x_free_rx_sge_range(bp, fp,
1263                                                                 ring_prod);
1264                                         bnx2x_free_tpa_pool(bp, fp,
1265                                                             MAX_AGG_QS(bp));
1266                                         fp->disable_tpa = 1;
1267                                         ring_prod = 0;
1268                                         break;
1269                                 }
1270                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1271                         }
1272
1273                         fp->rx_sge_prod = ring_prod;
1274                 }
1275         }
1276
1277         for_each_eth_queue(bp, j) {
1278                 struct bnx2x_fastpath *fp = &bp->fp[j];
1279
1280                 fp->rx_bd_cons = 0;
1281
1282                 /* Activate BD ring */
1283                 /* Warning!
1284                  * this will generate an interrupt (to the TSTORM)
1285                  * must only be done after chip is initialized
1286                  */
1287                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1288                                      fp->rx_sge_prod);
1289
1290                 if (j != 0)
1291                         continue;
1292
1293                 if (CHIP_IS_E1(bp)) {
1294                         REG_WR(bp, BAR_USTRORM_INTMEM +
1295                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1296                                U64_LO(fp->rx_comp_mapping));
1297                         REG_WR(bp, BAR_USTRORM_INTMEM +
1298                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1299                                U64_HI(fp->rx_comp_mapping));
1300                 }
1301         }
1302 }
1303
1304 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1305 {
1306         u8 cos;
1307         struct bnx2x *bp = fp->bp;
1308
1309         for_each_cos_in_tx_queue(fp, cos) {
1310                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1311                 unsigned pkts_compl = 0, bytes_compl = 0;
1312
1313                 u16 sw_prod = txdata->tx_pkt_prod;
1314                 u16 sw_cons = txdata->tx_pkt_cons;
1315
1316                 while (sw_cons != sw_prod) {
1317                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1318                                           &pkts_compl, &bytes_compl);
1319                         sw_cons++;
1320                 }
1321
1322                 netdev_tx_reset_queue(
1323                         netdev_get_tx_queue(bp->dev,
1324                                             txdata->txq_index));
1325         }
1326 }
1327
1328 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1329 {
1330         int i;
1331
1332         for_each_tx_queue_cnic(bp, i) {
1333                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1334         }
1335 }
1336
1337 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1338 {
1339         int i;
1340
1341         for_each_eth_queue(bp, i) {
1342                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1343         }
1344 }
1345
1346 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1347 {
1348         struct bnx2x *bp = fp->bp;
1349         int i;
1350
1351         /* ring wasn't allocated */
1352         if (fp->rx_buf_ring == NULL)
1353                 return;
1354
1355         for (i = 0; i < NUM_RX_BD; i++) {
1356                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1357                 u8 *data = rx_buf->data;
1358
1359                 if (data == NULL)
1360                         continue;
1361                 dma_unmap_single(&bp->pdev->dev,
1362                                  dma_unmap_addr(rx_buf, mapping),
1363                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1364
1365                 rx_buf->data = NULL;
1366                 bnx2x_frag_free(fp, data);
1367         }
1368 }
1369
1370 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1371 {
1372         int j;
1373
1374         for_each_rx_queue_cnic(bp, j) {
1375                 bnx2x_free_rx_bds(&bp->fp[j]);
1376         }
1377 }
1378
1379 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1380 {
1381         int j;
1382
1383         for_each_eth_queue(bp, j) {
1384                 struct bnx2x_fastpath *fp = &bp->fp[j];
1385
1386                 bnx2x_free_rx_bds(fp);
1387
1388                 if (!fp->disable_tpa)
1389                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1390         }
1391 }
1392
1393 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1394 {
1395         bnx2x_free_tx_skbs_cnic(bp);
1396         bnx2x_free_rx_skbs_cnic(bp);
1397 }
1398
1399 void bnx2x_free_skbs(struct bnx2x *bp)
1400 {
1401         bnx2x_free_tx_skbs(bp);
1402         bnx2x_free_rx_skbs(bp);
1403 }
1404
1405 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1406 {
1407         /* load old values */
1408         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1409
1410         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1411                 /* leave all but MAX value */
1412                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1413
1414                 /* set new MAX value */
1415                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1416                                 & FUNC_MF_CFG_MAX_BW_MASK;
1417
1418                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1419         }
1420 }
1421
1422 /**
1423  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1424  *
1425  * @bp:         driver handle
1426  * @nvecs:      number of vectors to be released
1427  */
1428 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1429 {
1430         int i, offset = 0;
1431
1432         if (nvecs == offset)
1433                 return;
1434         free_irq(bp->msix_table[offset].vector, bp->dev);
1435         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1436            bp->msix_table[offset].vector);
1437         offset++;
1438
1439         if (CNIC_SUPPORT(bp)) {
1440                 if (nvecs == offset)
1441                         return;
1442                 offset++;
1443         }
1444
1445         for_each_eth_queue(bp, i) {
1446                 if (nvecs == offset)
1447                         return;
1448                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1449                    i, bp->msix_table[offset].vector);
1450
1451                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1452         }
1453 }
1454
1455 void bnx2x_free_irq(struct bnx2x *bp)
1456 {
1457         if (bp->flags & USING_MSIX_FLAG &&
1458             !(bp->flags & USING_SINGLE_MSIX_FLAG))
1459                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1460                                      CNIC_SUPPORT(bp) + 1);
1461         else
1462                 free_irq(bp->dev->irq, bp->dev);
1463 }
1464
1465 int bnx2x_enable_msix(struct bnx2x *bp)
1466 {
1467         int msix_vec = 0, i, rc, req_cnt;
1468
1469         bp->msix_table[msix_vec].entry = msix_vec;
1470         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1471            bp->msix_table[0].entry);
1472         msix_vec++;
1473
1474         /* Cnic requires an msix vector for itself */
1475         if (CNIC_SUPPORT(bp)) {
1476                 bp->msix_table[msix_vec].entry = msix_vec;
1477                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1478                                msix_vec, bp->msix_table[msix_vec].entry);
1479                 msix_vec++;
1480         }
1481
1482         /* We need separate vectors for ETH queues only (not FCoE) */
1483         for_each_eth_queue(bp, i) {
1484                 bp->msix_table[msix_vec].entry = msix_vec;
1485                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1486                                msix_vec, msix_vec, i);
1487                 msix_vec++;
1488         }
1489
1490         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp) + 1;
1491
1492         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1493
1494         /*
1495          * reconfigure number of tx/rx queues according to available
1496          * MSI-X vectors
1497          */
1498         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1499                 /* how less vectors we will have? */
1500                 int diff = req_cnt - rc;
1501
1502                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1503
1504                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1505
1506                 if (rc) {
1507                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1508                         goto no_msix;
1509                 }
1510                 /*
1511                  * decrease number of queues by number of unallocated entries
1512                  */
1513                 bp->num_ethernet_queues -= diff;
1514                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1515
1516                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1517                                bp->num_queues);
1518         } else if (rc > 0) {
1519                 /* Get by with single vector */
1520                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1521                 if (rc) {
1522                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1523                                        rc);
1524                         goto no_msix;
1525                 }
1526
1527                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1528                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1529
1530                 BNX2X_DEV_INFO("set number of queues to 1\n");
1531                 bp->num_ethernet_queues = 1;
1532                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1533         } else if (rc < 0) {
1534                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1535                 goto no_msix;
1536         }
1537
1538         bp->flags |= USING_MSIX_FLAG;
1539
1540         return 0;
1541
1542 no_msix:
1543         /* fall to INTx if not enough memory */
1544         if (rc == -ENOMEM)
1545                 bp->flags |= DISABLE_MSI_FLAG;
1546
1547         return rc;
1548 }
1549
1550 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1551 {
1552         int i, rc, offset = 0;
1553
1554         rc = request_irq(bp->msix_table[offset++].vector,
1555                          bnx2x_msix_sp_int, 0,
1556                          bp->dev->name, bp->dev);
1557         if (rc) {
1558                 BNX2X_ERR("request sp irq failed\n");
1559                 return -EBUSY;
1560         }
1561
1562         if (CNIC_SUPPORT(bp))
1563                 offset++;
1564
1565         for_each_eth_queue(bp, i) {
1566                 struct bnx2x_fastpath *fp = &bp->fp[i];
1567                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1568                          bp->dev->name, i);
1569
1570                 rc = request_irq(bp->msix_table[offset].vector,
1571                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1572                 if (rc) {
1573                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1574                               bp->msix_table[offset].vector, rc);
1575                         bnx2x_free_msix_irqs(bp, offset);
1576                         return -EBUSY;
1577                 }
1578
1579                 offset++;
1580         }
1581
1582         i = BNX2X_NUM_ETH_QUEUES(bp);
1583         offset = 1 + CNIC_SUPPORT(bp);
1584         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1585                bp->msix_table[0].vector,
1586                0, bp->msix_table[offset].vector,
1587                i - 1, bp->msix_table[offset + i - 1].vector);
1588
1589         return 0;
1590 }
1591
1592 int bnx2x_enable_msi(struct bnx2x *bp)
1593 {
1594         int rc;
1595
1596         rc = pci_enable_msi(bp->pdev);
1597         if (rc) {
1598                 BNX2X_DEV_INFO("MSI is not attainable\n");
1599                 return -1;
1600         }
1601         bp->flags |= USING_MSI_FLAG;
1602
1603         return 0;
1604 }
1605
1606 static int bnx2x_req_irq(struct bnx2x *bp)
1607 {
1608         unsigned long flags;
1609         unsigned int irq;
1610
1611         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1612                 flags = 0;
1613         else
1614                 flags = IRQF_SHARED;
1615
1616         if (bp->flags & USING_MSIX_FLAG)
1617                 irq = bp->msix_table[0].vector;
1618         else
1619                 irq = bp->pdev->irq;
1620
1621         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1622 }
1623
1624 static int bnx2x_setup_irqs(struct bnx2x *bp)
1625 {
1626         int rc = 0;
1627         if (bp->flags & USING_MSIX_FLAG &&
1628             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1629                 rc = bnx2x_req_msix_irqs(bp);
1630                 if (rc)
1631                         return rc;
1632         } else {
1633                 bnx2x_ack_int(bp);
1634                 rc = bnx2x_req_irq(bp);
1635                 if (rc) {
1636                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1637                         return rc;
1638                 }
1639                 if (bp->flags & USING_MSI_FLAG) {
1640                         bp->dev->irq = bp->pdev->irq;
1641                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1642                                     bp->dev->irq);
1643                 }
1644                 if (bp->flags & USING_MSIX_FLAG) {
1645                         bp->dev->irq = bp->msix_table[0].vector;
1646                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1647                                     bp->dev->irq);
1648                 }
1649         }
1650
1651         return 0;
1652 }
1653
1654 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1655 {
1656         int i;
1657
1658         for_each_rx_queue_cnic(bp, i)
1659                 napi_enable(&bnx2x_fp(bp, i, napi));
1660 }
1661
1662 static void bnx2x_napi_enable(struct bnx2x *bp)
1663 {
1664         int i;
1665
1666         for_each_eth_queue(bp, i)
1667                 napi_enable(&bnx2x_fp(bp, i, napi));
1668 }
1669
1670 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1671 {
1672         int i;
1673
1674         for_each_rx_queue_cnic(bp, i)
1675                 napi_disable(&bnx2x_fp(bp, i, napi));
1676 }
1677
1678 static void bnx2x_napi_disable(struct bnx2x *bp)
1679 {
1680         int i;
1681
1682         for_each_eth_queue(bp, i)
1683                 napi_disable(&bnx2x_fp(bp, i, napi));
1684 }
1685
1686 void bnx2x_netif_start(struct bnx2x *bp)
1687 {
1688         if (netif_running(bp->dev)) {
1689                 bnx2x_napi_enable(bp);
1690                 if (CNIC_LOADED(bp))
1691                         bnx2x_napi_enable_cnic(bp);
1692                 bnx2x_int_enable(bp);
1693                 if (bp->state == BNX2X_STATE_OPEN)
1694                         netif_tx_wake_all_queues(bp->dev);
1695         }
1696 }
1697
1698 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1699 {
1700         bnx2x_int_disable_sync(bp, disable_hw);
1701         bnx2x_napi_disable(bp);
1702         if (CNIC_LOADED(bp))
1703                 bnx2x_napi_disable_cnic(bp);
1704 }
1705
1706 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1707 {
1708         struct bnx2x *bp = netdev_priv(dev);
1709
1710         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1711                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1712                 u16 ether_type = ntohs(hdr->h_proto);
1713
1714                 /* Skip VLAN tag if present */
1715                 if (ether_type == ETH_P_8021Q) {
1716                         struct vlan_ethhdr *vhdr =
1717                                 (struct vlan_ethhdr *)skb->data;
1718
1719                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1720                 }
1721
1722                 /* If ethertype is FCoE or FIP - use FCoE ring */
1723                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1724                         return bnx2x_fcoe_tx(bp, txq_index);
1725         }
1726
1727         /* select a non-FCoE queue */
1728         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1729 }
1730
1731
1732 void bnx2x_set_num_queues(struct bnx2x *bp)
1733 {
1734         /* RSS queues */
1735         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1736
1737         /* override in STORAGE SD modes */
1738         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1739                 bp->num_ethernet_queues = 1;
1740
1741         /* Add special queues */
1742         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1743         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1744
1745         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1746 }
1747
1748 /**
1749  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1750  *
1751  * @bp:         Driver handle
1752  *
1753  * We currently support for at most 16 Tx queues for each CoS thus we will
1754  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1755  * bp->max_cos.
1756  *
1757  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1758  * index after all ETH L2 indices.
1759  *
1760  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1761  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1762  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1763  *
1764  * The proper configuration of skb->queue_mapping is handled by
1765  * bnx2x_select_queue() and __skb_tx_hash().
1766  *
1767  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1768  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1769  */
1770 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1771 {
1772         int rc, tx, rx;
1773
1774         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1775         rx = BNX2X_NUM_ETH_QUEUES(bp);
1776
1777 /* account for fcoe queue */
1778         if (include_cnic && !NO_FCOE(bp)) {
1779                 rx++;
1780                 tx++;
1781         }
1782
1783         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1784         if (rc) {
1785                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1786                 return rc;
1787         }
1788         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1789         if (rc) {
1790                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1791                 return rc;
1792         }
1793
1794         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1795                           tx, rx);
1796
1797         return rc;
1798 }
1799
1800 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1801 {
1802         int i;
1803
1804         for_each_queue(bp, i) {
1805                 struct bnx2x_fastpath *fp = &bp->fp[i];
1806                 u32 mtu;
1807
1808                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1809                 if (IS_FCOE_IDX(i))
1810                         /*
1811                          * Although there are no IP frames expected to arrive to
1812                          * this ring we still want to add an
1813                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1814                          * overrun attack.
1815                          */
1816                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1817                 else
1818                         mtu = bp->dev->mtu;
1819                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1820                                   IP_HEADER_ALIGNMENT_PADDING +
1821                                   ETH_OVREHEAD +
1822                                   mtu +
1823                                   BNX2X_FW_RX_ALIGN_END;
1824                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1825                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1826                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1827                 else
1828                         fp->rx_frag_size = 0;
1829         }
1830 }
1831
1832 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1833 {
1834         int i;
1835         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1836
1837         /* Prepare the initial contents fo the indirection table if RSS is
1838          * enabled
1839          */
1840         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1841                 bp->rss_conf_obj.ind_table[i] =
1842                         bp->fp->cl_id +
1843                         ethtool_rxfh_indir_default(i, num_eth_queues);
1844
1845         /*
1846          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1847          * per-port, so if explicit configuration is needed , do it only
1848          * for a PMF.
1849          *
1850          * For 57712 and newer on the other hand it's a per-function
1851          * configuration.
1852          */
1853         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1854 }
1855
1856 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1857                         bool config_hash)
1858 {
1859         struct bnx2x_config_rss_params params = {NULL};
1860
1861         /* Although RSS is meaningless when there is a single HW queue we
1862          * still need it enabled in order to have HW Rx hash generated.
1863          *
1864          * if (!is_eth_multi(bp))
1865          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1866          */
1867
1868         params.rss_obj = rss_obj;
1869
1870         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1871
1872         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1873
1874         /* RSS configuration */
1875         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1876         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1877         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1878         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1879         if (rss_obj->udp_rss_v4)
1880                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1881         if (rss_obj->udp_rss_v6)
1882                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1883
1884         /* Hash bits */
1885         params.rss_result_mask = MULTI_MASK;
1886
1887         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1888
1889         if (config_hash) {
1890                 /* RSS keys */
1891                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
1892                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1893         }
1894
1895         return bnx2x_config_rss(bp, &params);
1896 }
1897
1898 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1899 {
1900         struct bnx2x_func_state_params func_params = {NULL};
1901
1902         /* Prepare parameters for function state transitions */
1903         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1904
1905         func_params.f_obj = &bp->func_obj;
1906         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1907
1908         func_params.params.hw_init.load_phase = load_code;
1909
1910         return bnx2x_func_state_change(bp, &func_params);
1911 }
1912
1913 /*
1914  * Cleans the object that have internal lists without sending
1915  * ramrods. Should be run when interrutps are disabled.
1916  */
1917 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1918 {
1919         int rc;
1920         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1921         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1922         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
1923
1924         /***************** Cleanup MACs' object first *************************/
1925
1926         /* Wait for completion of requested */
1927         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1928         /* Perform a dry cleanup */
1929         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1930
1931         /* Clean ETH primary MAC */
1932         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1933         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
1934                                  &ramrod_flags);
1935         if (rc != 0)
1936                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1937
1938         /* Cleanup UC list */
1939         vlan_mac_flags = 0;
1940         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1941         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1942                                  &ramrod_flags);
1943         if (rc != 0)
1944                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1945
1946         /***************** Now clean mcast object *****************************/
1947         rparam.mcast_obj = &bp->mcast_obj;
1948         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1949
1950         /* Add a DEL command... */
1951         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1952         if (rc < 0)
1953                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1954                           rc);
1955
1956         /* ...and wait until all pending commands are cleared */
1957         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1958         while (rc != 0) {
1959                 if (rc < 0) {
1960                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1961                                   rc);
1962                         return;
1963                 }
1964
1965                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1966         }
1967 }
1968
1969 #ifndef BNX2X_STOP_ON_ERROR
1970 #define LOAD_ERROR_EXIT(bp, label) \
1971         do { \
1972                 (bp)->state = BNX2X_STATE_ERROR; \
1973                 goto label; \
1974         } while (0)
1975
1976 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1977         do { \
1978                 bp->cnic_loaded = false; \
1979                 goto label; \
1980         } while (0)
1981 #else /*BNX2X_STOP_ON_ERROR*/
1982 #define LOAD_ERROR_EXIT(bp, label) \
1983         do { \
1984                 (bp)->state = BNX2X_STATE_ERROR; \
1985                 (bp)->panic = 1; \
1986                 return -EBUSY; \
1987         } while (0)
1988 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1989         do { \
1990                 bp->cnic_loaded = false; \
1991                 (bp)->panic = 1; \
1992                 return -EBUSY; \
1993         } while (0)
1994 #endif /*BNX2X_STOP_ON_ERROR*/
1995
1996 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1997 {
1998         /* build FW version dword */
1999         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2000                     (BCM_5710_FW_MINOR_VERSION << 8) +
2001                     (BCM_5710_FW_REVISION_VERSION << 16) +
2002                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
2003
2004         /* read loaded FW from chip */
2005         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2006
2007         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
2008
2009         if (loaded_fw != my_fw) {
2010                 if (is_err)
2011                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
2012                                   loaded_fw, my_fw);
2013                 return false;
2014         }
2015
2016         return true;
2017 }
2018
2019 /**
2020  * bnx2x_bz_fp - zero content of the fastpath structure.
2021  *
2022  * @bp:         driver handle
2023  * @index:      fastpath index to be zeroed
2024  *
2025  * Makes sure the contents of the bp->fp[index].napi is kept
2026  * intact.
2027  */
2028 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2029 {
2030         struct bnx2x_fastpath *fp = &bp->fp[index];
2031         struct bnx2x_fp_stats *fp_stats = &bp->fp_stats[index];
2032
2033         int cos;
2034         struct napi_struct orig_napi = fp->napi;
2035         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2036         /* bzero bnx2x_fastpath contents */
2037         if (bp->stats_init) {
2038                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2039                 memset(fp, 0, sizeof(*fp));
2040         } else {
2041                 /* Keep Queue statistics */
2042                 struct bnx2x_eth_q_stats *tmp_eth_q_stats;
2043                 struct bnx2x_eth_q_stats_old *tmp_eth_q_stats_old;
2044
2045                 tmp_eth_q_stats = kzalloc(sizeof(struct bnx2x_eth_q_stats),
2046                                           GFP_KERNEL);
2047                 if (tmp_eth_q_stats)
2048                         memcpy(tmp_eth_q_stats, &fp_stats->eth_q_stats,
2049                                sizeof(struct bnx2x_eth_q_stats));
2050
2051                 tmp_eth_q_stats_old =
2052                         kzalloc(sizeof(struct bnx2x_eth_q_stats_old),
2053                                 GFP_KERNEL);
2054                 if (tmp_eth_q_stats_old)
2055                         memcpy(tmp_eth_q_stats_old, &fp_stats->eth_q_stats_old,
2056                                sizeof(struct bnx2x_eth_q_stats_old));
2057
2058                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2059                 memset(fp, 0, sizeof(*fp));
2060
2061                 if (tmp_eth_q_stats) {
2062                         memcpy(&fp_stats->eth_q_stats, tmp_eth_q_stats,
2063                                sizeof(struct bnx2x_eth_q_stats));
2064                         kfree(tmp_eth_q_stats);
2065                 }
2066
2067                 if (tmp_eth_q_stats_old) {
2068                         memcpy(&fp_stats->eth_q_stats_old, tmp_eth_q_stats_old,
2069                                sizeof(struct bnx2x_eth_q_stats_old));
2070                         kfree(tmp_eth_q_stats_old);
2071                 }
2072
2073         }
2074
2075         /* Restore the NAPI object as it has been already initialized */
2076         fp->napi = orig_napi;
2077         fp->tpa_info = orig_tpa_info;
2078         fp->bp = bp;
2079         fp->index = index;
2080         if (IS_ETH_FP(fp))
2081                 fp->max_cos = bp->max_cos;
2082         else
2083                 /* Special queues support only one CoS */
2084                 fp->max_cos = 1;
2085
2086         /* Init txdata pointers */
2087         if (IS_FCOE_FP(fp))
2088                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2089         if (IS_ETH_FP(fp))
2090                 for_each_cos_in_tx_queue(fp, cos)
2091                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2092                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2093
2094         /*
2095          * set the tpa flag for each queue. The tpa flag determines the queue
2096          * minimal size so it must be set prior to queue memory allocation
2097          */
2098         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2099                                   (bp->flags & GRO_ENABLE_FLAG &&
2100                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2101         if (bp->flags & TPA_ENABLE_FLAG)
2102                 fp->mode = TPA_MODE_LRO;
2103         else if (bp->flags & GRO_ENABLE_FLAG)
2104                 fp->mode = TPA_MODE_GRO;
2105
2106         /* We don't want TPA on an FCoE L2 ring */
2107         if (IS_FCOE_FP(fp))
2108                 fp->disable_tpa = 1;
2109 }
2110
2111 int bnx2x_load_cnic(struct bnx2x *bp)
2112 {
2113         int i, rc, port = BP_PORT(bp);
2114
2115         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2116
2117         mutex_init(&bp->cnic_mutex);
2118
2119         rc = bnx2x_alloc_mem_cnic(bp);
2120         if (rc) {
2121                 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2122                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2123         }
2124
2125         rc = bnx2x_alloc_fp_mem_cnic(bp);
2126         if (rc) {
2127                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2128                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2129         }
2130
2131         /* Update the number of queues with the cnic queues */
2132         rc = bnx2x_set_real_num_queues(bp, 1);
2133         if (rc) {
2134                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2135                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2136         }
2137
2138         /* Add all CNIC NAPI objects */
2139         bnx2x_add_all_napi_cnic(bp);
2140         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2141         bnx2x_napi_enable_cnic(bp);
2142
2143         rc = bnx2x_init_hw_func_cnic(bp);
2144         if (rc)
2145                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2146
2147         bnx2x_nic_init_cnic(bp);
2148
2149         /* Enable Timer scan */
2150         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2151
2152         for_each_cnic_queue(bp, i) {
2153                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2154                 if (rc) {
2155                         BNX2X_ERR("Queue setup failed\n");
2156                         LOAD_ERROR_EXIT(bp, load_error_cnic2);
2157                 }
2158         }
2159
2160         /* Initialize Rx filter. */
2161         netif_addr_lock_bh(bp->dev);
2162         bnx2x_set_rx_mode(bp->dev);
2163         netif_addr_unlock_bh(bp->dev);
2164
2165         /* re-read iscsi info */
2166         bnx2x_get_iscsi_info(bp);
2167         bnx2x_setup_cnic_irq_info(bp);
2168         bnx2x_setup_cnic_info(bp);
2169         bp->cnic_loaded = true;
2170         if (bp->state == BNX2X_STATE_OPEN)
2171                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2172
2173
2174         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2175
2176         return 0;
2177
2178 #ifndef BNX2X_STOP_ON_ERROR
2179 load_error_cnic2:
2180         /* Disable Timer scan */
2181         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2182
2183 load_error_cnic1:
2184         bnx2x_napi_disable_cnic(bp);
2185         /* Update the number of queues without the cnic queues */
2186         rc = bnx2x_set_real_num_queues(bp, 0);
2187         if (rc)
2188                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2189 load_error_cnic0:
2190         BNX2X_ERR("CNIC-related load failed\n");
2191         bnx2x_free_fp_mem_cnic(bp);
2192         bnx2x_free_mem_cnic(bp);
2193         return rc;
2194 #endif /* ! BNX2X_STOP_ON_ERROR */
2195 }
2196
2197
2198 /* must be called with rtnl_lock */
2199 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2200 {
2201         int port = BP_PORT(bp);
2202         u32 load_code;
2203         int i, rc;
2204
2205         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2206         DP(NETIF_MSG_IFUP,
2207            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2208
2209 #ifdef BNX2X_STOP_ON_ERROR
2210         if (unlikely(bp->panic)) {
2211                 BNX2X_ERR("Can't load NIC when there is panic\n");
2212                 return -EPERM;
2213         }
2214 #endif
2215
2216         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2217
2218         /* Set the initial link reported state to link down */
2219         bnx2x_acquire_phy_lock(bp);
2220         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2221         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2222                 &bp->last_reported_link.link_report_flags);
2223         bnx2x_release_phy_lock(bp);
2224
2225         /* must be called before memory allocation and HW init */
2226         bnx2x_ilt_set_info(bp);
2227
2228         /*
2229          * Zero fastpath structures preserving invariants like napi, which are
2230          * allocated only once, fp index, max_cos, bp pointer.
2231          * Also set fp->disable_tpa and txdata_ptr.
2232          */
2233         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2234         for_each_queue(bp, i)
2235                 bnx2x_bz_fp(bp, i);
2236         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2237                                   bp->num_cnic_queues) *
2238                                   sizeof(struct bnx2x_fp_txdata));
2239
2240         bp->fcoe_init = false;
2241
2242         /* Set the receive queues buffer size */
2243         bnx2x_set_rx_buf_size(bp);
2244
2245         if (bnx2x_alloc_mem(bp))
2246                 return -ENOMEM;
2247
2248         /* As long as bnx2x_alloc_mem() may possibly update
2249          * bp->num_queues, bnx2x_set_real_num_queues() should always
2250          * come after it. At this stage cnic queues are not counted.
2251          */
2252         rc = bnx2x_set_real_num_queues(bp, 0);
2253         if (rc) {
2254                 BNX2X_ERR("Unable to set real_num_queues\n");
2255                 LOAD_ERROR_EXIT(bp, load_error0);
2256         }
2257
2258         /* configure multi cos mappings in kernel.
2259          * this configuration may be overriden by a multi class queue discipline
2260          * or by a dcbx negotiation result.
2261          */
2262         bnx2x_setup_tc(bp->dev, bp->max_cos);
2263
2264         /* Add all NAPI objects */
2265         bnx2x_add_all_napi(bp);
2266         DP(NETIF_MSG_IFUP, "napi added\n");
2267         bnx2x_napi_enable(bp);
2268
2269         /* set pf load just before approaching the MCP */
2270         bnx2x_set_pf_load(bp);
2271
2272         /* Send LOAD_REQUEST command to MCP
2273          * Returns the type of LOAD command:
2274          * if it is the first port to be initialized
2275          * common blocks should be initialized, otherwise - not
2276          */
2277         if (!BP_NOMCP(bp)) {
2278                 /* init fw_seq */
2279                 bp->fw_seq =
2280                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2281                          DRV_MSG_SEQ_NUMBER_MASK);
2282                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2283
2284                 /* Get current FW pulse sequence */
2285                 bp->fw_drv_pulse_wr_seq =
2286                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2287                          DRV_PULSE_SEQ_MASK);
2288                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2289
2290                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2291                                              DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2292                 if (!load_code) {
2293                         BNX2X_ERR("MCP response failure, aborting\n");
2294                         rc = -EBUSY;
2295                         LOAD_ERROR_EXIT(bp, load_error1);
2296                 }
2297                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2298                         BNX2X_ERR("Driver load refused\n");
2299                         rc = -EBUSY; /* other port in diagnostic mode */
2300                         LOAD_ERROR_EXIT(bp, load_error1);
2301                 }
2302                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2303                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2304                         /* abort nic load if version mismatch */
2305                         if (!bnx2x_test_firmware_version(bp, true)) {
2306                                 rc = -EBUSY;
2307                                 LOAD_ERROR_EXIT(bp, load_error2);
2308                         }
2309                 }
2310
2311         } else {
2312                 int path = BP_PATH(bp);
2313
2314                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2315                    path, load_count[path][0], load_count[path][1],
2316                    load_count[path][2]);
2317                 load_count[path][0]++;
2318                 load_count[path][1 + port]++;
2319                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2320                    path, load_count[path][0], load_count[path][1],
2321                    load_count[path][2]);
2322                 if (load_count[path][0] == 1)
2323                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
2324                 else if (load_count[path][1 + port] == 1)
2325                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
2326                 else
2327                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
2328         }
2329
2330         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2331             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2332             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2333                 bp->port.pmf = 1;
2334                 /*
2335                  * We need the barrier to ensure the ordering between the
2336                  * writing to bp->port.pmf here and reading it from the
2337                  * bnx2x_periodic_task().
2338                  */
2339                 smp_mb();
2340         } else
2341                 bp->port.pmf = 0;
2342
2343         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
2344
2345         /* Init Function state controlling object */
2346         bnx2x__init_func_obj(bp);
2347
2348         /* Initialize HW */
2349         rc = bnx2x_init_hw(bp, load_code);
2350         if (rc) {
2351                 BNX2X_ERR("HW init failed, aborting\n");
2352                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2353                 LOAD_ERROR_EXIT(bp, load_error2);
2354         }
2355
2356         /* Connect to IRQs */
2357         rc = bnx2x_setup_irqs(bp);
2358         if (rc) {
2359                 BNX2X_ERR("IRQs setup failed\n");
2360                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2361                 LOAD_ERROR_EXIT(bp, load_error2);
2362         }
2363
2364         /* Setup NIC internals and enable interrupts */
2365         bnx2x_nic_init(bp, load_code);
2366
2367         /* Init per-function objects */
2368         bnx2x_init_bp_objs(bp);
2369
2370         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2371             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2372             (bp->common.shmem2_base)) {
2373                 if (SHMEM2_HAS(bp, dcc_support))
2374                         SHMEM2_WR(bp, dcc_support,
2375                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2376                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2377                 if (SHMEM2_HAS(bp, afex_driver_support))
2378                         SHMEM2_WR(bp, afex_driver_support,
2379                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2380         }
2381
2382         /* Set AFEX default VLAN tag to an invalid value */
2383         bp->afex_def_vlan_tag = -1;
2384
2385         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2386         rc = bnx2x_func_start(bp);
2387         if (rc) {
2388                 BNX2X_ERR("Function start failed!\n");
2389                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2390                 LOAD_ERROR_EXIT(bp, load_error3);
2391         }
2392
2393         /* Send LOAD_DONE command to MCP */
2394         if (!BP_NOMCP(bp)) {
2395                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2396                 if (!load_code) {
2397                         BNX2X_ERR("MCP response failure, aborting\n");
2398                         rc = -EBUSY;
2399                         LOAD_ERROR_EXIT(bp, load_error3);
2400                 }
2401         }
2402
2403         rc = bnx2x_setup_leading(bp);
2404         if (rc) {
2405                 BNX2X_ERR("Setup leading failed!\n");
2406                 LOAD_ERROR_EXIT(bp, load_error3);
2407         }
2408
2409         for_each_nondefault_eth_queue(bp, i) {
2410                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2411                 if (rc) {
2412                         BNX2X_ERR("Queue setup failed\n");
2413                         LOAD_ERROR_EXIT(bp, load_error3);
2414                 }
2415         }
2416
2417         rc = bnx2x_init_rss_pf(bp);
2418         if (rc) {
2419                 BNX2X_ERR("PF RSS init failed\n");
2420                 LOAD_ERROR_EXIT(bp, load_error3);
2421         }
2422
2423         /* Now when Clients are configured we are ready to work */
2424         bp->state = BNX2X_STATE_OPEN;
2425
2426         /* Configure a ucast MAC */
2427         rc = bnx2x_set_eth_mac(bp, true);
2428         if (rc) {
2429                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2430                 LOAD_ERROR_EXIT(bp, load_error3);
2431         }
2432
2433         if (bp->pending_max) {
2434                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2435                 bp->pending_max = 0;
2436         }
2437
2438         if (bp->port.pmf)
2439                 bnx2x_initial_phy_init(bp, load_mode);
2440         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2441
2442         /* Start fast path */
2443
2444         /* Initialize Rx filter. */
2445         netif_addr_lock_bh(bp->dev);
2446         bnx2x_set_rx_mode(bp->dev);
2447         netif_addr_unlock_bh(bp->dev);
2448
2449         /* Start the Tx */
2450         switch (load_mode) {
2451         case LOAD_NORMAL:
2452                 /* Tx queue should be only reenabled */
2453                 netif_tx_wake_all_queues(bp->dev);
2454                 break;
2455
2456         case LOAD_OPEN:
2457                 netif_tx_start_all_queues(bp->dev);
2458                 smp_mb__after_clear_bit();
2459                 break;
2460
2461         case LOAD_DIAG:
2462         case LOAD_LOOPBACK_EXT:
2463                 bp->state = BNX2X_STATE_DIAG;
2464                 break;
2465
2466         default:
2467                 break;
2468         }
2469
2470         if (bp->port.pmf)
2471                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2472         else
2473                 bnx2x__link_status_update(bp);
2474
2475         /* start the timer */
2476         mod_timer(&bp->timer, jiffies + bp->current_interval);
2477
2478         if (CNIC_ENABLED(bp))
2479                 bnx2x_load_cnic(bp);
2480
2481         /* mark driver is loaded in shmem2 */
2482         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2483                 u32 val;
2484                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2485                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2486                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2487                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2488         }
2489
2490         /* Wait for all pending SP commands to complete */
2491         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2492                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2493                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2494                 return -EBUSY;
2495         }
2496
2497         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2498         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2499                 bnx2x_dcbx_init(bp, false);
2500
2501         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2502
2503         return 0;
2504
2505 #ifndef BNX2X_STOP_ON_ERROR
2506 load_error3:
2507         bnx2x_int_disable_sync(bp, 1);
2508
2509         /* Clean queueable objects */
2510         bnx2x_squeeze_objects(bp);
2511
2512         /* Free SKBs, SGEs, TPA pool and driver internals */
2513         bnx2x_free_skbs(bp);
2514         for_each_rx_queue(bp, i)
2515                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2516
2517         /* Release IRQs */
2518         bnx2x_free_irq(bp);
2519 load_error2:
2520         if (!BP_NOMCP(bp)) {
2521                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2522                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2523         }
2524
2525         bp->port.pmf = 0;
2526 load_error1:
2527         bnx2x_napi_disable(bp);
2528         /* clear pf_load status, as it was already set */
2529         bnx2x_clear_pf_load(bp);
2530 load_error0:
2531         bnx2x_free_mem(bp);
2532
2533         return rc;
2534 #endif /* ! BNX2X_STOP_ON_ERROR */
2535 }
2536
2537 /* must be called with rtnl_lock */
2538 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2539 {
2540         int i;
2541         bool global = false;
2542
2543         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2544
2545         /* mark driver is unloaded in shmem2 */
2546         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2547                 u32 val;
2548                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2549                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2550                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2551         }
2552
2553         if ((bp->state == BNX2X_STATE_CLOSED) ||
2554             (bp->state == BNX2X_STATE_ERROR)) {
2555                 /* We can get here if the driver has been unloaded
2556                  * during parity error recovery and is either waiting for a
2557                  * leader to complete or for other functions to unload and
2558                  * then ifdown has been issued. In this case we want to
2559                  * unload and let other functions to complete a recovery
2560                  * process.
2561                  */
2562                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2563                 bp->is_leader = 0;
2564                 bnx2x_release_leader_lock(bp);
2565                 smp_mb();
2566
2567                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2568                 BNX2X_ERR("Can't unload in closed or error state\n");
2569                 return -EINVAL;
2570         }
2571
2572         /*
2573          * It's important to set the bp->state to the value different from
2574          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2575          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2576          */
2577         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2578         smp_mb();
2579
2580         if (CNIC_LOADED(bp))
2581                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2582
2583         /* Stop Tx */
2584         bnx2x_tx_disable(bp);
2585         netdev_reset_tc(bp->dev);
2586
2587         bp->rx_mode = BNX2X_RX_MODE_NONE;
2588
2589         del_timer_sync(&bp->timer);
2590
2591         /* Set ALWAYS_ALIVE bit in shmem */
2592         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2593
2594         bnx2x_drv_pulse(bp);
2595
2596         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2597         bnx2x_save_statistics(bp);
2598
2599         /* Cleanup the chip if needed */
2600         if (unload_mode != UNLOAD_RECOVERY)
2601                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2602         else {
2603                 /* Send the UNLOAD_REQUEST to the MCP */
2604                 bnx2x_send_unload_req(bp, unload_mode);
2605
2606                 /*
2607                  * Prevent transactions to host from the functions on the
2608                  * engine that doesn't reset global blocks in case of global
2609                  * attention once gloabl blocks are reset and gates are opened
2610                  * (the engine which leader will perform the recovery
2611                  * last).
2612                  */
2613                 if (!CHIP_IS_E1x(bp))
2614                         bnx2x_pf_disable(bp);
2615
2616                 /* Disable HW interrupts, NAPI */
2617                 bnx2x_netif_stop(bp, 1);
2618                 /* Delete all NAPI objects */
2619                 bnx2x_del_all_napi(bp);
2620                 if (CNIC_LOADED(bp))
2621                         bnx2x_del_all_napi_cnic(bp);
2622                 /* Release IRQs */
2623                 bnx2x_free_irq(bp);
2624
2625                 /* Report UNLOAD_DONE to MCP */
2626                 bnx2x_send_unload_done(bp, false);
2627         }
2628
2629         /*
2630          * At this stage no more interrupts will arrive so we may safly clean
2631          * the queueable objects here in case they failed to get cleaned so far.
2632          */
2633         bnx2x_squeeze_objects(bp);
2634
2635         /* There should be no more pending SP commands at this stage */
2636         bp->sp_state = 0;
2637
2638         bp->port.pmf = 0;
2639
2640         /* Free SKBs, SGEs, TPA pool and driver internals */
2641         bnx2x_free_skbs(bp);
2642         if (CNIC_LOADED(bp))
2643                 bnx2x_free_skbs_cnic(bp);
2644         for_each_rx_queue(bp, i)
2645                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2646
2647         if (CNIC_LOADED(bp)) {
2648                 bnx2x_free_fp_mem_cnic(bp);
2649                 bnx2x_free_mem_cnic(bp);
2650         }
2651         bnx2x_free_mem(bp);
2652
2653         bp->state = BNX2X_STATE_CLOSED;
2654         bp->cnic_loaded = false;
2655
2656         /* Check if there are pending parity attentions. If there are - set
2657          * RECOVERY_IN_PROGRESS.
2658          */
2659         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2660                 bnx2x_set_reset_in_progress(bp);
2661
2662                 /* Set RESET_IS_GLOBAL if needed */
2663                 if (global)
2664                         bnx2x_set_reset_global(bp);
2665         }
2666
2667
2668         /* The last driver must disable a "close the gate" if there is no
2669          * parity attention or "process kill" pending.
2670          */
2671         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2672                 bnx2x_disable_close_the_gate(bp);
2673
2674         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2675
2676         return 0;
2677 }
2678
2679 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2680 {
2681         u16 pmcsr;
2682
2683         /* If there is no power capability, silently succeed */
2684         if (!bp->pm_cap) {
2685                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2686                 return 0;
2687         }
2688
2689         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2690
2691         switch (state) {
2692         case PCI_D0:
2693                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2694                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2695                                        PCI_PM_CTRL_PME_STATUS));
2696
2697                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2698                         /* delay required during transition out of D3hot */
2699                         msleep(20);
2700                 break;
2701
2702         case PCI_D3hot:
2703                 /* If there are other clients above don't
2704                    shut down the power */
2705                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2706                         return 0;
2707                 /* Don't shut down the power for emulation and FPGA */
2708                 if (CHIP_REV_IS_SLOW(bp))
2709                         return 0;
2710
2711                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2712                 pmcsr |= 3;
2713
2714                 if (bp->wol)
2715                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2716
2717                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2718                                       pmcsr);
2719
2720                 /* No more memory access after this point until
2721                 * device is brought back to D0.
2722                 */
2723                 break;
2724
2725         default:
2726                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2727                 return -EINVAL;
2728         }
2729         return 0;
2730 }
2731
2732 /*
2733  * net_device service functions
2734  */
2735 int bnx2x_poll(struct napi_struct *napi, int budget)
2736 {
2737         int work_done = 0;
2738         u8 cos;
2739         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2740                                                  napi);
2741         struct bnx2x *bp = fp->bp;
2742
2743         while (1) {
2744 #ifdef BNX2X_STOP_ON_ERROR
2745                 if (unlikely(bp->panic)) {
2746                         napi_complete(napi);
2747                         return 0;
2748                 }
2749 #endif
2750
2751                 for_each_cos_in_tx_queue(fp, cos)
2752                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
2753                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
2754
2755
2756                 if (bnx2x_has_rx_work(fp)) {
2757                         work_done += bnx2x_rx_int(fp, budget - work_done);
2758
2759                         /* must not complete if we consumed full budget */
2760                         if (work_done >= budget)
2761                                 break;
2762                 }
2763
2764                 /* Fall out from the NAPI loop if needed */
2765                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2766
2767                         /* No need to update SB for FCoE L2 ring as long as
2768                          * it's connected to the default SB and the SB
2769                          * has been updated when NAPI was scheduled.
2770                          */
2771                         if (IS_FCOE_FP(fp)) {
2772                                 napi_complete(napi);
2773                                 break;
2774                         }
2775                         bnx2x_update_fpsb_idx(fp);
2776                         /* bnx2x_has_rx_work() reads the status block,
2777                          * thus we need to ensure that status block indices
2778                          * have been actually read (bnx2x_update_fpsb_idx)
2779                          * prior to this check (bnx2x_has_rx_work) so that
2780                          * we won't write the "newer" value of the status block
2781                          * to IGU (if there was a DMA right after
2782                          * bnx2x_has_rx_work and if there is no rmb, the memory
2783                          * reading (bnx2x_update_fpsb_idx) may be postponed
2784                          * to right before bnx2x_ack_sb). In this case there
2785                          * will never be another interrupt until there is
2786                          * another update of the status block, while there
2787                          * is still unhandled work.
2788                          */
2789                         rmb();
2790
2791                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2792                                 napi_complete(napi);
2793                                 /* Re-enable interrupts */
2794                                 DP(NETIF_MSG_RX_STATUS,
2795                                    "Update index to %d\n", fp->fp_hc_idx);
2796                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2797                                              le16_to_cpu(fp->fp_hc_idx),
2798                                              IGU_INT_ENABLE, 1);
2799                                 break;
2800                         }
2801                 }
2802         }
2803
2804         return work_done;
2805 }
2806
2807 /* we split the first BD into headers and data BDs
2808  * to ease the pain of our fellow microcode engineers
2809  * we use one mapping for both BDs
2810  */
2811 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2812                                    struct bnx2x_fp_txdata *txdata,
2813                                    struct sw_tx_bd *tx_buf,
2814                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2815                                    u16 bd_prod, int nbd)
2816 {
2817         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2818         struct eth_tx_bd *d_tx_bd;
2819         dma_addr_t mapping;
2820         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2821
2822         /* first fix first BD */
2823         h_tx_bd->nbd = cpu_to_le16(nbd);
2824         h_tx_bd->nbytes = cpu_to_le16(hlen);
2825
2826         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2827            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2828
2829         /* now get a new data BD
2830          * (after the pbd) and fill it */
2831         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2832         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2833
2834         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2835                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2836
2837         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2838         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2839         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2840
2841         /* this marks the BD as one that has no individual mapping */
2842         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2843
2844         DP(NETIF_MSG_TX_QUEUED,
2845            "TSO split data size is %d (%x:%x)\n",
2846            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2847
2848         /* update tx_bd */
2849         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2850
2851         return bd_prod;
2852 }
2853
2854 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2855 {
2856         if (fix > 0)
2857                 csum = (u16) ~csum_fold(csum_sub(csum,
2858                                 csum_partial(t_header - fix, fix, 0)));
2859
2860         else if (fix < 0)
2861                 csum = (u16) ~csum_fold(csum_add(csum,
2862                                 csum_partial(t_header, -fix, 0)));
2863
2864         return swab16(csum);
2865 }
2866
2867 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2868 {
2869         u32 rc;
2870
2871         if (skb->ip_summed != CHECKSUM_PARTIAL)
2872                 rc = XMIT_PLAIN;
2873
2874         else {
2875                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2876                         rc = XMIT_CSUM_V6;
2877                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2878                                 rc |= XMIT_CSUM_TCP;
2879
2880                 } else {
2881                         rc = XMIT_CSUM_V4;
2882                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2883                                 rc |= XMIT_CSUM_TCP;
2884                 }
2885         }
2886
2887         if (skb_is_gso_v6(skb))
2888                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2889         else if (skb_is_gso(skb))
2890                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2891
2892         return rc;
2893 }
2894
2895 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2896 /* check if packet requires linearization (packet is too fragmented)
2897    no need to check fragmentation if page size > 8K (there will be no
2898    violation to FW restrictions) */
2899 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2900                              u32 xmit_type)
2901 {
2902         int to_copy = 0;
2903         int hlen = 0;
2904         int first_bd_sz = 0;
2905
2906         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2907         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2908
2909                 if (xmit_type & XMIT_GSO) {
2910                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2911                         /* Check if LSO packet needs to be copied:
2912                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2913                         int wnd_size = MAX_FETCH_BD - 3;
2914                         /* Number of windows to check */
2915                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2916                         int wnd_idx = 0;
2917                         int frag_idx = 0;
2918                         u32 wnd_sum = 0;
2919
2920                         /* Headers length */
2921                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2922                                 tcp_hdrlen(skb);
2923
2924                         /* Amount of data (w/o headers) on linear part of SKB*/
2925                         first_bd_sz = skb_headlen(skb) - hlen;
2926
2927                         wnd_sum  = first_bd_sz;
2928
2929                         /* Calculate the first sum - it's special */
2930                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2931                                 wnd_sum +=
2932                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2933
2934                         /* If there was data on linear skb data - check it */
2935                         if (first_bd_sz > 0) {
2936                                 if (unlikely(wnd_sum < lso_mss)) {
2937                                         to_copy = 1;
2938                                         goto exit_lbl;
2939                                 }
2940
2941                                 wnd_sum -= first_bd_sz;
2942                         }
2943
2944                         /* Others are easier: run through the frag list and
2945                            check all windows */
2946                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2947                                 wnd_sum +=
2948                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2949
2950                                 if (unlikely(wnd_sum < lso_mss)) {
2951                                         to_copy = 1;
2952                                         break;
2953                                 }
2954                                 wnd_sum -=
2955                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2956                         }
2957                 } else {
2958                         /* in non-LSO too fragmented packet should always
2959                            be linearized */
2960                         to_copy = 1;
2961                 }
2962         }
2963
2964 exit_lbl:
2965         if (unlikely(to_copy))
2966                 DP(NETIF_MSG_TX_QUEUED,
2967                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2968                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2969                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2970
2971         return to_copy;
2972 }
2973 #endif
2974
2975 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2976                                         u32 xmit_type)
2977 {
2978         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2979                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2980                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2981         if ((xmit_type & XMIT_GSO_V6) &&
2982             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2983                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2984 }
2985
2986 /**
2987  * bnx2x_set_pbd_gso - update PBD in GSO case.
2988  *
2989  * @skb:        packet skb
2990  * @pbd:        parse BD
2991  * @xmit_type:  xmit flags
2992  */
2993 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2994                                      struct eth_tx_parse_bd_e1x *pbd,
2995                                      u32 xmit_type)
2996 {
2997         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2998         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2999         pbd->tcp_flags = pbd_tcp_flags(skb);
3000
3001         if (xmit_type & XMIT_GSO_V4) {
3002                 pbd->ip_id = swab16(ip_hdr(skb)->id);
3003                 pbd->tcp_pseudo_csum =
3004                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3005                                                   ip_hdr(skb)->daddr,
3006                                                   0, IPPROTO_TCP, 0));
3007
3008         } else
3009                 pbd->tcp_pseudo_csum =
3010                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3011                                                 &ipv6_hdr(skb)->daddr,
3012                                                 0, IPPROTO_TCP, 0));
3013
3014         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
3015 }
3016
3017 /**
3018  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3019  *
3020  * @bp:                 driver handle
3021  * @skb:                packet skb
3022  * @parsing_data:       data to be updated
3023  * @xmit_type:          xmit flags
3024  *
3025  * 57712 related
3026  */
3027 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3028         u32 *parsing_data, u32 xmit_type)
3029 {
3030         *parsing_data |=
3031                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3032                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
3033                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
3034
3035         if (xmit_type & XMIT_CSUM_TCP) {
3036                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3037                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3038                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3039
3040                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3041         } else
3042                 /* We support checksum offload for TCP and UDP only.
3043                  * No need to pass the UDP header length - it's a constant.
3044                  */
3045                 return skb_transport_header(skb) +
3046                                 sizeof(struct udphdr) - skb->data;
3047 }
3048
3049 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3050         struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
3051 {
3052         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3053
3054         if (xmit_type & XMIT_CSUM_V4)
3055                 tx_start_bd->bd_flags.as_bitfield |=
3056                                         ETH_TX_BD_FLAGS_IP_CSUM;
3057         else
3058                 tx_start_bd->bd_flags.as_bitfield |=
3059                                         ETH_TX_BD_FLAGS_IPV6;
3060
3061         if (!(xmit_type & XMIT_CSUM_TCP))
3062                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3063 }
3064
3065 /**
3066  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3067  *
3068  * @bp:         driver handle
3069  * @skb:        packet skb
3070  * @pbd:        parse BD to be updated
3071  * @xmit_type:  xmit flags
3072  */
3073 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3074         struct eth_tx_parse_bd_e1x *pbd,
3075         u32 xmit_type)
3076 {
3077         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3078
3079         /* for now NS flag is not used in Linux */
3080         pbd->global_data =
3081                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3082                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3083
3084         pbd->ip_hlen_w = (skb_transport_header(skb) -
3085                         skb_network_header(skb)) >> 1;
3086
3087         hlen += pbd->ip_hlen_w;
3088
3089         /* We support checksum offload for TCP and UDP only */
3090         if (xmit_type & XMIT_CSUM_TCP)
3091                 hlen += tcp_hdrlen(skb) / 2;
3092         else
3093                 hlen += sizeof(struct udphdr) / 2;
3094
3095         pbd->total_hlen_w = cpu_to_le16(hlen);
3096         hlen = hlen*2;
3097
3098         if (xmit_type & XMIT_CSUM_TCP) {
3099                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
3100
3101         } else {
3102                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3103
3104                 DP(NETIF_MSG_TX_QUEUED,
3105                    "hlen %d  fix %d  csum before fix %x\n",
3106                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3107
3108                 /* HW bug: fixup the CSUM */
3109                 pbd->tcp_pseudo_csum =
3110                         bnx2x_csum_fix(skb_transport_header(skb),
3111                                        SKB_CS(skb), fix);
3112
3113                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3114                    pbd->tcp_pseudo_csum);
3115         }
3116
3117         return hlen;
3118 }
3119
3120 /* called with netif_tx_lock
3121  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3122  * netif_wake_queue()
3123  */
3124 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3125 {
3126         struct bnx2x *bp = netdev_priv(dev);
3127
3128         struct netdev_queue *txq;
3129         struct bnx2x_fp_txdata *txdata;
3130         struct sw_tx_bd *tx_buf;
3131         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3132         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3133         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3134         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3135         u32 pbd_e2_parsing_data = 0;
3136         u16 pkt_prod, bd_prod;
3137         int nbd, txq_index;
3138         dma_addr_t mapping;
3139         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3140         int i;
3141         u8 hlen = 0;
3142         __le16 pkt_size = 0;
3143         struct ethhdr *eth;
3144         u8 mac_type = UNICAST_ADDRESS;
3145
3146 #ifdef BNX2X_STOP_ON_ERROR
3147         if (unlikely(bp->panic))
3148                 return NETDEV_TX_BUSY;
3149 #endif
3150
3151         txq_index = skb_get_queue_mapping(skb);
3152         txq = netdev_get_tx_queue(dev, txq_index);
3153
3154         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3155
3156         txdata = &bp->bnx2x_txq[txq_index];
3157
3158         /* enable this debug print to view the transmission queue being used
3159         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3160            txq_index, fp_index, txdata_index); */
3161
3162         /* enable this debug print to view the tranmission details
3163         DP(NETIF_MSG_TX_QUEUED,
3164            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3165            txdata->cid, fp_index, txdata_index, txdata, fp); */
3166
3167         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3168                         skb_shinfo(skb)->nr_frags +
3169                         BDS_PER_TX_PKT +
3170                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3171                 /* Handle special storage cases separately */
3172                 if (txdata->tx_ring_size == 0) {
3173                         struct bnx2x_eth_q_stats *q_stats =
3174                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3175                         q_stats->driver_filtered_tx_pkt++;
3176                         dev_kfree_skb(skb);
3177                         return NETDEV_TX_OK;
3178                 }
3179                         bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3180                         netif_tx_stop_queue(txq);
3181                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3182
3183                 return NETDEV_TX_BUSY;
3184         }
3185
3186         DP(NETIF_MSG_TX_QUEUED,
3187            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x\n",
3188            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3189            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
3190
3191         eth = (struct ethhdr *)skb->data;
3192
3193         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3194         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3195                 if (is_broadcast_ether_addr(eth->h_dest))
3196                         mac_type = BROADCAST_ADDRESS;
3197                 else
3198                         mac_type = MULTICAST_ADDRESS;
3199         }
3200
3201 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3202         /* First, check if we need to linearize the skb (due to FW
3203            restrictions). No need to check fragmentation if page size > 8K
3204            (there will be no violation to FW restrictions) */
3205         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3206                 /* Statistics of linearization */
3207                 bp->lin_cnt++;
3208                 if (skb_linearize(skb) != 0) {
3209                         DP(NETIF_MSG_TX_QUEUED,
3210                            "SKB linearization failed - silently dropping this SKB\n");
3211                         dev_kfree_skb_any(skb);
3212                         return NETDEV_TX_OK;
3213                 }
3214         }
3215 #endif
3216         /* Map skb linear data for DMA */
3217         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3218                                  skb_headlen(skb), DMA_TO_DEVICE);
3219         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3220                 DP(NETIF_MSG_TX_QUEUED,
3221                    "SKB mapping failed - silently dropping this SKB\n");
3222                 dev_kfree_skb_any(skb);
3223                 return NETDEV_TX_OK;
3224         }
3225         /*
3226         Please read carefully. First we use one BD which we mark as start,
3227         then we have a parsing info BD (used for TSO or xsum),
3228         and only then we have the rest of the TSO BDs.
3229         (don't forget to mark the last one as last,
3230         and to unmap only AFTER you write to the BD ...)
3231         And above all, all pdb sizes are in words - NOT DWORDS!
3232         */
3233
3234         /* get current pkt produced now - advance it just before sending packet
3235          * since mapping of pages may fail and cause packet to be dropped
3236          */
3237         pkt_prod = txdata->tx_pkt_prod;
3238         bd_prod = TX_BD(txdata->tx_bd_prod);
3239
3240         /* get a tx_buf and first BD
3241          * tx_start_bd may be changed during SPLIT,
3242          * but first_bd will always stay first
3243          */
3244         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3245         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3246         first_bd = tx_start_bd;
3247
3248         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3249         SET_FLAG(tx_start_bd->general_data,
3250                  ETH_TX_START_BD_PARSE_NBDS,
3251                  0);
3252
3253         /* header nbd */
3254         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
3255
3256         /* remember the first BD of the packet */
3257         tx_buf->first_bd = txdata->tx_bd_prod;
3258         tx_buf->skb = skb;
3259         tx_buf->flags = 0;
3260
3261         DP(NETIF_MSG_TX_QUEUED,
3262            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3263            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3264
3265         if (vlan_tx_tag_present(skb)) {
3266                 tx_start_bd->vlan_or_ethertype =
3267                     cpu_to_le16(vlan_tx_tag_get(skb));
3268                 tx_start_bd->bd_flags.as_bitfield |=
3269                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3270         } else
3271                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3272
3273         /* turn on parsing and get a BD */
3274         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3275
3276         if (xmit_type & XMIT_CSUM)
3277                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3278
3279         if (!CHIP_IS_E1x(bp)) {
3280                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3281                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3282                 /* Set PBD in checksum offload case */
3283                 if (xmit_type & XMIT_CSUM)
3284                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3285                                                      &pbd_e2_parsing_data,
3286                                                      xmit_type);
3287                 if (IS_MF_SI(bp)) {
3288                         /*
3289                          * fill in the MAC addresses in the PBD - for local
3290                          * switching
3291                          */
3292                         bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
3293                                               &pbd_e2->src_mac_addr_mid,
3294                                               &pbd_e2->src_mac_addr_lo,
3295                                               eth->h_source);
3296                         bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
3297                                               &pbd_e2->dst_mac_addr_mid,
3298                                               &pbd_e2->dst_mac_addr_lo,
3299                                               eth->h_dest);
3300                 }
3301
3302                 SET_FLAG(pbd_e2_parsing_data,
3303                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3304         } else {
3305                 u16 global_data = 0;
3306                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3307                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3308                 /* Set PBD in checksum offload case */
3309                 if (xmit_type & XMIT_CSUM)
3310                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3311
3312                 SET_FLAG(global_data,
3313                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3314                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3315         }
3316
3317         /* Setup the data pointer of the first BD of the packet */
3318         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3319         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3320         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3321         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3322         pkt_size = tx_start_bd->nbytes;
3323
3324         DP(NETIF_MSG_TX_QUEUED,
3325            "first bd @%p  addr (%x:%x)  nbd %d  nbytes %d  flags %x  vlan %x\n",
3326            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3327            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
3328            tx_start_bd->bd_flags.as_bitfield,
3329            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3330
3331         if (xmit_type & XMIT_GSO) {
3332
3333                 DP(NETIF_MSG_TX_QUEUED,
3334                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3335                    skb->len, hlen, skb_headlen(skb),
3336                    skb_shinfo(skb)->gso_size);
3337
3338                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3339
3340                 if (unlikely(skb_headlen(skb) > hlen))
3341                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3342                                                  &tx_start_bd, hlen,
3343                                                  bd_prod, ++nbd);
3344                 if (!CHIP_IS_E1x(bp))
3345                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3346                                              xmit_type);
3347                 else
3348                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
3349         }
3350
3351         /* Set the PBD's parsing_data field if not zero
3352          * (for the chips newer than 57711).
3353          */
3354         if (pbd_e2_parsing_data)
3355                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3356
3357         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3358
3359         /* Handle fragmented skb */
3360         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3361                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3362
3363                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3364                                            skb_frag_size(frag), DMA_TO_DEVICE);
3365                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3366                         unsigned int pkts_compl = 0, bytes_compl = 0;
3367
3368                         DP(NETIF_MSG_TX_QUEUED,
3369                            "Unable to map page - dropping packet...\n");
3370
3371                         /* we need unmap all buffers already mapped
3372                          * for this SKB;
3373                          * first_bd->nbd need to be properly updated
3374                          * before call to bnx2x_free_tx_pkt
3375                          */
3376                         first_bd->nbd = cpu_to_le16(nbd);
3377                         bnx2x_free_tx_pkt(bp, txdata,
3378                                           TX_BD(txdata->tx_pkt_prod),
3379                                           &pkts_compl, &bytes_compl);
3380                         return NETDEV_TX_OK;
3381                 }
3382
3383                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3384                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3385                 if (total_pkt_bd == NULL)
3386                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3387
3388                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3389                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3390                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3391                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3392                 nbd++;
3393
3394                 DP(NETIF_MSG_TX_QUEUED,
3395                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3396                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3397                    le16_to_cpu(tx_data_bd->nbytes));
3398         }
3399
3400         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3401
3402         /* update with actual num BDs */
3403         first_bd->nbd = cpu_to_le16(nbd);
3404
3405         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3406
3407         /* now send a tx doorbell, counting the next BD
3408          * if the packet contains or ends with it
3409          */
3410         if (TX_BD_POFF(bd_prod) < nbd)
3411                 nbd++;
3412
3413         /* total_pkt_bytes should be set on the first data BD if
3414          * it's not an LSO packet and there is more than one
3415          * data BD. In this case pkt_size is limited by an MTU value.
3416          * However we prefer to set it for an LSO packet (while we don't
3417          * have to) in order to save some CPU cycles in a none-LSO
3418          * case, when we much more care about them.
3419          */
3420         if (total_pkt_bd != NULL)
3421                 total_pkt_bd->total_pkt_bytes = pkt_size;
3422
3423         if (pbd_e1x)
3424                 DP(NETIF_MSG_TX_QUEUED,
3425                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3426                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3427                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3428                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3429                     le16_to_cpu(pbd_e1x->total_hlen_w));
3430         if (pbd_e2)
3431                 DP(NETIF_MSG_TX_QUEUED,
3432                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3433                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
3434                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
3435                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
3436                    pbd_e2->parsing_data);
3437         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3438
3439         netdev_tx_sent_queue(txq, skb->len);
3440
3441         skb_tx_timestamp(skb);
3442
3443         txdata->tx_pkt_prod++;
3444         /*
3445          * Make sure that the BD data is updated before updating the producer
3446          * since FW might read the BD right after the producer is updated.
3447          * This is only applicable for weak-ordered memory model archs such
3448          * as IA-64. The following barrier is also mandatory since FW will
3449          * assumes packets must have BDs.
3450          */
3451         wmb();