drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2012 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip6_checksum.h>
  26 #include <linux/prefetch.h>
  27 #include "bnx2x_cmn.h"
  28 #include "bnx2x_init.h"
  29 #include "bnx2x_sp.h"
  30
  31
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target. Update txdata pointers and related
  44  * content.
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  54         int old_max_eth_txqs, new_max_eth_txqs;
  55         int old_txdata_index = 0, new_txdata_index = 0;
  56
  57         /* Copy the NAPI object as it has been already initialized */
  58         from_fp->napi = to_fp->napi;
  59
  60         /* Move bnx2x_fastpath contents */
  61         memcpy(to_fp, from_fp, sizeof(*to_fp));
  62         to_fp->index = to;
  63
  64         /* move sp_objs contents as well, as their indices match fp ones */
  65         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  66
  67         /* move fp_stats contents as well, as their indices match fp ones */
  68         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  69
  70         /* Update txdata pointers in fp and move txdata content accordingly:
  71          * Each fp consumes 'max_cos' txdata structures, so the index should be
  72          * decremented by max_cos x delta.
  73          */
  74
  75         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  76         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  77                                 (bp)->max_cos;
  78         if (from == FCOE_IDX(bp)) {
  79                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  81         }
  82
  83         memcpy(&bp->bnx2x_txq[old_txdata_index],
  84                &bp->bnx2x_txq[new_txdata_index],
  85                sizeof(struct bnx2x_fp_txdata));
  86         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  87 }
  88
  89 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
  90
  91 /* free skb in the packet ring at pos idx
  92  * return idx of last bd freed
  93  */
  94 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
  95                              u16 idx, unsigned int *pkts_compl,
  96                              unsigned int *bytes_compl)
  97 {
  98         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
  99         struct eth_tx_start_bd *tx_start_bd;
 100         struct eth_tx_bd *tx_data_bd;
 101         struct sk_buff *skb = tx_buf->skb;
 102         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 103         int nbd;
 104
 105         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 106         prefetch(&skb->end);
 107
 108         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 109            txdata->txq_index, idx, tx_buf, skb);
 110
 111         /* unmap first bd */
 112         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 113         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 114                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 115
 116
 117         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 118 #ifdef BNX2X_STOP_ON_ERROR
 119         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 120                 BNX2X_ERR("BAD nbd!\n");
 121                 bnx2x_panic();
 122         }
 123 #endif
 124         new_cons = nbd + tx_buf->first_bd;
 125
 126         /* Get the next bd */
 127         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 128
 129         /* Skip a parse bd... */
 130         --nbd;
 131         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 132
 133         /* ...and the TSO split header bd since they have no mapping */
 134         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 135                 --nbd;
 136                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 137         }
 138
 139         /* now free frags */
 140         while (nbd > 0) {
 141
 142                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 143                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 144                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 145                 if (--nbd)
 146                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 147         }
 148
 149         /* release skb */
 150         WARN_ON(!skb);
 151         if (likely(skb)) {
 152                 (*pkts_compl)++;
 153                 (*bytes_compl) += skb->len;
 154         }
 155
 156         dev_kfree_skb_any(skb);
 157         tx_buf->first_bd = 0;
 158         tx_buf->skb = NULL;
 159
 160         return new_cons;
 161 }
 162
 163 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 164 {
 165         struct netdev_queue *txq;
 166         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 167         unsigned int pkts_compl = 0, bytes_compl = 0;
 168
 169 #ifdef BNX2X_STOP_ON_ERROR
 170         if (unlikely(bp->panic))
 171                 return -1;
 172 #endif
 173
 174         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 175         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 176         sw_cons = txdata->tx_pkt_cons;
 177
 178         while (sw_cons != hw_cons) {
 179                 u16 pkt_cons;
 180
 181                 pkt_cons = TX_BD(sw_cons);
 182
 183                 DP(NETIF_MSG_TX_DONE,
 184                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 185                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 186
 187                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 188                     &pkts_compl, &bytes_compl);
 189
 190                 sw_cons++;
 191         }
 192
 193         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 194
 195         txdata->tx_pkt_cons = sw_cons;
 196         txdata->tx_bd_cons = bd_cons;
 197
 198         /* Need to make the tx_bd_cons update visible to start_xmit()
 199          * before checking for netif_tx_queue_stopped().  Without the
 200          * memory barrier, there is a small possibility that
 201          * start_xmit() will miss it and cause the queue to be stopped
 202          * forever.
 203          * On the other hand we need an rmb() here to ensure the proper
 204          * ordering of bit testing in the following
 205          * netif_tx_queue_stopped(txq) call.
 206          */
 207         smp_mb();
 208
 209         if (unlikely(netif_tx_queue_stopped(txq))) {
 210                 /* Taking tx_lock() is needed to prevent reenabling the queue
 211                  * while it's empty. This could have happen if rx_action() gets
 212                  * suspended in bnx2x_tx_int() after the condition before
 213                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 214                  *
 215                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 216                  * sends some packets consuming the whole queue again->
 217                  * stops the queue
 218                  */
 219
 220                 __netif_tx_lock(txq, smp_processor_id());
 221
 222                 if ((netif_tx_queue_stopped(txq)) &&
 223                     (bp->state == BNX2X_STATE_OPEN) &&
 224                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 225                         netif_tx_wake_queue(txq);
 226
 227                 __netif_tx_unlock(txq);
 228         }
 229         return 0;
 230 }
 231
 232 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 233                                              u16 idx)
 234 {
 235         u16 last_max = fp->last_max_sge;
 236
 237         if (SUB_S16(idx, last_max) > 0)
 238                 fp->last_max_sge = idx;
 239 }
 240
 241 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 242                                          u16 sge_len,
 243                                          struct eth_end_agg_rx_cqe *cqe)
 244 {
 245         struct bnx2x *bp = fp->bp;
 246         u16 last_max, last_elem, first_elem;
 247         u16 delta = 0;
 248         u16 i;
 249
 250         if (!sge_len)
 251                 return;
 252
 253         /* First mark all used pages */
 254         for (i = 0; i < sge_len; i++)
 255                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 256                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 257
 258         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 259            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 260
 261         /* Here we assume that the last SGE index is the biggest */
 262         prefetch((void *)(fp->sge_mask));
 263         bnx2x_update_last_max_sge(fp,
 264                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 265
 266         last_max = RX_SGE(fp->last_max_sge);
 267         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 268         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 269
 270         /* If ring is not full */
 271         if (last_elem + 1 != first_elem)
 272                 last_elem++;
 273
 274         /* Now update the prod */
 275         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 276                 if (likely(fp->sge_mask[i]))
 277                         break;
 278
 279                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 280                 delta += BIT_VEC64_ELEM_SZ;
 281         }
 282
 283         if (delta > 0) {
 284                 fp->rx_sge_prod += delta;
 285                 /* clear page-end entries */
 286                 bnx2x_clear_sge_mask_next_elems(fp);
 287         }
 288
 289         DP(NETIF_MSG_RX_STATUS,
 290            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 291            fp->last_max_sge, fp->rx_sge_prod);
 292 }
 293
 294 /* Set Toeplitz hash value in the skb using the value from the
 295  * CQE (calculated by HW).
 296  */
 297 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 298                             const struct eth_fast_path_rx_cqe *cqe,
 299                             bool *l4_rxhash)
 300 {
 301         /* Set Toeplitz hash from CQE */
 302         if ((bp->dev->features & NETIF_F_RXHASH) &&
 303             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 304                 enum eth_rss_hash_type htype;
 305
 306                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 307                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 308                              (htype == TCP_IPV6_HASH_TYPE);
 309                 return le32_to_cpu(cqe->rss_hash_result);
 310         }
 311         *l4_rxhash = false;
 312         return 0;
 313 }
 314
 315 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 316                             u16 cons, u16 prod,
 317                             struct eth_fast_path_rx_cqe *cqe)
 318 {
 319         struct bnx2x *bp = fp->bp;
 320         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 321         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 322         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 323         dma_addr_t mapping;
 324         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 325         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 326
 327         /* print error if current state != stop */
 328         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 329                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 330
 331         /* Try to map an empty data buffer from the aggregation info  */
 332         mapping = dma_map_single(&bp->pdev->dev,
 333                                  first_buf->data + NET_SKB_PAD,
 334                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 335         /*
 336          *  ...if it fails - move the skb from the consumer to the producer
 337          *  and set the current aggregation state as ERROR to drop it
 338          *  when TPA_STOP arrives.
 339          */
 340
 341         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 342                 /* Move the BD from the consumer to the producer */
 343                 bnx2x_reuse_rx_data(fp, cons, prod);
 344                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 345                 return;
 346         }
 347
 348         /* move empty data from pool to prod */
 349         prod_rx_buf->data = first_buf->data;
 350         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 351         /* point prod_bd to new data */
 352         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 353         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 354
 355         /* move partial skb from cons to pool (don't unmap yet) */
 356         *first_buf = *cons_rx_buf;
 357
 358         /* mark bin state as START */
 359         tpa_info->parsing_flags =
 360                 le16_to_cpu(cqe->pars_flags.flags);
 361         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 362         tpa_info->tpa_state = BNX2X_TPA_START;
 363         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 364         tpa_info->placement_offset = cqe->placement_offset;
 365         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 366         if (fp->mode == TPA_MODE_GRO) {
 367                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 368                 tpa_info->full_page =
 369                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
 370                 tpa_info->gro_size = gro_size;
 371         }
 372
 373 #ifdef BNX2X_STOP_ON_ERROR
 374         fp->tpa_queue_used |= (1 << queue);
 375 #ifdef _ASM_GENERIC_INT_L64_H
 376         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 377 #else
 378         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 379 #endif
 380            fp->tpa_queue_used);
 381 #endif
 382 }
 383
 384 /* Timestamp option length allowed for TPA aggregation:
 385  *
 386  *              nop nop kind length echo val
 387  */
 388 #define TPA_TSTAMP_OPT_LEN      12
 389 /**
 390  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 391  *
 392  * @bp:                 driver handle
 393  * @parsing_flags:      parsing flags from the START CQE
 394  * @len_on_bd:          total length of the first packet for the
 395  *                      aggregation.
 396  *
 397  * Approximate value of the MSS for this aggregation calculated using
 398  * the first packet of it.
 399  */
 400 static u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 401                              u16 len_on_bd)
 402 {
 403         /*
 404          * TPA arrgregation won't have either IP options or TCP options
 405          * other than timestamp or IPv6 extension headers.
 406          */
 407         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 408
 409         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 410             PRS_FLAG_OVERETH_IPV6)
 411                 hdrs_len += sizeof(struct ipv6hdr);
 412         else /* IPv4 */
 413                 hdrs_len += sizeof(struct iphdr);
 414
 415
 416         /* Check if there was a TCP timestamp, if there is it's will
 417          * always be 12 bytes length: nop nop kind length echo val.
 418          *
 419          * Otherwise FW would close the aggregation.
 420          */
 421         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 422                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 423
 424         return len_on_bd - hdrs_len;
 425 }
 426
 427 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 428                               struct bnx2x_fastpath *fp, u16 index)
 429 {
 430         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 431         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 432         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 433         dma_addr_t mapping;
 434
 435         if (unlikely(page == NULL)) {
 436                 BNX2X_ERR("Can't alloc sge\n");
 437                 return -ENOMEM;
 438         }
 439
 440         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 441                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 442         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 443                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 444                 BNX2X_ERR("Can't map sge\n");
 445                 return -ENOMEM;
 446         }
 447
 448         sw_buf->page = page;
 449         dma_unmap_addr_set(sw_buf, mapping, mapping);
 450
 451         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 452         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 453
 454         return 0;
 455 }
 456
 457 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 458                                struct bnx2x_agg_info *tpa_info,
 459                                u16 pages,
 460                                struct sk_buff *skb,
 461                                struct eth_end_agg_rx_cqe *cqe,
 462                                u16 cqe_idx)
 463 {
 464         struct sw_rx_page *rx_pg, old_rx_pg;
 465         u32 i, frag_len, frag_size;
 466         int err, j, frag_id = 0;
 467         u16 len_on_bd = tpa_info->len_on_bd;
 468         u16 full_page = 0, gro_size = 0;
 469
 470         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 471
 472         if (fp->mode == TPA_MODE_GRO) {
 473                 gro_size = tpa_info->gro_size;
 474                 full_page = tpa_info->full_page;
 475         }
 476
 477         /* This is needed in order to enable forwarding support */
 478         if (frag_size) {
 479                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
 480                                         tpa_info->parsing_flags, len_on_bd);
 481
 482                 /* set for GRO */
 483                 if (fp->mode == TPA_MODE_GRO)
 484                         skb_shinfo(skb)->gso_type =
 485                             (GET_FLAG(tpa_info->parsing_flags,
 486                                       PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 487                                                 PRS_FLAG_OVERETH_IPV6) ?
 488                                 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
 489         }
 490
 491
 492 #ifdef BNX2X_STOP_ON_ERROR
 493         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 494                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 495                           pages, cqe_idx);
 496                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 497                 bnx2x_panic();
 498                 return -EINVAL;
 499         }
 500 #endif
 501
 502         /* Run through the SGL and compose the fragmented skb */
 503         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 504                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 505
 506                 /* FW gives the indices of the SGE as if the ring is an array
 507                    (meaning that "next" element will consume 2 indices) */
 508                 if (fp->mode == TPA_MODE_GRO)
 509                         frag_len = min_t(u32, frag_size, (u32)full_page);
 510                 else /* LRO */
 511                         frag_len = min_t(u32, frag_size,
 512                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
 513
 514                 rx_pg = &fp->rx_page_ring[sge_idx];
 515                 old_rx_pg = *rx_pg;
 516
 517                 /* If we fail to allocate a substitute page, we simply stop
 518                    where we are and drop the whole packet */
 519                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 520                 if (unlikely(err)) {
 521                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 522                         return err;
 523                 }
 524
 525                 /* Unmap the page as we r going to pass it to the stack */
 526                 dma_unmap_page(&bp->pdev->dev,
 527                                dma_unmap_addr(&old_rx_pg, mapping),
 528                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 529                 /* Add one frag and update the appropriate fields in the skb */
 530                 if (fp->mode == TPA_MODE_LRO)
 531                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 532                 else { /* GRO */
 533                         int rem;
 534                         int offset = 0;
 535                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 536                                 int len = rem > gro_size ? gro_size : rem;
 537                                 skb_fill_page_desc(skb, frag_id++,
 538                                                    old_rx_pg.page, offset, len);
 539                                 if (offset)
 540                                         get_page(old_rx_pg.page);
 541                                 offset += len;
 542                         }
 543                 }
 544
 545                 skb->data_len += frag_len;
 546                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
 547                 skb->len += frag_len;
 548
 549                 frag_size -= frag_len;
 550         }
 551
 552         return 0;
 553 }
 554
 555 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 556 {
 557         if (fp->rx_frag_size)
 558                 put_page(virt_to_head_page(data));
 559         else
 560                 kfree(data);
 561 }
 562
 563 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 564 {
 565         if (fp->rx_frag_size)
 566                 return netdev_alloc_frag(fp->rx_frag_size);
 567
 568         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 569 }
 570
 571
 572 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 573                            struct bnx2x_agg_info *tpa_info,
 574                            u16 pages,
 575                            struct eth_end_agg_rx_cqe *cqe,
 576                            u16 cqe_idx)
 577 {
 578         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 579         u8 pad = tpa_info->placement_offset;
 580         u16 len = tpa_info->len_on_bd;
 581         struct sk_buff *skb = NULL;
 582         u8 *new_data, *data = rx_buf->data;
 583         u8 old_tpa_state = tpa_info->tpa_state;
 584
 585         tpa_info->tpa_state = BNX2X_TPA_STOP;
 586
 587         /* If we there was an error during the handling of the TPA_START -
 588          * drop this aggregation.
 589          */
 590         if (old_tpa_state == BNX2X_TPA_ERROR)
 591                 goto drop;
 592
 593         /* Try to allocate the new data */
 594         new_data = bnx2x_frag_alloc(fp);
 595         /* Unmap skb in the pool anyway, as we are going to change
 596            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 597            fails. */
 598         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 599                          fp->rx_buf_size, DMA_FROM_DEVICE);
 600         if (likely(new_data))
 601                 skb = build_skb(data, fp->rx_frag_size);
 602
 603         if (likely(skb)) {
 604 #ifdef BNX2X_STOP_ON_ERROR
 605                 if (pad + len > fp->rx_buf_size) {
 606                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 607                                   pad, len, fp->rx_buf_size);
 608                         bnx2x_panic();
 609                         return;
 610                 }
 611 #endif
 612
 613                 skb_reserve(skb, pad + NET_SKB_PAD);
 614                 skb_put(skb, len);
 615                 skb->rxhash = tpa_info->rxhash;
 616                 skb->l4_rxhash = tpa_info->l4_rxhash;
 617
 618                 skb->protocol = eth_type_trans(skb, bp->dev);
 619                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 620
 621                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 622                                          skb, cqe, cqe_idx)) {
 623                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 624                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 625                         napi_gro_receive(&fp->napi, skb);
 626                 } else {
 627                         DP(NETIF_MSG_RX_STATUS,
 628                            "Failed to allocate new pages - dropping packet!\n");
 629                         dev_kfree_skb_any(skb);
 630                 }
 631
 632
 633                 /* put new data in bin */
 634                 rx_buf->data = new_data;
 635
 636                 return;
 637         }
 638         bnx2x_frag_free(fp, new_data);
 639 drop:
 640         /* drop the packet and keep the buffer in the bin */
 641         DP(NETIF_MSG_RX_STATUS,
 642            "Failed to allocate or map a new skb - dropping packet!\n");
 643         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 644 }
 645
 646 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 647                                struct bnx2x_fastpath *fp, u16 index)
 648 {
 649         u8 *data;
 650         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 651         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 652         dma_addr_t mapping;
 653
 654         data = bnx2x_frag_alloc(fp);
 655         if (unlikely(data == NULL))
 656                 return -ENOMEM;
 657
 658         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 659                                  fp->rx_buf_size,
 660                                  DMA_FROM_DEVICE);
 661         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 662                 bnx2x_frag_free(fp, data);
 663                 BNX2X_ERR("Can't map rx data\n");
 664                 return -ENOMEM;
 665         }
 666
 667         rx_buf->data = data;
 668         dma_unmap_addr_set(rx_buf, mapping, mapping);
 669
 670         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 671         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 672
 673         return 0;
 674 }
 675
 676 static
 677 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 678                                  struct bnx2x_fastpath *fp,
 679                                  struct bnx2x_eth_q_stats *qstats)
 680 {
 681         /* Do nothing if no L4 csum validation was done.
 682          * We do not check whether IP csum was validated. For IPv4 we assume
 683          * that if the card got as far as validating the L4 csum, it also
 684          * validated the IP csum. IPv6 has no IP csum.
 685          */
 686         if (cqe->fast_path_cqe.status_flags &
 687             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 688                 return;
 689
 690         /* If L4 validation was done, check if an error was found. */
 691
 692         if (cqe->fast_path_cqe.type_error_flags &
 693             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 694              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 695                 qstats->hw_csum_err++;
 696         else
 697                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 698 }
 699
 700 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 701 {
 702         struct bnx2x *bp = fp->bp;
 703         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 704         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 705         int rx_pkt = 0;
 706
 707 #ifdef BNX2X_STOP_ON_ERROR
 708         if (unlikely(bp->panic))
 709                 return 0;
 710 #endif
 711
 712         /* CQ "next element" is of the size of the regular element,
 713            that's why it's ok here */
 714         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 715         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 716                 hw_comp_cons++;
 717
 718         bd_cons = fp->rx_bd_cons;
 719         bd_prod = fp->rx_bd_prod;
 720         bd_prod_fw = bd_prod;
 721         sw_comp_cons = fp->rx_comp_cons;
 722         sw_comp_prod = fp->rx_comp_prod;
 723
 724         /* Memory barrier necessary as speculative reads of the rx
 725          * buffer can be ahead of the index in the status block
 726          */
 727         rmb();
 728
 729         DP(NETIF_MSG_RX_STATUS,
 730            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 731            fp->index, hw_comp_cons, sw_comp_cons);
 732
 733         while (sw_comp_cons != hw_comp_cons) {
 734                 struct sw_rx_bd *rx_buf = NULL;
 735                 struct sk_buff *skb;
 736                 union eth_rx_cqe *cqe;
 737                 struct eth_fast_path_rx_cqe *cqe_fp;
 738                 u8 cqe_fp_flags;
 739                 enum eth_rx_cqe_type cqe_fp_type;
 740                 u16 len, pad, queue;
 741                 u8 *data;
 742                 bool l4_rxhash;
 743
 744 #ifdef BNX2X_STOP_ON_ERROR
 745                 if (unlikely(bp->panic))
 746                         return 0;
 747 #endif
 748
 749                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 750                 bd_prod = RX_BD(bd_prod);
 751                 bd_cons = RX_BD(bd_cons);
 752
 753                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 754                 cqe_fp = &cqe->fast_path_cqe;
 755                 cqe_fp_flags = cqe_fp->type_error_flags;
 756                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 757
 758                 DP(NETIF_MSG_RX_STATUS,
 759                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 760                    CQE_TYPE(cqe_fp_flags),
 761                    cqe_fp_flags, cqe_fp->status_flags,
 762                    le32_to_cpu(cqe_fp->rss_hash_result),
 763                    le16_to_cpu(cqe_fp->vlan_tag),
 764                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 765
 766                 /* is this a slowpath msg? */
 767                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 768                         bnx2x_sp_event(fp, cqe);
 769                         goto next_cqe;
 770                 }
 771
 772                 rx_buf = &fp->rx_buf_ring[bd_cons];
 773                 data = rx_buf->data;
 774
 775                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 776                         struct bnx2x_agg_info *tpa_info;
 777                         u16 frag_size, pages;
 778 #ifdef BNX2X_STOP_ON_ERROR
 779                         /* sanity check */
 780                         if (fp->disable_tpa &&
 781                             (CQE_TYPE_START(cqe_fp_type) ||
 782                              CQE_TYPE_STOP(cqe_fp_type)))
 783                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 784                                           CQE_TYPE(cqe_fp_type));
 785 #endif
 786
 787                         if (CQE_TYPE_START(cqe_fp_type)) {
 788                                 u16 queue = cqe_fp->queue_index;
 789                                 DP(NETIF_MSG_RX_STATUS,
 790                                    "calling tpa_start on queue %d\n",
 791                                    queue);
 792
 793                                 bnx2x_tpa_start(fp, queue,
 794                                                 bd_cons, bd_prod,
 795                                                 cqe_fp);
 796
 797                                 goto next_rx;
 798
 799                         }
 800                         queue = cqe->end_agg_cqe.queue_index;
 801                         tpa_info = &fp->tpa_info[queue];
 802                         DP(NETIF_MSG_RX_STATUS,
 803                            "calling tpa_stop on queue %d\n",
 804                            queue);
 805
 806                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 807                                     tpa_info->len_on_bd;
 808
 809                         if (fp->mode == TPA_MODE_GRO)
 810                                 pages = (frag_size + tpa_info->full_page - 1) /
 811                                          tpa_info->full_page;
 812                         else
 813                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 814                                         SGE_PAGE_SHIFT;
 815
 816                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 817                                        &cqe->end_agg_cqe, comp_ring_cons);
 818 #ifdef BNX2X_STOP_ON_ERROR
 819                         if (bp->panic)
 820                                 return 0;
 821 #endif
 822
 823                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 824                         goto next_cqe;
 825                 }
 826                 /* non TPA */
 827                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 828                 pad = cqe_fp->placement_offset;
 829                 dma_sync_single_for_cpu(&bp->pdev->dev,
 830                                         dma_unmap_addr(rx_buf, mapping),
 831                                         pad + RX_COPY_THRESH,
 832                                         DMA_FROM_DEVICE);
 833                 pad += NET_SKB_PAD;
 834                 prefetch(data + pad); /* speedup eth_type_trans() */
 835                 /* is this an error packet? */
 836                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 837                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 838                            "ERROR  flags %x  rx packet %u\n",
 839                            cqe_fp_flags, sw_comp_cons);
 840                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 841                         goto reuse_rx;
 842                 }
 843
 844                 /* Since we don't have a jumbo ring
 845                  * copy small packets if mtu > 1500
 846                  */
 847                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 848                     (len <= RX_COPY_THRESH)) {
 849                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 850                         if (skb == NULL) {
 851                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 852                                    "ERROR  packet dropped because of alloc failure\n");
 853                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 854                                 goto reuse_rx;
 855                         }
 856                         memcpy(skb->data, data + pad, len);
 857                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 858                 } else {
 859                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 860                                 dma_unmap_single(&bp->pdev->dev,
 861                                                  dma_unmap_addr(rx_buf, mapping),
 862                                                  fp->rx_buf_size,
 863                                                  DMA_FROM_DEVICE);
 864                                 skb = build_skb(data, fp->rx_frag_size);
 865                                 if (unlikely(!skb)) {
 866                                         bnx2x_frag_free(fp, data);
 867                                         bnx2x_fp_qstats(bp, fp)->
 868                                                         rx_skb_alloc_failed++;
 869                                         goto next_rx;
 870                                 }
 871                                 skb_reserve(skb, pad);
 872                         } else {
 873                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 874                                    "ERROR  packet dropped because of alloc failure\n");
 875                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 876 reuse_rx:
 877                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 878                                 goto next_rx;
 879                         }
 880                 }
 881
 882                 skb_put(skb, len);
 883                 skb->protocol = eth_type_trans(skb, bp->dev);
 884
 885                 /* Set Toeplitz hash for a none-LRO skb */
 886                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 887                 skb->l4_rxhash = l4_rxhash;
 888
 889                 skb_checksum_none_assert(skb);
 890
 891                 if (bp->dev->features & NETIF_F_RXCSUM)
 892                         bnx2x_csum_validate(skb, cqe, fp,
 893                                             bnx2x_fp_qstats(bp, fp));
 894
 895                 skb_record_rx_queue(skb, fp->rx_queue);
 896
 897                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 898                     PARSING_FLAGS_VLAN)
 899                         __vlan_hwaccel_put_tag(skb,
 900                                                le16_to_cpu(cqe_fp->vlan_tag));
 901                 napi_gro_receive(&fp->napi, skb);
 902
 903
 904 next_rx:
 905                 rx_buf->data = NULL;
 906
 907                 bd_cons = NEXT_RX_IDX(bd_cons);
 908                 bd_prod = NEXT_RX_IDX(bd_prod);
 909                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 910                 rx_pkt++;
 911 next_cqe:
 912                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 913                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 914
 915                 if (rx_pkt == budget)
 916                         break;
 917         } /* while */
 918
 919         fp->rx_bd_cons = bd_cons;
 920         fp->rx_bd_prod = bd_prod_fw;
 921         fp->rx_comp_cons = sw_comp_cons;
 922         fp->rx_comp_prod = sw_comp_prod;
 923
 924         /* Update producers */
 925         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 926                              fp->rx_sge_prod);
 927
 928         fp->rx_pkt += rx_pkt;
 929         fp->rx_calls++;
 930
 931         return rx_pkt;
 932 }
 933
 934 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 935 {
 936         struct bnx2x_fastpath *fp = fp_cookie;
 937         struct bnx2x *bp = fp->bp;
 938         u8 cos;
 939
 940         DP(NETIF_MSG_INTR,
 941            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
 942            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 943         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 944
 945 #ifdef BNX2X_STOP_ON_ERROR
 946         if (unlikely(bp->panic))
 947                 return IRQ_HANDLED;
 948 #endif
 949
 950         /* Handle Rx and Tx according to MSI-X vector */
 951         prefetch(fp->rx_cons_sb);
 952
 953         for_each_cos_in_tx_queue(fp, cos)
 954                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
 955
 956         prefetch(&fp->sb_running_index[SM_RX_ID]);
 957         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 958
 959         return IRQ_HANDLED;
 960 }
 961
 962 /* HW Lock for shared dual port PHYs */
 963 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 964 {
 965         mutex_lock(&bp->port.phy_mutex);
 966
 967         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 968 }
 969
 970 void bnx2x_release_phy_lock(struct bnx2x *bp)
 971 {
 972         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 973
 974         mutex_unlock(&bp->port.phy_mutex);
 975 }
 976
 977 /* calculates MF speed according to current linespeed and MF configuration */
 978 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 979 {
 980         u16 line_speed = bp->link_vars.line_speed;
 981         if (IS_MF(bp)) {
 982                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 983                                                    bp->mf_config[BP_VN(bp)]);
 984
 985                 /* Calculate the current MAX line speed limit for the MF
 986                  * devices
 987                  */
 988                 if (IS_MF_SI(bp))
 989                         line_speed = (line_speed * maxCfg) / 100;
 990                 else { /* SD mode */
 991                         u16 vn_max_rate = maxCfg * 100;
 992
 993                         if (vn_max_rate < line_speed)
 994                                 line_speed = vn_max_rate;
 995                 }
 996         }
 997
 998         return line_speed;
 999 }
1000
1001 /**
1002  * bnx2x_fill_report_data - fill link report data to report
1003  *
1004  * @bp:         driver handle
1005  * @data:       link state to update
1006  *
1007  * It uses a none-atomic bit operations because is called under the mutex.
1008  */
1009 static void bnx2x_fill_report_data(struct bnx2x *bp,
1010                                    struct bnx2x_link_report_data *data)
1011 {
1012         u16 line_speed = bnx2x_get_mf_speed(bp);
1013
1014         memset(data, 0, sizeof(*data));
1015
1016         /* Fill the report data: efective line speed */
1017         data->line_speed = line_speed;
1018
1019         /* Link is down */
1020         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1021                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1022                           &data->link_report_flags);
1023
1024         /* Full DUPLEX */
1025         if (bp->link_vars.duplex == DUPLEX_FULL)
1026                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1027
1028         /* Rx Flow Control is ON */
1029         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1030                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1031
1032         /* Tx Flow Control is ON */
1033         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1034                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1035 }
1036
1037 /**
1038  * bnx2x_link_report - report link status to OS.
1039  *
1040  * @bp:         driver handle
1041  *
1042  * Calls the __bnx2x_link_report() under the same locking scheme
1043  * as a link/PHY state managing code to ensure a consistent link
1044  * reporting.
1045  */
1046
1047 void bnx2x_link_report(struct bnx2x *bp)
1048 {
1049         bnx2x_acquire_phy_lock(bp);
1050         __bnx2x_link_report(bp);
1051         bnx2x_release_phy_lock(bp);
1052 }
1053
1054 /**
1055  * __bnx2x_link_report - report link status to OS.
1056  *
1057  * @bp:         driver handle
1058  *
1059  * None atomic inmlementation.
1060  * Should be called under the phy_lock.
1061  */
1062 void __bnx2x_link_report(struct bnx2x *bp)
1063 {
1064         struct bnx2x_link_report_data cur_data;
1065
1066         /* reread mf_cfg */
1067         if (!CHIP_IS_E1(bp))
1068                 bnx2x_read_mf_cfg(bp);
1069
1070         /* Read the current link report info */
1071         bnx2x_fill_report_data(bp, &cur_data);
1072
1073         /* Don't report link down or exactly the same link status twice */
1074         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1075             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1076                       &bp->last_reported_link.link_report_flags) &&
1077              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1078                       &cur_data.link_report_flags)))
1079                 return;
1080
1081         bp->link_cnt++;
1082
1083         /* We are going to report a new link parameters now -
1084          * remember the current data for the next time.
1085          */
1086         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1087
1088         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1089                      &cur_data.link_report_flags)) {
1090                 netif_carrier_off(bp->dev);
1091                 netdev_err(bp->dev, "NIC Link is Down\n");
1092                 return;
1093         } else {
1094                 const char *duplex;
1095                 const char *flow;
1096
1097                 netif_carrier_on(bp->dev);
1098
1099                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1100                                        &cur_data.link_report_flags))
1101                         duplex = "full";
1102                 else
1103                         duplex = "half";
1104
1105                 /* Handle the FC at the end so that only these flags would be
1106                  * possibly set. This way we may easily check if there is no FC
1107                  * enabled.
1108                  */
1109                 if (cur_data.link_report_flags) {
1110                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1111                                      &cur_data.link_report_flags)) {
1112                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1113                                      &cur_data.link_report_flags))
1114                                         flow = "ON - receive & transmit";
1115                                 else
1116                                         flow = "ON - receive";
1117                         } else {
1118                                 flow = "ON - transmit";
1119                         }
1120                 } else {
1121                         flow = "none";
1122                 }
1123                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1124                             cur_data.line_speed, duplex, flow);
1125         }
1126 }
1127
1128 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1129 {
1130         int i;
1131
1132         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1133                 struct eth_rx_sge *sge;
1134
1135                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1136                 sge->addr_hi =
1137                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1138                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1139
1140                 sge->addr_lo =
1141                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1142                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1143         }
1144 }
1145
1146 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1147                                 struct bnx2x_fastpath *fp, int last)
1148 {
1149         int i;
1150
1151         for (i = 0; i < last; i++) {
1152                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1153                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1154                 u8 *data = first_buf->data;
1155
1156                 if (data == NULL) {
1157                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1158                         continue;
1159                 }
1160                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1161                         dma_unmap_single(&bp->pdev->dev,
1162                                          dma_unmap_addr(first_buf, mapping),
1163                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1164                 bnx2x_frag_free(fp, data);
1165                 first_buf->data = NULL;
1166         }
1167 }
1168
1169 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1170 {
1171         int j;
1172
1173         for_each_rx_queue_cnic(bp, j) {
1174                 struct bnx2x_fastpath *fp = &bp->fp[j];
1175
1176                 fp->rx_bd_cons = 0;
1177
1178                 /* Activate BD ring */
1179                 /* Warning!
1180                  * this will generate an interrupt (to the TSTORM)
1181                  * must only be done after chip is initialized
1182                  */
1183                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1184                                      fp->rx_sge_prod);
1185         }
1186 }
1187
1188 void bnx2x_init_rx_rings(struct bnx2x *bp)
1189 {
1190         int func = BP_FUNC(bp);
1191         u16 ring_prod;
1192         int i, j;
1193
1194         /* Allocate TPA resources */
1195         for_each_eth_queue(bp, j) {
1196                 struct bnx2x_fastpath *fp = &bp->fp[j];
1197
1198                 DP(NETIF_MSG_IFUP,
1199                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1200
1201                 if (!fp->disable_tpa) {
1202                         /* Fill the per-aggregtion pool */
1203                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1204                                 struct bnx2x_agg_info *tpa_info =
1205                                         &fp->tpa_info[i];
1206                                 struct sw_rx_bd *first_buf =
1207                                         &tpa_info->first_buf;
1208
1209                                 first_buf->data = bnx2x_frag_alloc(fp);
1210                                 if (!first_buf->data) {
1211                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1212                                                   j);
1213                                         bnx2x_free_tpa_pool(bp, fp, i);
1214                                         fp->disable_tpa = 1;
1215                                         break;
1216                                 }
1217                                 dma_unmap_addr_set(first_buf, mapping, 0);
1218                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1219                         }
1220
1221                         /* "next page" elements initialization */
1222                         bnx2x_set_next_page_sgl(fp);
1223
1224                         /* set SGEs bit mask */
1225                         bnx2x_init_sge_ring_bit_mask(fp);
1226
1227                         /* Allocate SGEs and initialize the ring elements */
1228                         for (i = 0, ring_prod = 0;
1229                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1230
1231                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1232                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1233                                                   i);
1234                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1235                                                   j);
1236                                         /* Cleanup already allocated elements */
1237                                         bnx2x_free_rx_sge_range(bp, fp,
1238                                                                 ring_prod);
1239                                         bnx2x_free_tpa_pool(bp, fp,
1240                                                             MAX_AGG_QS(bp));
1241                                         fp->disable_tpa = 1;
1242                                         ring_prod = 0;
1243                                         break;
1244                                 }
1245                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1246                         }
1247
1248                         fp->rx_sge_prod = ring_prod;
1249                 }
1250         }
1251
1252         for_each_eth_queue(bp, j) {
1253                 struct bnx2x_fastpath *fp = &bp->fp[j];
1254
1255                 fp->rx_bd_cons = 0;
1256
1257                 /* Activate BD ring */
1258                 /* Warning!
1259                  * this will generate an interrupt (to the TSTORM)
1260                  * must only be done after chip is initialized
1261                  */
1262                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1263                                      fp->rx_sge_prod);
1264
1265                 if (j != 0)
1266                         continue;
1267
1268                 if (CHIP_IS_E1(bp)) {
1269                         REG_WR(bp, BAR_USTRORM_INTMEM +
1270                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1271                                U64_LO(fp->rx_comp_mapping));
1272                         REG_WR(bp, BAR_USTRORM_INTMEM +
1273                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1274                                U64_HI(fp->rx_comp_mapping));
1275                 }
1276         }
1277 }
1278
1279 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1280 {
1281         u8 cos;
1282         struct bnx2x *bp = fp->bp;
1283
1284         for_each_cos_in_tx_queue(fp, cos) {
1285                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1286                 unsigned pkts_compl = 0, bytes_compl = 0;
1287
1288                 u16 sw_prod = txdata->tx_pkt_prod;
1289                 u16 sw_cons = txdata->tx_pkt_cons;
1290
1291                 while (sw_cons != sw_prod) {
1292                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1293                                           &pkts_compl, &bytes_compl);
1294                         sw_cons++;
1295                 }
1296
1297                 netdev_tx_reset_queue(
1298                         netdev_get_tx_queue(bp->dev,
1299                                             txdata->txq_index));
1300         }
1301 }
1302
1303 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1304 {
1305         int i;
1306
1307         for_each_tx_queue_cnic(bp, i) {
1308                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1309         }
1310 }
1311
1312 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1313 {
1314         int i;
1315
1316         for_each_eth_queue(bp, i) {
1317                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1318         }
1319 }
1320
1321 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1322 {
1323         struct bnx2x *bp = fp->bp;
1324         int i;
1325
1326         /* ring wasn't allocated */
1327         if (fp->rx_buf_ring == NULL)
1328                 return;
1329
1330         for (i = 0; i < NUM_RX_BD; i++) {
1331                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1332                 u8 *data = rx_buf->data;
1333
1334                 if (data == NULL)
1335                         continue;
1336                 dma_unmap_single(&bp->pdev->dev,
1337                                  dma_unmap_addr(rx_buf, mapping),
1338                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1339
1340                 rx_buf->data = NULL;
1341                 bnx2x_frag_free(fp, data);
1342         }
1343 }
1344
1345 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1346 {
1347         int j;
1348
1349         for_each_rx_queue_cnic(bp, j) {
1350                 bnx2x_free_rx_bds(&bp->fp[j]);
1351         }
1352 }
1353
1354 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1355 {
1356         int j;
1357
1358         for_each_eth_queue(bp, j) {
1359                 struct bnx2x_fastpath *fp = &bp->fp[j];
1360
1361                 bnx2x_free_rx_bds(fp);
1362
1363                 if (!fp->disable_tpa)
1364                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1365         }
1366 }
1367
1368 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1369 {
1370         bnx2x_free_tx_skbs_cnic(bp);
1371         bnx2x_free_rx_skbs_cnic(bp);
1372 }
1373
1374 void bnx2x_free_skbs(struct bnx2x *bp)
1375 {
1376         bnx2x_free_tx_skbs(bp);
1377         bnx2x_free_rx_skbs(bp);
1378 }
1379
1380 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1381 {
1382         /* load old values */
1383         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1384
1385         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1386                 /* leave all but MAX value */
1387                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1388
1389                 /* set new MAX value */
1390                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1391                                 & FUNC_MF_CFG_MAX_BW_MASK;
1392
1393                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1394         }
1395 }
1396
1397 /**
1398  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1399  *
1400  * @bp:         driver handle
1401  * @nvecs:      number of vectors to be released
1402  */
1403 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1404 {
1405         int i, offset = 0;
1406
1407         if (nvecs == offset)
1408                 return;
1409         free_irq(bp->msix_table[offset].vector, bp->dev);
1410         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1411            bp->msix_table[offset].vector);
1412         offset++;
1413
1414         if (CNIC_SUPPORT(bp)) {
1415                 if (nvecs == offset)
1416                         return;
1417                 offset++;
1418         }
1419
1420         for_each_eth_queue(bp, i) {
1421                 if (nvecs == offset)
1422                         return;
1423                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1424                    i, bp->msix_table[offset].vector);
1425
1426                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1427         }
1428 }
1429
1430 void bnx2x_free_irq(struct bnx2x *bp)
1431 {
1432         if (bp->flags & USING_MSIX_FLAG &&
1433             !(bp->flags & USING_SINGLE_MSIX_FLAG))
1434                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1435                                      CNIC_SUPPORT(bp) + 1);
1436         else
1437                 free_irq(bp->dev->irq, bp->dev);
1438 }
1439
1440 int bnx2x_enable_msix(struct bnx2x *bp)
1441 {
1442         int msix_vec = 0, i, rc, req_cnt;
1443
1444         bp->msix_table[msix_vec].entry = msix_vec;
1445         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1446            bp->msix_table[0].entry);
1447         msix_vec++;
1448
1449         /* Cnic requires an msix vector for itself */
1450         if (CNIC_SUPPORT(bp)) {
1451                 bp->msix_table[msix_vec].entry = msix_vec;
1452                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1453                                msix_vec, bp->msix_table[msix_vec].entry);
1454                 msix_vec++;
1455         }
1456
1457         /* We need separate vectors for ETH queues only (not FCoE) */
1458         for_each_eth_queue(bp, i) {
1459                 bp->msix_table[msix_vec].entry = msix_vec;
1460                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1461                                msix_vec, msix_vec, i);
1462                 msix_vec++;
1463         }
1464
1465         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp) + 1;
1466
1467         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1468
1469         /*
1470          * reconfigure number of tx/rx queues according to available
1471          * MSI-X vectors
1472          */
1473         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1474                 /* how less vectors we will have? */
1475                 int diff = req_cnt - rc;
1476
1477                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1478
1479                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1480
1481                 if (rc) {
1482                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1483                         goto no_msix;
1484                 }
1485                 /*
1486                  * decrease number of queues by number of unallocated entries
1487                  */
1488                 bp->num_ethernet_queues -= diff;
1489                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1490
1491                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1492                                bp->num_queues);
1493         } else if (rc > 0) {
1494                 /* Get by with single vector */
1495                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1496                 if (rc) {
1497                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1498                                        rc);
1499                         goto no_msix;
1500                 }
1501
1502                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1503                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1504
1505                 BNX2X_DEV_INFO("set number of queues to 1\n");
1506                 bp->num_ethernet_queues = 1;
1507                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1508         } else if (rc < 0) {
1509                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1510                 goto no_msix;
1511         }
1512
1513         bp->flags |= USING_MSIX_FLAG;
1514
1515         return 0;
1516
1517 no_msix:
1518         /* fall to INTx if not enough memory */
1519         if (rc == -ENOMEM)
1520                 bp->flags |= DISABLE_MSI_FLAG;
1521
1522         return rc;
1523 }
1524
1525 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1526 {
1527         int i, rc, offset = 0;
1528
1529         rc = request_irq(bp->msix_table[offset++].vector,
1530                          bnx2x_msix_sp_int, 0,
1531                          bp->dev->name, bp->dev);
1532         if (rc) {
1533                 BNX2X_ERR("request sp irq failed\n");
1534                 return -EBUSY;
1535         }
1536
1537         if (CNIC_SUPPORT(bp))
1538                 offset++;
1539
1540         for_each_eth_queue(bp, i) {
1541                 struct bnx2x_fastpath *fp = &bp->fp[i];
1542                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1543                          bp->dev->name, i);
1544
1545                 rc = request_irq(bp->msix_table[offset].vector,
1546                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1547                 if (rc) {
1548                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1549                               bp->msix_table[offset].vector, rc);
1550                         bnx2x_free_msix_irqs(bp, offset);
1551                         return -EBUSY;
1552                 }
1553
1554                 offset++;
1555         }
1556
1557         i = BNX2X_NUM_ETH_QUEUES(bp);
1558         offset = 1 + CNIC_SUPPORT(bp);
1559         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1560                bp->msix_table[0].vector,
1561                0, bp->msix_table[offset].vector,
1562                i - 1, bp->msix_table[offset + i - 1].vector);
1563
1564         return 0;
1565 }
1566
1567 int bnx2x_enable_msi(struct bnx2x *bp)
1568 {
1569         int rc;
1570
1571         rc = pci_enable_msi(bp->pdev);
1572         if (rc) {
1573                 BNX2X_DEV_INFO("MSI is not attainable\n");
1574                 return -1;
1575         }
1576         bp->flags |= USING_MSI_FLAG;
1577
1578         return 0;
1579 }
1580
1581 static int bnx2x_req_irq(struct bnx2x *bp)
1582 {
1583         unsigned long flags;
1584         unsigned int irq;
1585
1586         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1587                 flags = 0;
1588         else
1589                 flags = IRQF_SHARED;
1590
1591         if (bp->flags & USING_MSIX_FLAG)
1592                 irq = bp->msix_table[0].vector;
1593         else
1594                 irq = bp->pdev->irq;
1595
1596         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1597 }
1598
1599 static int bnx2x_setup_irqs(struct bnx2x *bp)
1600 {
1601         int rc = 0;
1602         if (bp->flags & USING_MSIX_FLAG &&
1603             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1604                 rc = bnx2x_req_msix_irqs(bp);
1605                 if (rc)
1606                         return rc;
1607         } else {
1608                 bnx2x_ack_int(bp);
1609                 rc = bnx2x_req_irq(bp);
1610                 if (rc) {
1611                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1612                         return rc;
1613                 }
1614                 if (bp->flags & USING_MSI_FLAG) {
1615                         bp->dev->irq = bp->pdev->irq;
1616                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1617                                     bp->dev->irq);
1618                 }
1619                 if (bp->flags & USING_MSIX_FLAG) {
1620                         bp->dev->irq = bp->msix_table[0].vector;
1621                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1622                                     bp->dev->irq);
1623                 }
1624         }
1625
1626         return 0;
1627 }
1628
1629 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1630 {
1631         int i;
1632
1633         for_each_rx_queue_cnic(bp, i)
1634                 napi_enable(&bnx2x_fp(bp, i, napi));
1635 }
1636
1637 static void bnx2x_napi_enable(struct bnx2x *bp)
1638 {
1639         int i;
1640
1641         for_each_eth_queue(bp, i)
1642                 napi_enable(&bnx2x_fp(bp, i, napi));
1643 }
1644
1645 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1646 {
1647         int i;
1648
1649         for_each_rx_queue_cnic(bp, i)
1650                 napi_disable(&bnx2x_fp(bp, i, napi));
1651 }
1652
1653 static void bnx2x_napi_disable(struct bnx2x *bp)
1654 {
1655         int i;
1656
1657         for_each_eth_queue(bp, i)
1658                 napi_disable(&bnx2x_fp(bp, i, napi));
1659 }
1660
1661 void bnx2x_netif_start(struct bnx2x *bp)
1662 {
1663         if (netif_running(bp->dev)) {
1664                 bnx2x_napi_enable(bp);
1665                 if (CNIC_LOADED(bp))
1666                         bnx2x_napi_enable_cnic(bp);
1667                 bnx2x_int_enable(bp);
1668                 if (bp->state == BNX2X_STATE_OPEN)
1669                         netif_tx_wake_all_queues(bp->dev);
1670         }
1671 }
1672
1673 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1674 {
1675         bnx2x_int_disable_sync(bp, disable_hw);
1676         bnx2x_napi_disable(bp);
1677         if (CNIC_LOADED(bp))
1678                 bnx2x_napi_disable_cnic(bp);
1679 }
1680
1681 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1682 {
1683         struct bnx2x *bp = netdev_priv(dev);
1684
1685         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1686                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1687                 u16 ether_type = ntohs(hdr->h_proto);
1688
1689                 /* Skip VLAN tag if present */
1690                 if (ether_type == ETH_P_8021Q) {
1691                         struct vlan_ethhdr *vhdr =
1692                                 (struct vlan_ethhdr *)skb->data;
1693
1694                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1695                 }
1696
1697                 /* If ethertype is FCoE or FIP - use FCoE ring */
1698                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1699                         return bnx2x_fcoe_tx(bp, txq_index);
1700         }
1701
1702         /* select a non-FCoE queue */
1703         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1704 }
1705
1706
1707 void bnx2x_set_num_queues(struct bnx2x *bp)
1708 {
1709         /* RSS queues */
1710         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1711
1712         /* override in STORAGE SD modes */
1713         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1714                 bp->num_ethernet_queues = 1;
1715
1716         /* Add special queues */
1717         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1718         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1719
1720         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1721 }
1722
1723 /**
1724  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1725  *
1726  * @bp:         Driver handle
1727  *
1728  * We currently support for at most 16 Tx queues for each CoS thus we will
1729  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1730  * bp->max_cos.
1731  *
1732  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1733  * index after all ETH L2 indices.
1734  *
1735  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1736  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1737  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1738  *
1739  * The proper configuration of skb->queue_mapping is handled by
1740  * bnx2x_select_queue() and __skb_tx_hash().
1741  *
1742  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1743  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1744  */
1745 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1746 {
1747         int rc, tx, rx;
1748
1749         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1750         rx = BNX2X_NUM_ETH_QUEUES(bp);
1751
1752 /* account for fcoe queue */
1753         if (include_cnic && !NO_FCOE(bp)) {
1754                 rx++;
1755                 tx++;
1756         }
1757
1758         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1759         if (rc) {
1760                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1761                 return rc;
1762         }
1763         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1764         if (rc) {
1765                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1766                 return rc;
1767         }
1768
1769         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1770                           tx, rx);
1771
1772         return rc;
1773 }
1774
1775 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1776 {
1777         int i;
1778
1779         for_each_queue(bp, i) {
1780                 struct bnx2x_fastpath *fp = &bp->fp[i];
1781                 u32 mtu;
1782
1783                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1784                 if (IS_FCOE_IDX(i))
1785                         /*
1786                          * Although there are no IP frames expected to arrive to
1787                          * this ring we still want to add an
1788                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1789                          * overrun attack.
1790                          */
1791                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1792                 else
1793                         mtu = bp->dev->mtu;
1794                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1795                                   IP_HEADER_ALIGNMENT_PADDING +
1796                                   ETH_OVREHEAD +
1797                                   mtu +
1798                                   BNX2X_FW_RX_ALIGN_END;
1799                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1800                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1801                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1802                 else
1803                         fp->rx_frag_size = 0;
1804         }
1805 }
1806
1807 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1808 {
1809         int i;
1810         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1811
1812         /* Prepare the initial contents fo the indirection table if RSS is
1813          * enabled
1814          */
1815         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1816                 bp->rss_conf_obj.ind_table[i] =
1817                         bp->fp->cl_id +
1818                         ethtool_rxfh_indir_default(i, num_eth_queues);
1819
1820         /*
1821          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1822          * per-port, so if explicit configuration is needed , do it only
1823          * for a PMF.
1824          *
1825          * For 57712 and newer on the other hand it's a per-function
1826          * configuration.
1827          */
1828         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1829 }
1830
1831 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1832                         bool config_hash)
1833 {
1834         struct bnx2x_config_rss_params params = {NULL};
1835
1836         /* Although RSS is meaningless when there is a single HW queue we
1837          * still need it enabled in order to have HW Rx hash generated.
1838          *
1839          * if (!is_eth_multi(bp))
1840          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1841          */
1842
1843         params.rss_obj = rss_obj;
1844
1845         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1846
1847         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1848
1849         /* RSS configuration */
1850         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1851         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1852         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1853         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1854         if (rss_obj->udp_rss_v4)
1855                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1856         if (rss_obj->udp_rss_v6)
1857                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1858
1859         /* Hash bits */
1860         params.rss_result_mask = MULTI_MASK;
1861
1862         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1863
1864         if (config_hash) {
1865                 /* RSS keys */
1866                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
1867                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1868         }
1869
1870         return bnx2x_config_rss(bp, &params);
1871 }
1872
1873 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1874 {
1875         struct bnx2x_func_state_params func_params = {NULL};
1876
1877         /* Prepare parameters for function state transitions */
1878         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1879
1880         func_params.f_obj = &bp->func_obj;
1881         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1882
1883         func_params.params.hw_init.load_phase = load_code;
1884
1885         return bnx2x_func_state_change(bp, &func_params);
1886 }
1887
1888 /*
1889  * Cleans the object that have internal lists without sending
1890  * ramrods. Should be run when interrutps are disabled.
1891  */
1892 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1893 {
1894         int rc;
1895         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1896         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1897         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
1898
1899         /***************** Cleanup MACs' object first *************************/
1900
1901         /* Wait for completion of requested */
1902         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1903         /* Perform a dry cleanup */
1904         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1905
1906         /* Clean ETH primary MAC */
1907         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1908         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
1909                                  &ramrod_flags);
1910         if (rc != 0)
1911                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1912
1913         /* Cleanup UC list */
1914         vlan_mac_flags = 0;
1915         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1916         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1917                                  &ramrod_flags);
1918         if (rc != 0)
1919                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1920
1921         /***************** Now clean mcast object *****************************/
1922         rparam.mcast_obj = &bp->mcast_obj;
1923         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1924
1925         /* Add a DEL command... */
1926         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1927         if (rc < 0)
1928                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1929                           rc);
1930
1931         /* ...and wait until all pending commands are cleared */
1932         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1933         while (rc != 0) {
1934                 if (rc < 0) {
1935                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1936                                   rc);
1937                         return;
1938                 }
1939
1940                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1941         }
1942 }
1943
1944 #ifndef BNX2X_STOP_ON_ERROR
1945 #define LOAD_ERROR_EXIT(bp, label) \
1946         do { \
1947                 (bp)->state = BNX2X_STATE_ERROR; \
1948                 goto label; \
1949         } while (0)
1950
1951 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1952         do { \
1953                 bp->cnic_loaded = false; \
1954                 goto label; \
1955         } while (0)
1956 #else /*BNX2X_STOP_ON_ERROR*/
1957 #define LOAD_ERROR_EXIT(bp, label) \
1958         do { \
1959                 (bp)->state = BNX2X_STATE_ERROR; \
1960                 (bp)->panic = 1; \
1961                 return -EBUSY; \
1962         } while (0)
1963 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1964         do { \
1965                 bp->cnic_loaded = false; \
1966                 (bp)->panic = 1; \
1967                 return -EBUSY; \
1968         } while (0)
1969 #endif /*BNX2X_STOP_ON_ERROR*/
1970
1971 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1972 {
1973         /* build FW version dword */
1974         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1975                     (BCM_5710_FW_MINOR_VERSION << 8) +
1976                     (BCM_5710_FW_REVISION_VERSION << 16) +
1977                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
1978
1979         /* read loaded FW from chip */
1980         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
1981
1982         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
1983
1984         if (loaded_fw != my_fw) {
1985                 if (is_err)
1986                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
1987                                   loaded_fw, my_fw);
1988                 return false;
1989         }
1990
1991         return true;
1992 }
1993
1994 /**
1995  * bnx2x_bz_fp - zero content of the fastpath structure.
1996  *
1997  * @bp:         driver handle
1998  * @index:      fastpath index to be zeroed
1999  *
2000  * Makes sure the contents of the bp->fp[index].napi is kept
2001  * intact.
2002  */
2003 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2004 {
2005         struct bnx2x_fastpath *fp = &bp->fp[index];
2006         struct bnx2x_fp_stats *fp_stats = &bp->fp_stats[index];
2007
2008         int cos;
2009         struct napi_struct orig_napi = fp->napi;
2010         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2011         /* bzero bnx2x_fastpath contents */
2012         if (bp->stats_init) {
2013                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2014                 memset(fp, 0, sizeof(*fp));
2015         } else {
2016                 /* Keep Queue statistics */
2017                 struct bnx2x_eth_q_stats *tmp_eth_q_stats;
2018                 struct bnx2x_eth_q_stats_old *tmp_eth_q_stats_old;
2019
2020                 tmp_eth_q_stats = kzalloc(sizeof(struct bnx2x_eth_q_stats),
2021                                           GFP_KERNEL);
2022                 if (tmp_eth_q_stats)
2023                         memcpy(tmp_eth_q_stats, &fp_stats->eth_q_stats,
2024                                sizeof(struct bnx2x_eth_q_stats));
2025
2026                 tmp_eth_q_stats_old =
2027                         kzalloc(sizeof(struct bnx2x_eth_q_stats_old),
2028                                 GFP_KERNEL);
2029                 if (tmp_eth_q_stats_old)
2030                         memcpy(tmp_eth_q_stats_old, &fp_stats->eth_q_stats_old,
2031                                sizeof(struct bnx2x_eth_q_stats_old));
2032
2033                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2034                 memset(fp, 0, sizeof(*fp));
2035
2036                 if (tmp_eth_q_stats) {
2037                         memcpy(&fp_stats->eth_q_stats, tmp_eth_q_stats,
2038                                sizeof(struct bnx2x_eth_q_stats));
2039                         kfree(tmp_eth_q_stats);
2040                 }
2041
2042                 if (tmp_eth_q_stats_old) {
2043                         memcpy(&fp_stats->eth_q_stats_old, tmp_eth_q_stats_old,
2044                                sizeof(struct bnx2x_eth_q_stats_old));
2045                         kfree(tmp_eth_q_stats_old);
2046                 }
2047
2048         }
2049
2050         /* Restore the NAPI object as it has been already initialized */
2051         fp->napi = orig_napi;
2052         fp->tpa_info = orig_tpa_info;
2053         fp->bp = bp;
2054         fp->index = index;
2055         if (IS_ETH_FP(fp))
2056                 fp->max_cos = bp->max_cos;
2057         else
2058                 /* Special queues support only one CoS */
2059                 fp->max_cos = 1;
2060
2061         /* Init txdata pointers */
2062         if (IS_FCOE_FP(fp))
2063                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2064         if (IS_ETH_FP(fp))
2065                 for_each_cos_in_tx_queue(fp, cos)
2066                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2067                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2068
2069         /*
2070          * set the tpa flag for each queue. The tpa flag determines the queue
2071          * minimal size so it must be set prior to queue memory allocation
2072          */
2073         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2074                                   (bp->flags & GRO_ENABLE_FLAG &&
2075                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2076         if (bp->flags & TPA_ENABLE_FLAG)
2077                 fp->mode = TPA_MODE_LRO;
2078         else if (bp->flags & GRO_ENABLE_FLAG)
2079                 fp->mode = TPA_MODE_GRO;
2080
2081         /* We don't want TPA on an FCoE L2 ring */
2082         if (IS_FCOE_FP(fp))
2083                 fp->disable_tpa = 1;
2084 }
2085
2086 int bnx2x_load_cnic(struct bnx2x *bp)
2087 {
2088         int i, rc, port = BP_PORT(bp);
2089
2090         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2091
2092         mutex_init(&bp->cnic_mutex);
2093
2094         rc = bnx2x_alloc_mem_cnic(bp);
2095         if (rc) {
2096                 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2097                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2098         }
2099
2100         rc = bnx2x_alloc_fp_mem_cnic(bp);
2101         if (rc) {
2102                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2103                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2104         }
2105
2106         /* Update the number of queues with the cnic queues */
2107         rc = bnx2x_set_real_num_queues(bp, 1);
2108         if (rc) {
2109                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2110                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2111         }
2112
2113         /* Add all CNIC NAPI objects */
2114         bnx2x_add_all_napi_cnic(bp);
2115         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2116         bnx2x_napi_enable_cnic(bp);
2117
2118         rc = bnx2x_init_hw_func_cnic(bp);
2119         if (rc)
2120                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2121
2122         bnx2x_nic_init_cnic(bp);
2123
2124         /* Enable Timer scan */
2125         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2126
2127         for_each_cnic_queue(bp, i) {
2128                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2129                 if (rc) {
2130                         BNX2X_ERR("Queue setup failed\n");
2131                         LOAD_ERROR_EXIT(bp, load_error_cnic2);
2132                 }
2133         }
2134
2135         /* Initialize Rx filter. */
2136         netif_addr_lock_bh(bp->dev);
2137         bnx2x_set_rx_mode(bp->dev);
2138         netif_addr_unlock_bh(bp->dev);
2139
2140         /* re-read iscsi info */
2141         bnx2x_get_iscsi_info(bp);
2142         bnx2x_setup_cnic_irq_info(bp);
2143         bnx2x_setup_cnic_info(bp);
2144         bp->cnic_loaded = true;
2145         if (bp->state == BNX2X_STATE_OPEN)
2146                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2147
2148
2149         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2150
2151         return 0;
2152
2153 #ifndef BNX2X_STOP_ON_ERROR
2154 load_error_cnic2:
2155         /* Disable Timer scan */
2156         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2157
2158 load_error_cnic1:
2159         bnx2x_napi_disable_cnic(bp);
2160         /* Update the number of queues without the cnic queues */
2161         rc = bnx2x_set_real_num_queues(bp, 0);
2162         if (rc)
2163                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2164 load_error_cnic0:
2165         BNX2X_ERR("CNIC-related load failed\n");
2166         bnx2x_free_fp_mem_cnic(bp);
2167         bnx2x_free_mem_cnic(bp);
2168         return rc;
2169 #endif /* ! BNX2X_STOP_ON_ERROR */
2170 }
2171
2172
2173 /* must be called with rtnl_lock */
2174 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2175 {
2176         int port = BP_PORT(bp);
2177         u32 load_code;
2178         int i, rc;
2179
2180         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2181         DP(NETIF_MSG_IFUP,
2182            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2183
2184 #ifdef BNX2X_STOP_ON_ERROR
2185         if (unlikely(bp->panic)) {
2186                 BNX2X_ERR("Can't load NIC when there is panic\n");
2187                 return -EPERM;
2188         }
2189 #endif
2190
2191         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2192
2193         /* Set the initial link reported state to link down */
2194         bnx2x_acquire_phy_lock(bp);
2195         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2196         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2197                 &bp->last_reported_link.link_report_flags);
2198         bnx2x_release_phy_lock(bp);
2199
2200         /* must be called before memory allocation and HW init */
2201         bnx2x_ilt_set_info(bp);
2202
2203         /*
2204          * Zero fastpath structures preserving invariants like napi, which are
2205          * allocated only once, fp index, max_cos, bp pointer.
2206          * Also set fp->disable_tpa and txdata_ptr.
2207          */
2208         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2209         for_each_queue(bp, i)
2210                 bnx2x_bz_fp(bp, i);
2211         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2212                                   bp->num_cnic_queues) *
2213                                   sizeof(struct bnx2x_fp_txdata));
2214
2215         bp->fcoe_init = false;
2216
2217         /* Set the receive queues buffer size */
2218         bnx2x_set_rx_buf_size(bp);
2219
2220         if (bnx2x_alloc_mem(bp))
2221                 return -ENOMEM;
2222
2223         /* As long as bnx2x_alloc_mem() may possibly update
2224          * bp->num_queues, bnx2x_set_real_num_queues() should always
2225          * come after it. At this stage cnic queues are not counted.
2226          */
2227         rc = bnx2x_set_real_num_queues(bp, 0);
2228         if (rc) {
2229                 BNX2X_ERR("Unable to set real_num_queues\n");
2230                 LOAD_ERROR_EXIT(bp, load_error0);
2231         }
2232
2233         /* configure multi cos mappings in kernel.
2234          * this configuration may be overriden by a multi class queue discipline
2235          * or by a dcbx negotiation result.
2236          */
2237         bnx2x_setup_tc(bp->dev, bp->max_cos);
2238
2239         /* Add all NAPI objects */
2240         bnx2x_add_all_napi(bp);
2241         DP(NETIF_MSG_IFUP, "napi added\n");
2242         bnx2x_napi_enable(bp);
2243
2244         /* set pf load just before approaching the MCP */
2245         bnx2x_set_pf_load(bp);
2246
2247         /* Send LOAD_REQUEST command to MCP
2248          * Returns the type of LOAD command:
2249          * if it is the first port to be initialized
2250          * common blocks should be initialized, otherwise - not
2251          */
2252         if (!BP_NOMCP(bp)) {
2253                 /* init fw_seq */
2254                 bp->fw_seq =
2255                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2256                          DRV_MSG_SEQ_NUMBER_MASK);
2257                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2258
2259                 /* Get current FW pulse sequence */
2260                 bp->fw_drv_pulse_wr_seq =
2261                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2262                          DRV_PULSE_SEQ_MASK);
2263                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2264
2265                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2266                                              DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2267                 if (!load_code) {
2268                         BNX2X_ERR("MCP response failure, aborting\n");
2269                         rc = -EBUSY;
2270                         LOAD_ERROR_EXIT(bp, load_error1);
2271                 }
2272                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2273                         BNX2X_ERR("Driver load refused\n");
2274                         rc = -EBUSY; /* other port in diagnostic mode */
2275                         LOAD_ERROR_EXIT(bp, load_error1);
2276                 }
2277                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2278                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2279                         /* abort nic load if version mismatch */
2280                         if (!bnx2x_test_firmware_version(bp, true)) {
2281                                 rc = -EBUSY;
2282                                 LOAD_ERROR_EXIT(bp, load_error2);
2283                         }
2284                 }
2285
2286         } else {
2287                 int path = BP_PATH(bp);
2288
2289                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2290                    path, load_count[path][0], load_count[path][1],
2291                    load_count[path][2]);
2292                 load_count[path][0]++;
2293                 load_count[path][1 + port]++;
2294                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2295                    path, load_count[path][0], load_count[path][1],
2296                    load_count[path][2]);
2297                 if (load_count[path][0] == 1)
2298                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
2299                 else if (load_count[path][1 + port] == 1)
2300                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
2301                 else
2302                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
2303         }
2304
2305         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2306             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2307             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2308                 bp->port.pmf = 1;
2309                 /*
2310                  * We need the barrier to ensure the ordering between the
2311                  * writing to bp->port.pmf here and reading it from the
2312                  * bnx2x_periodic_task().
2313                  */
2314                 smp_mb();
2315         } else
2316                 bp->port.pmf = 0;
2317
2318         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
2319
2320         /* Init Function state controlling object */
2321         bnx2x__init_func_obj(bp);
2322
2323         /* Initialize HW */
2324         rc = bnx2x_init_hw(bp, load_code);
2325         if (rc) {
2326                 BNX2X_ERR("HW init failed, aborting\n");
2327                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2328                 LOAD_ERROR_EXIT(bp, load_error2);
2329         }
2330
2331         /* Connect to IRQs */
2332         rc = bnx2x_setup_irqs(bp);
2333         if (rc) {
2334                 BNX2X_ERR("IRQs setup failed\n");
2335                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2336                 LOAD_ERROR_EXIT(bp, load_error2);
2337         }
2338
2339         /* Setup NIC internals and enable interrupts */
2340         bnx2x_nic_init(bp, load_code);
2341
2342         /* Init per-function objects */
2343         bnx2x_init_bp_objs(bp);
2344
2345         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2346             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2347             (bp->common.shmem2_base)) {
2348                 if (SHMEM2_HAS(bp, dcc_support))
2349                         SHMEM2_WR(bp, dcc_support,
2350                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2351                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2352                 if (SHMEM2_HAS(bp, afex_driver_support))
2353                         SHMEM2_WR(bp, afex_driver_support,
2354                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2355         }
2356
2357         /* Set AFEX default VLAN tag to an invalid value */
2358         bp->afex_def_vlan_tag = -1;
2359
2360         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2361         rc = bnx2x_func_start(bp);
2362         if (rc) {
2363                 BNX2X_ERR("Function start failed!\n");
2364                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2365                 LOAD_ERROR_EXIT(bp, load_error3);
2366         }
2367
2368         /* Send LOAD_DONE command to MCP */
2369         if (!BP_NOMCP(bp)) {
2370                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2371                 if (!load_code) {
2372                         BNX2X_ERR("MCP response failure, aborting\n");
2373                         rc = -EBUSY;
2374                         LOAD_ERROR_EXIT(bp, load_error3);
2375                 }
2376         }
2377
2378         rc = bnx2x_setup_leading(bp);
2379         if (rc) {
2380                 BNX2X_ERR("Setup leading failed!\n");
2381                 LOAD_ERROR_EXIT(bp, load_error3);
2382         }
2383
2384         for_each_nondefault_eth_queue(bp, i) {
2385                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2386                 if (rc) {
2387                         BNX2X_ERR("Queue setup failed\n");
2388                         LOAD_ERROR_EXIT(bp, load_error3);
2389                 }
2390         }
2391
2392         rc = bnx2x_init_rss_pf(bp);
2393         if (rc) {
2394                 BNX2X_ERR("PF RSS init failed\n");
2395                 LOAD_ERROR_EXIT(bp, load_error3);
2396         }
2397
2398         /* Now when Clients are configured we are ready to work */
2399         bp->state = BNX2X_STATE_OPEN;
2400
2401         /* Configure a ucast MAC */
2402         rc = bnx2x_set_eth_mac(bp, true);
2403         if (rc) {
2404                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2405                 LOAD_ERROR_EXIT(bp, load_error3);
2406         }
2407
2408         if (bp->pending_max) {
2409                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2410                 bp->pending_max = 0;
2411         }
2412
2413         if (bp->port.pmf)
2414                 bnx2x_initial_phy_init(bp, load_mode);
2415         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2416
2417         /* Start fast path */
2418
2419         /* Initialize Rx filter. */
2420         netif_addr_lock_bh(bp->dev);
2421         bnx2x_set_rx_mode(bp->dev);
2422         netif_addr_unlock_bh(bp->dev);
2423
2424         /* Start the Tx */
2425         switch (load_mode) {
2426         case LOAD_NORMAL:
2427                 /* Tx queue should be only reenabled */
2428                 netif_tx_wake_all_queues(bp->dev);
2429                 break;
2430
2431         case LOAD_OPEN:
2432                 netif_tx_start_all_queues(bp->dev);
2433                 smp_mb__after_clear_bit();
2434                 break;
2435
2436         case LOAD_DIAG:
2437         case LOAD_LOOPBACK_EXT:
2438                 bp->state = BNX2X_STATE_DIAG;
2439                 break;
2440
2441         default:
2442                 break;
2443         }
2444
2445         if (bp->port.pmf)
2446                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2447         else
2448                 bnx2x__link_status_update(bp);
2449
2450         /* start the timer */
2451         mod_timer(&bp->timer, jiffies + bp->current_interval);
2452
2453         if (CNIC_ENABLED(bp))
2454                 bnx2x_load_cnic(bp);
2455
2456         /* mark driver is loaded in shmem2 */
2457         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2458                 u32 val;
2459                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2460                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2461                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2462                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2463         }
2464
2465         /* Wait for all pending SP commands to complete */
2466         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2467                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2468                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2469                 return -EBUSY;
2470         }
2471
2472         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2473         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2474                 bnx2x_dcbx_init(bp, false);
2475
2476         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2477
2478         return 0;
2479
2480 #ifndef BNX2X_STOP_ON_ERROR
2481 load_error3:
2482         bnx2x_int_disable_sync(bp, 1);
2483
2484         /* Clean queueable objects */
2485         bnx2x_squeeze_objects(bp);
2486
2487         /* Free SKBs, SGEs, TPA pool and driver internals */
2488         bnx2x_free_skbs(bp);
2489         for_each_rx_queue(bp, i)
2490                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2491
2492         /* Release IRQs */
2493         bnx2x_free_irq(bp);
2494 load_error2:
2495         if (!BP_NOMCP(bp)) {
2496                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2497                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2498         }
2499
2500         bp->port.pmf = 0;
2501 load_error1:
2502         bnx2x_napi_disable(bp);
2503         /* clear pf_load status, as it was already set */
2504         bnx2x_clear_pf_load(bp);
2505 load_error0:
2506         bnx2x_free_mem(bp);
2507
2508         return rc;
2509 #endif /* ! BNX2X_STOP_ON_ERROR */
2510 }
2511
2512 /* must be called with rtnl_lock */
2513 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2514 {
2515         int i;
2516         bool global = false;
2517
2518         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2519
2520         /* mark driver is unloaded in shmem2 */
2521         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2522                 u32 val;
2523                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2524                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2525                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2526         }
2527
2528         if ((bp->state == BNX2X_STATE_CLOSED) ||
2529             (bp->state == BNX2X_STATE_ERROR)) {
2530                 /* We can get here if the driver has been unloaded
2531                  * during parity error recovery and is either waiting for a
2532                  * leader to complete or for other functions to unload and
2533                  * then ifdown has been issued. In this case we want to
2534                  * unload and let other functions to complete a recovery
2535                  * process.
2536                  */
2537                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2538                 bp->is_leader = 0;
2539                 bnx2x_release_leader_lock(bp);
2540                 smp_mb();
2541
2542                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2543                 BNX2X_ERR("Can't unload in closed or error state\n");
2544                 return -EINVAL;
2545         }
2546
2547         /*
2548          * It's important to set the bp->state to the value different from
2549          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2550          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2551          */
2552         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2553         smp_mb();
2554
2555         if (CNIC_LOADED(bp))
2556                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2557
2558         /* Stop Tx */
2559         bnx2x_tx_disable(bp);
2560         netdev_reset_tc(bp->dev);
2561
2562         bp->rx_mode = BNX2X_RX_MODE_NONE;
2563
2564         del_timer_sync(&bp->timer);
2565
2566         /* Set ALWAYS_ALIVE bit in shmem */
2567         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2568
2569         bnx2x_drv_pulse(bp);
2570
2571         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2572         bnx2x_save_statistics(bp);
2573
2574         /* Cleanup the chip if needed */
2575         if (unload_mode != UNLOAD_RECOVERY)
2576                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2577         else {
2578                 /* Send the UNLOAD_REQUEST to the MCP */
2579                 bnx2x_send_unload_req(bp, unload_mode);
2580
2581                 /*
2582                  * Prevent transactions to host from the functions on the
2583                  * engine that doesn't reset global blocks in case of global
2584                  * attention once gloabl blocks are reset and gates are opened
2585                  * (the engine which leader will perform the recovery
2586                  * last).
2587                  */
2588                 if (!CHIP_IS_E1x(bp))
2589                         bnx2x_pf_disable(bp);
2590
2591                 /* Disable HW interrupts, NAPI */
2592                 bnx2x_netif_stop(bp, 1);
2593                 /* Delete all NAPI objects */
2594                 bnx2x_del_all_napi(bp);
2595                 if (CNIC_LOADED(bp))
2596                         bnx2x_del_all_napi_cnic(bp);
2597                 /* Release IRQs */
2598                 bnx2x_free_irq(bp);
2599
2600                 /* Report UNLOAD_DONE to MCP */
2601                 bnx2x_send_unload_done(bp, false);
2602         }
2603
2604         /*
2605          * At this stage no more interrupts will arrive so we may safly clean
2606          * the queueable objects here in case they failed to get cleaned so far.
2607          */
2608         bnx2x_squeeze_objects(bp);
2609
2610         /* There should be no more pending SP commands at this stage */
2611         bp->sp_state = 0;
2612
2613         bp->port.pmf = 0;
2614
2615         /* Free SKBs, SGEs, TPA pool and driver internals */
2616         bnx2x_free_skbs(bp);
2617         if (CNIC_LOADED(bp))
2618                 bnx2x_free_skbs_cnic(bp);
2619         for_each_rx_queue(bp, i)
2620                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2621
2622         if (CNIC_LOADED(bp)) {
2623                 bnx2x_free_fp_mem_cnic(bp);
2624                 bnx2x_free_mem_cnic(bp);
2625         }
2626         bnx2x_free_mem(bp);
2627
2628         bp->state = BNX2X_STATE_CLOSED;
2629         bp->cnic_loaded = false;
2630
2631         /* Check if there are pending parity attentions. If there are - set
2632          * RECOVERY_IN_PROGRESS.
2633          */
2634         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2635                 bnx2x_set_reset_in_progress(bp);
2636
2637                 /* Set RESET_IS_GLOBAL if needed */
2638                 if (global)
2639                         bnx2x_set_reset_global(bp);
2640         }
2641
2642
2643         /* The last driver must disable a "close the gate" if there is no
2644          * parity attention or "process kill" pending.
2645          */
2646         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2647                 bnx2x_disable_close_the_gate(bp);
2648
2649         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2650
2651         return 0;
2652 }
2653
2654 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2655 {
2656         u16 pmcsr;
2657
2658         /* If there is no power capability, silently succeed */
2659         if (!bp->pm_cap) {
2660                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2661                 return 0;
2662         }
2663
2664         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2665
2666         switch (state) {
2667         case PCI_D0:
2668                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2669                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2670                                        PCI_PM_CTRL_PME_STATUS));
2671
2672                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2673                         /* delay required during transition out of D3hot */
2674                         msleep(20);
2675                 break;
2676
2677         case PCI_D3hot:
2678                 /* If there are other clients above don't
2679                    shut down the power */
2680                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2681                         return 0;
2682                 /* Don't shut down the power for emulation and FPGA */
2683                 if (CHIP_REV_IS_SLOW(bp))
2684                         return 0;
2685
2686                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2687                 pmcsr |= 3;
2688
2689                 if (bp->wol)
2690                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2691
2692                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2693                                       pmcsr);
2694
2695                 /* No more memory access after this point until
2696                 * device is brought back to D0.
2697                 */
2698                 break;
2699
2700         default:
2701                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2702                 return -EINVAL;
2703         }
2704         return 0;
2705 }
2706
2707 /*
2708  * net_device service functions
2709  */
2710 int bnx2x_poll(struct napi_struct *napi, int budget)
2711 {
2712         int work_done = 0;
2713         u8 cos;
2714         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2715                                                  napi);
2716         struct bnx2x *bp = fp->bp;
2717
2718         while (1) {
2719 #ifdef BNX2X_STOP_ON_ERROR
2720                 if (unlikely(bp->panic)) {
2721                         napi_complete(napi);
2722                         return 0;
2723                 }
2724 #endif
2725
2726                 for_each_cos_in_tx_queue(fp, cos)
2727                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
2728                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
2729
2730
2731                 if (bnx2x_has_rx_work(fp)) {
2732                         work_done += bnx2x_rx_int(fp, budget - work_done);
2733
2734                         /* must not complete if we consumed full budget */
2735                         if (work_done >= budget)
2736                                 break;
2737                 }
2738
2739                 /* Fall out from the NAPI loop if needed */
2740                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2741
2742                         /* No need to update SB for FCoE L2 ring as long as
2743                          * it's connected to the default SB and the SB
2744                          * has been updated when NAPI was scheduled.
2745                          */
2746                         if (IS_FCOE_FP(fp)) {
2747                                 napi_complete(napi);
2748                                 break;
2749                         }
2750                         bnx2x_update_fpsb_idx(fp);
2751                         /* bnx2x_has_rx_work() reads the status block,
2752                          * thus we need to ensure that status block indices
2753                          * have been actually read (bnx2x_update_fpsb_idx)
2754                          * prior to this check (bnx2x_has_rx_work) so that
2755                          * we won't write the "newer" value of the status block
2756                          * to IGU (if there was a DMA right after
2757                          * bnx2x_has_rx_work and if there is no rmb, the memory
2758                          * reading (bnx2x_update_fpsb_idx) may be postponed
2759                          * to right before bnx2x_ack_sb). In this case there
2760                          * will never be another interrupt until there is
2761                          * another update of the status block, while there
2762                          * is still unhandled work.
2763                          */
2764                         rmb();
2765
2766                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2767                                 napi_complete(napi);
2768                                 /* Re-enable interrupts */
2769                                 DP(NETIF_MSG_RX_STATUS,
2770                                    "Update index to %d\n", fp->fp_hc_idx);
2771                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2772                                              le16_to_cpu(fp->fp_hc_idx),
2773                                              IGU_INT_ENABLE, 1);
2774                                 break;
2775                         }
2776                 }
2777         }
2778
2779         return work_done;
2780 }
2781
2782 /* we split the first BD into headers and data BDs
2783  * to ease the pain of our fellow microcode engineers
2784  * we use one mapping for both BDs
2785  */
2786 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2787                                    struct bnx2x_fp_txdata *txdata,
2788                                    struct sw_tx_bd *tx_buf,
2789                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2790                                    u16 bd_prod, int nbd)
2791 {
2792         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2793         struct eth_tx_bd *d_tx_bd;
2794         dma_addr_t mapping;
2795         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2796
2797         /* first fix first BD */
2798         h_tx_bd->nbd = cpu_to_le16(nbd);
2799         h_tx_bd->nbytes = cpu_to_le16(hlen);
2800
2801         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2802            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2803
2804         /* now get a new data BD
2805          * (after the pbd) and fill it */
2806         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2807         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2808
2809         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2810                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2811
2812         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2813         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2814         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2815
2816         /* this marks the BD as one that has no individual mapping */
2817         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2818
2819         DP(NETIF_MSG_TX_QUEUED,
2820            "TSO split data size is %d (%x:%x)\n",
2821            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2822
2823         /* update tx_bd */
2824         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2825
2826         return bd_prod;
2827 }
2828
2829 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2830 {
2831         if (fix > 0)
2832                 csum = (u16) ~csum_fold(csum_sub(csum,
2833                                 csum_partial(t_header - fix, fix, 0)));
2834
2835         else if (fix < 0)
2836                 csum = (u16) ~csum_fold(csum_add(csum,
2837                                 csum_partial(t_header, -fix, 0)));
2838
2839         return swab16(csum);
2840 }
2841
2842 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2843 {
2844         u32 rc;
2845
2846         if (skb->ip_summed != CHECKSUM_PARTIAL)
2847                 rc = XMIT_PLAIN;
2848
2849         else {
2850                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2851                         rc = XMIT_CSUM_V6;
2852                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2853                                 rc |= XMIT_CSUM_TCP;
2854
2855                 } else {
2856                         rc = XMIT_CSUM_V4;
2857                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2858                                 rc |= XMIT_CSUM_TCP;
2859                 }
2860         }
2861
2862         if (skb_is_gso_v6(skb))
2863                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2864         else if (skb_is_gso(skb))
2865                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2866
2867         return rc;
2868 }
2869
2870 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2871 /* check if packet requires linearization (packet is too fragmented)
2872    no need to check fragmentation if page size > 8K (there will be no
2873    violation to FW restrictions) */
2874 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2875                              u32 xmit_type)
2876 {
2877         int to_copy = 0;
2878         int hlen = 0;
2879         int first_bd_sz = 0;
2880
2881         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2882         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2883
2884                 if (xmit_type & XMIT_GSO) {
2885                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2886                         /* Check if LSO packet needs to be copied:
2887                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2888                         int wnd_size = MAX_FETCH_BD - 3;
2889                         /* Number of windows to check */
2890                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2891                         int wnd_idx = 0;
2892                         int frag_idx = 0;
2893                         u32 wnd_sum = 0;
2894
2895                         /* Headers length */
2896                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2897                                 tcp_hdrlen(skb);
2898
2899                         /* Amount of data (w/o headers) on linear part of SKB*/
2900                         first_bd_sz = skb_headlen(skb) - hlen;
2901
2902                         wnd_sum  = first_bd_sz;
2903
2904                         /* Calculate the first sum - it's special */
2905                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2906                                 wnd_sum +=
2907                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2908
2909                         /* If there was data on linear skb data - check it */
2910                         if (first_bd_sz > 0) {
2911                                 if (unlikely(wnd_sum < lso_mss)) {
2912                                         to_copy = 1;
2913                                         goto exit_lbl;
2914                                 }
2915
2916                                 wnd_sum -= first_bd_sz;
2917                         }
2918
2919                         /* Others are easier: run through the frag list and
2920                            check all windows */
2921                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2922                                 wnd_sum +=
2923                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2924
2925                                 if (unlikely(wnd_sum < lso_mss)) {
2926                                         to_copy = 1;
2927                                         break;
2928                                 }
2929                                 wnd_sum -=
2930                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2931                         }
2932                 } else {
2933                         /* in non-LSO too fragmented packet should always
2934                            be linearized */
2935                         to_copy = 1;
2936                 }
2937         }
2938
2939 exit_lbl:
2940         if (unlikely(to_copy))
2941                 DP(NETIF_MSG_TX_QUEUED,
2942                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2943                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2944                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2945
2946         return to_copy;
2947 }
2948 #endif
2949
2950 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2951                                         u32 xmit_type)
2952 {
2953         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2954                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2955                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2956         if ((xmit_type & XMIT_GSO_V6) &&
2957             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2958                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2959 }
2960
2961 /**
2962  * bnx2x_set_pbd_gso - update PBD in GSO case.
2963  *
2964  * @skb:        packet skb
2965  * @pbd:        parse BD
2966  * @xmit_type:  xmit flags
2967  */
2968 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2969                                      struct eth_tx_parse_bd_e1x *pbd,
2970                                      u32 xmit_type)
2971 {
2972         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2973         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2974         pbd->tcp_flags = pbd_tcp_flags(skb);
2975
2976         if (xmit_type & XMIT_GSO_V4) {
2977                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2978                 pbd->tcp_pseudo_csum =
2979                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2980                                                   ip_hdr(skb)->daddr,
2981                                                   0, IPPROTO_TCP, 0));
2982
2983         } else
2984                 pbd->tcp_pseudo_csum =
2985                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2986                                                 &ipv6_hdr(skb)->daddr,
2987                                                 0, IPPROTO_TCP, 0));
2988
2989         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2990 }
2991
2992 /**
2993  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2994  *
2995  * @bp:                 driver handle
2996  * @skb:                packet skb
2997  * @parsing_data:       data to be updated
2998  * @xmit_type:          xmit flags
2999  *
3000  * 57712 related
3001  */
3002 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3003         u32 *parsing_data, u32 xmit_type)
3004 {
3005         *parsing_data |=
3006                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3007                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
3008                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
3009
3010         if (xmit_type & XMIT_CSUM_TCP) {
3011                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3012                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3013                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3014
3015                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3016         } else
3017                 /* We support checksum offload for TCP and UDP only.
3018                  * No need to pass the UDP header length - it's a constant.
3019                  */
3020                 return skb_transport_header(skb) +
3021                                 sizeof(struct udphdr) - skb->data;
3022 }
3023
3024 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,