Merge branch 'kmemleak' of git://linux-arm.org/linux-2.6
[~shefty/rdma-dev.git] / drivers / infiniband / hw / nes / nes_hw.c
1 /*
2  * Copyright (c) 2006 - 2009 Intel-NE, Inc.  All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  *
32  */
33
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/ip.h>
39 #include <linux/tcp.h>
40 #include <linux/if_vlan.h>
41 #include <linux/inet_lro.h>
42
43 #include "nes.h"
44
45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
46 module_param(nes_lro_max_aggr, uint, 0444);
47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
48
49 static int wide_ppm_offset;
50 module_param(wide_ppm_offset, int, 0644);
51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
52
53 static u32 crit_err_count;
54 u32 int_mod_timer_init;
55 u32 int_mod_cq_depth_256;
56 u32 int_mod_cq_depth_128;
57 u32 int_mod_cq_depth_32;
58 u32 int_mod_cq_depth_24;
59 u32 int_mod_cq_depth_16;
60 u32 int_mod_cq_depth_4;
61 u32 int_mod_cq_depth_1;
62 static const u8 nes_max_critical_error_count = 100;
63 #include "nes_cm.h"
64
65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
73                                    struct nes_hw_aeqe *aeqe);
74 static void process_critical_error(struct nes_device *nesdev);
75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
77 static void nes_terminate_timeout(unsigned long context);
78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
79
80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
81 static unsigned char *nes_iwarp_state_str[] = {
82         "Non-Existant",
83         "Idle",
84         "RTS",
85         "Closing",
86         "RSVD1",
87         "Terminate",
88         "Error",
89         "RSVD2",
90 };
91
92 static unsigned char *nes_tcp_state_str[] = {
93         "Non-Existant",
94         "Closed",
95         "Listen",
96         "SYN Sent",
97         "SYN Rcvd",
98         "Established",
99         "Close Wait",
100         "FIN Wait 1",
101         "Closing",
102         "Last Ack",
103         "FIN Wait 2",
104         "Time Wait",
105         "RSVD1",
106         "RSVD2",
107         "RSVD3",
108         "RSVD4",
109 };
110 #endif
111
112
113 /**
114  * nes_nic_init_timer_defaults
115  */
116 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
117 {
118         unsigned long flags;
119         struct nes_adapter *nesadapter = nesdev->nesadapter;
120         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
121
122         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
123
124         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
125         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
126         if (jumbomode) {
127                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
128                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
129                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
130         } else {
131                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
132                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
133                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
134         }
135
136         /* todo use netdev->mtu to set thresholds */
137         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
138 }
139
140
141 /**
142  * nes_nic_init_timer
143  */
144 static void  nes_nic_init_timer(struct nes_device *nesdev)
145 {
146         unsigned long flags;
147         struct nes_adapter *nesadapter = nesdev->nesadapter;
148         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
149
150         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
151
152         if (shared_timer->timer_in_use_old == 0) {
153                 nesdev->deepcq_count = 0;
154                 shared_timer->timer_direction_upward = 0;
155                 shared_timer->timer_direction_downward = 0;
156                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
157                 shared_timer->timer_in_use_old = 0;
158
159         }
160         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
161                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
162                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
163                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
164         }
165         /* todo use netdev->mtu to set thresholds */
166         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
167 }
168
169
170 /**
171  * nes_nic_tune_timer
172  */
173 static void nes_nic_tune_timer(struct nes_device *nesdev)
174 {
175         unsigned long flags;
176         struct nes_adapter *nesadapter = nesdev->nesadapter;
177         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
178         u16 cq_count = nesdev->currcq_count;
179
180         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
181
182         if (shared_timer->cq_count_old <= cq_count)
183                 shared_timer->cq_direction_downward = 0;
184         else
185                 shared_timer->cq_direction_downward++;
186         shared_timer->cq_count_old = cq_count;
187         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
188                 if (cq_count <= shared_timer->threshold_low &&
189                     shared_timer->threshold_low > 4) {
190                         shared_timer->threshold_low = shared_timer->threshold_low/2;
191                         shared_timer->cq_direction_downward=0;
192                         nesdev->currcq_count = 0;
193                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
194                         return;
195                 }
196         }
197
198         if (cq_count > 1) {
199                 nesdev->deepcq_count += cq_count;
200                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
201                         shared_timer->timer_direction_upward++;
202                         shared_timer->timer_direction_downward = 0;
203                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
204                         shared_timer->timer_direction_upward = 0;
205                         shared_timer->timer_direction_downward = 0;
206                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
207                         shared_timer->timer_direction_downward++;
208                         shared_timer->timer_direction_upward = 0;
209                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
210                         shared_timer->timer_in_use -= 2;
211                         shared_timer->timer_direction_upward = 0;
212                         shared_timer->timer_direction_downward++;
213                 } else {
214                         shared_timer->timer_in_use -= 4;
215                         shared_timer->timer_direction_upward = 0;
216                         shared_timer->timer_direction_downward++;
217                 }
218
219                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
220                         shared_timer->timer_in_use += 3;
221                         shared_timer->timer_direction_upward = 0;
222                         shared_timer->timer_direction_downward = 0;
223                 }
224                 if (shared_timer->timer_direction_downward > 5) { /* using history */
225                         shared_timer->timer_in_use -= 4 ;
226                         shared_timer->timer_direction_downward = 0;
227                         shared_timer->timer_direction_upward = 0;
228                 }
229         }
230
231         /* boundary checking */
232         if (shared_timer->timer_in_use > shared_timer->threshold_high)
233                 shared_timer->timer_in_use = shared_timer->threshold_high;
234         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
235                 shared_timer->timer_in_use = shared_timer->threshold_low;
236
237         nesdev->currcq_count = 0;
238
239         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
240 }
241
242
243 /**
244  * nes_init_adapter - initialize adapter
245  */
246 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
247         struct nes_adapter *nesadapter = NULL;
248         unsigned long num_pds;
249         u32 u32temp;
250         u32 port_count;
251         u16 max_rq_wrs;
252         u16 max_sq_wrs;
253         u32 max_mr;
254         u32 max_256pbl;
255         u32 max_4kpbl;
256         u32 max_qp;
257         u32 max_irrq;
258         u32 max_cq;
259         u32 hte_index_mask;
260         u32 adapter_size;
261         u32 arp_table_size;
262         u16 vendor_id;
263         u16 device_id;
264         u8  OneG_Mode;
265         u8  func_index;
266
267         /* search the list of existing adapters */
268         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
269                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
270                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
271                                 nesdev->pcidev->devfn,
272                                 PCI_SLOT(nesadapter->devfn),
273                                 nesadapter->bus_number,
274                                 PCI_SLOT(nesdev->pcidev->devfn),
275                                 nesdev->pcidev->bus->number );
276                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
277                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
278                         nesadapter->ref_count++;
279                         return nesadapter;
280                 }
281         }
282
283         /* no adapter found */
284         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
285         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
286                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
287                                 hw_rev);
288                 return NULL;
289         }
290
291         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
292                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
296
297         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
298
299
300         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
301                 return NULL;
302
303         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
304         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
305
306         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
307         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
308                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
309                                 max_qp, u32temp);
310                 max_qp = (u32)1 << (u32temp & 0x001f);
311         }
312
313         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
314         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
315                         max_qp, hte_index_mask);
316
317         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
318
319         max_irrq = 1 << (u32temp & 0x001f);
320
321         if (max_qp > max_irrq) {
322                 max_qp = max_irrq;
323                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
324                                 max_qp);
325         }
326
327         /* there should be no reason to allocate more pds than qps */
328         if (num_pds > max_qp)
329                 num_pds = max_qp;
330
331         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
332         max_mr = (u32)8192 << (u32temp & 0x7);
333
334         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
335         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
336         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
337         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
338
339         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
340         arp_table_size = 1 << u32temp;
341
342         adapter_size = (sizeof(struct nes_adapter) +
343                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
349         adapter_size += sizeof(struct nes_qp **) * max_qp;
350
351         /* allocate a new adapter struct */
352         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
353         if (nesadapter == NULL) {
354                 return NULL;
355         }
356
357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
359
360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
362                 kfree(nesadapter);
363                 return NULL;
364         }
365
366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
367                                 (nesadapter->mac_addr_low >> 24);
368
369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
370                                  PCI_DEVICE_ID, &device_id);
371         nesadapter->vendor_part_id = device_id;
372
373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
374                                                         OneG_Mode)) {
375                 kfree(nesadapter);
376                 return NULL;
377         }
378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
379
380         memset(nesadapter->pft_mcast_map, 255,
381                sizeof nesadapter->pft_mcast_map);
382
383         /* populate the new nesadapter */
384         nesadapter->devfn = nesdev->pcidev->devfn;
385         nesadapter->bus_number = nesdev->pcidev->bus->number;
386         nesadapter->ref_count = 1;
387         nesadapter->timer_int_req = 0xffff0000;
388         nesadapter->OneG_Mode = OneG_Mode;
389         nesadapter->doorbell_start = nesdev->doorbell_region;
390
391         /* nesadapter->tick_delta = clk_divisor; */
392         nesadapter->hw_rev = hw_rev;
393         nesadapter->port_count = port_count;
394
395         nesadapter->max_qp = max_qp;
396         nesadapter->hte_index_mask = hte_index_mask;
397         nesadapter->max_irrq = max_irrq;
398         nesadapter->max_mr = max_mr;
399         nesadapter->max_256pbl = max_256pbl - 1;
400         nesadapter->max_4kpbl = max_4kpbl - 1;
401         nesadapter->max_cq = max_cq;
402         nesadapter->free_256pbl = max_256pbl - 1;
403         nesadapter->free_4kpbl = max_4kpbl - 1;
404         nesadapter->max_pd = num_pds;
405         nesadapter->arp_table_size = arp_table_size;
406
407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
412         } else {
413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
417         }
418         /* Setup and enable the periodic timer */
419         if (nesadapter->et_rx_coalesce_usecs_irq)
420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
422         else
423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
424
425         nesadapter->base_pd = 1;
426
427         nesadapter->device_cap_flags =
428                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
429
430         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
431                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
432         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
433         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
434         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
435         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
436         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
437
438
439         /* mark the usual suspect QPs and CQs as in use */
440         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
441                 set_bit(u32temp, nesadapter->allocated_qps);
442                 set_bit(u32temp, nesadapter->allocated_cqs);
443         }
444
445         for (u32temp = 0; u32temp < 20; u32temp++)
446                 set_bit(u32temp, nesadapter->allocated_pds);
447         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
448
449         max_rq_wrs = ((u32temp >> 8) & 3);
450         switch (max_rq_wrs) {
451                 case 0:
452                         max_rq_wrs = 4;
453                         break;
454                 case 1:
455                         max_rq_wrs = 16;
456                         break;
457                 case 2:
458                         max_rq_wrs = 32;
459                         break;
460                 case 3:
461                         max_rq_wrs = 512;
462                         break;
463         }
464
465         max_sq_wrs = (u32temp & 3);
466         switch (max_sq_wrs) {
467                 case 0:
468                         max_sq_wrs = 4;
469                         break;
470                 case 1:
471                         max_sq_wrs = 16;
472                         break;
473                 case 2:
474                         max_sq_wrs = 32;
475                         break;
476                 case 3:
477                         max_sq_wrs = 512;
478                         break;
479         }
480         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
481         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
482
483         nesadapter->max_sge = 4;
484         nesadapter->max_cqe = 32767;
485
486         if (nes_read_eeprom_values(nesdev, nesadapter)) {
487                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
488                 kfree(nesadapter);
489                 return NULL;
490         }
491
492         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
493         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
494                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
495
496         /* setup port configuration */
497         if (nesadapter->port_count == 1) {
498                 nesadapter->log_port = 0x00000000;
499                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
500                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
501                 else
502                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
503         } else {
504                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
505                         nesadapter->log_port = 0x000000D8;
506                 } else {
507                         if (nesadapter->port_count == 2)
508                                 nesadapter->log_port = 0x00000044;
509                         else
510                                 nesadapter->log_port = 0x000000e4;
511                 }
512                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
513         }
514
515         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
516                                                 nesadapter->log_port);
517         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
518                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
519
520         spin_lock_init(&nesadapter->resource_lock);
521         spin_lock_init(&nesadapter->phy_lock);
522         spin_lock_init(&nesadapter->pbl_lock);
523         spin_lock_init(&nesadapter->periodic_timer_lock);
524
525         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
526         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
529
530         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
531                 u32 pcs_control_status0, pcs_control_status1;
532                 u32 reset_value;
533                 u32 i = 0;
534                 u32 int_cnt = 0;
535                 u32 ext_cnt = 0;
536                 unsigned long flags;
537                 u32 j = 0;
538
539                 pcs_control_status0 = nes_read_indexed(nesdev,
540                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
541                 pcs_control_status1 = nes_read_indexed(nesdev,
542                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
543
544                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
545                         pcs_control_status0 = nes_read_indexed(nesdev,
546                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
547                         pcs_control_status1 = nes_read_indexed(nesdev,
548                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
549                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
550                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
551                                 int_cnt++;
552                         msleep(1);
553                 }
554                 if (int_cnt > 1) {
555                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
556                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
557                         mh_detected++;
558                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
559                         reset_value |= 0x0000003d;
560                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
561
562                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
563                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
564                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
565
566                         pcs_control_status0 = nes_read_indexed(nesdev,
567                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
568                         pcs_control_status1 = nes_read_indexed(nesdev,
569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
570
571                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
572                                 pcs_control_status0 = nes_read_indexed(nesdev,
573                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
574                                 pcs_control_status1 = nes_read_indexed(nesdev,
575                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
576                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
577                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
578                                         if (++ext_cnt > int_cnt) {
579                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
580                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
581                                                                 0x0000F088);
582                                                 mh_detected++;
583                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
584                                                 reset_value |= 0x0000003d;
585                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
586
587                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
588                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
589                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
590                                                 break;
591                                         }
592                                 }
593                                 msleep(1);
594                         }
595                 }
596         }
597
598         if (nesadapter->hw_rev == NE020_REV) {
599                 init_timer(&nesadapter->mh_timer);
600                 nesadapter->mh_timer.function = nes_mh_fix;
601                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
602                 nesadapter->mh_timer.data = (unsigned long)nesdev;
603                 add_timer(&nesadapter->mh_timer);
604         } else {
605                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
606         }
607
608         init_timer(&nesadapter->lc_timer);
609         nesadapter->lc_timer.function = nes_clc;
610         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
611         nesadapter->lc_timer.data = (unsigned long)nesdev;
612         add_timer(&nesadapter->lc_timer);
613
614         list_add_tail(&nesadapter->list, &nes_adapter_list);
615
616         for (func_index = 0; func_index < 8; func_index++) {
617                 pci_bus_read_config_word(nesdev->pcidev->bus,
618                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
619                                         func_index), 0, &vendor_id);
620                 if (vendor_id == 0xffff)
621                         break;
622         }
623         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
624                 func_index, pci_name(nesdev->pcidev));
625         nesadapter->adapter_fcn_count = func_index;
626
627         return nesadapter;
628 }
629
630
631 /**
632  * nes_reset_adapter_ne020
633  */
634 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
635 {
636         u32 port_count;
637         u32 u32temp;
638         u32 i;
639
640         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
641         port_count = ((u32temp & 0x00000300) >> 8) + 1;
642         /* TODO: assuming that both SERDES are set the same for now */
643         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
644         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
645                         u32temp, port_count);
646         if (*OneG_Mode)
647                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
648         u32temp &= 0xff00ffc0;
649         switch (port_count) {
650                 case 1:
651                         u32temp |= 0x00ee0000;
652                         break;
653                 case 2:
654                         u32temp |= 0x00cc0000;
655                         break;
656                 case 4:
657                         u32temp |= 0x00000000;
658                         break;
659                 default:
660                         return 0;
661                         break;
662         }
663
664         /* check and do full reset if needed */
665         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
666                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
667                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
668
669                 i = 0;
670                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
671                         mdelay(1);
672                 if (i > 10000) {
673                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
674                         return 0;
675                 }
676
677                 i = 0;
678                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
679                         mdelay(1);
680                 if (i > 10000) {
681                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
682                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
683                         return 0;
684                 }
685         }
686
687         /* port reset */
688         switch (port_count) {
689                 case 1:
690                         u32temp |= 0x00ee0010;
691                         break;
692                 case 2:
693                         u32temp |= 0x00cc0030;
694                         break;
695                 case 4:
696                         u32temp |= 0x00000030;
697                         break;
698         }
699
700         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
701         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
702
703         i = 0;
704         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
705                 mdelay(1);
706         if (i > 10000) {
707                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
708                 return 0;
709         }
710
711         /* serdes 0 */
712         i = 0;
713         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
714                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
715                 mdelay(1);
716         if (i > 5000) {
717                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
718                 return 0;
719         }
720
721         /* serdes 1 */
722         if (port_count > 1) {
723                 i = 0;
724                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
725                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
726                         mdelay(1);
727                 if (i > 5000) {
728                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
729                         return 0;
730                 }
731         }
732
733         return port_count;
734 }
735
736
737 /**
738  * nes_init_serdes
739  */
740 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
741                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
742 {
743         int i;
744         u32 u32temp;
745         u32 sds;
746
747         if (hw_rev != NE020_REV) {
748                 /* init serdes 0 */
749                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
750                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
751                 else
752                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
753
754                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
755                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
756                         sds |= 0x00000100;
757                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
758                 }
759                 if (!OneG_Mode)
760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
761
762                 if (port_count < 2)
763                         return 0;
764
765                 /* init serdes 1 */
766                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
767                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
768
769                 switch (nesadapter->phy_type[1]) {
770                 case NES_PHY_TYPE_ARGUS:
771                 case NES_PHY_TYPE_SFP_D:
772                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
773                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
774                         break;
775                 case NES_PHY_TYPE_CX4:
776                         if (wide_ppm_offset)
777                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
778                         break;
779                 case NES_PHY_TYPE_PUMA_1G:
780                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
781                         sds |= 0x000000100;
782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
783                 }
784                 if (!OneG_Mode) {
785                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
786                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
787                         sds &= 0xFFFFFFBF;
788                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
789                 }
790         } else {
791                 /* init serdes 0 */
792                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
793                 i = 0;
794                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
795                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
796                         mdelay(1);
797                 if (i > 5000) {
798                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
799                         return 1;
800                 }
801                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
802                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
805                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
806                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
807                 if (OneG_Mode)
808                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
809                 else
810                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
811
812                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
813                 if (port_count > 1) {
814                         /* init serdes 1 */
815                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
816                         i = 0;
817                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
818                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
819                                 mdelay(1);
820                         if (i > 5000) {
821                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
822                                 /* return 1; */
823                         }
824                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
825                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
830                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
831                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
832                 }
833         }
834         return 0;
835 }
836
837
838 /**
839  * nes_init_csr_ne020
840  * Initialize registers for ne020 hardware
841  */
842 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
843 {
844         u32 u32temp;
845
846         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
847
848         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
849         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
850         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
851         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
852         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
853         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
854         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
855         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
856
857         /* TODO: move these MAC register settings to NIC bringup */
858         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
859         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
860         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
861         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
862         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
863         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
864         if (port_count > 1) {
865                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
866                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
867                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
868                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
869                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
870                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
871                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
872         }
873         if (port_count > 2) {
874                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
875                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
876                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
877                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
878                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
879                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
880                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
881
882                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
883                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
884                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
885                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
886                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
887                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
888                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
889         }
890
891         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
892         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
893         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
894                                                          0x00000001);
895         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
896         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
897         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
898         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
899         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
900
901         /* TODO: move this to code, get from EEPROM */
902         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
903         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
904         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
905
906         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
907         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
908
909         if (hw_rev != NE020_REV) {
910                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
911                 u32temp |= 0x80000000;
912                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
913                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
914                 u32temp &= 0x7fffffff;
915                 u32temp |= 0x7fff0010;
916                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
917                 if (port_count > 1) {
918                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
919                         u32temp &= 0x7fffffff;
920                         u32temp |= 0x7fff0010;
921                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
922                 }
923         }
924 }
925
926
927 /**
928  * nes_destroy_adapter - destroy the adapter structure
929  */
930 void nes_destroy_adapter(struct nes_adapter *nesadapter)
931 {
932         struct nes_adapter *tmp_adapter;
933
934         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
935                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
936                                 tmp_adapter);
937         }
938
939         nesadapter->ref_count--;
940         if (!nesadapter->ref_count) {
941                 if (nesadapter->hw_rev == NE020_REV) {
942                         del_timer(&nesadapter->mh_timer);
943                 }
944                 del_timer(&nesadapter->lc_timer);
945
946                 list_del(&nesadapter->list);
947                 kfree(nesadapter);
948         }
949 }
950
951
952 /**
953  * nes_init_cqp
954  */
955 int nes_init_cqp(struct nes_device *nesdev)
956 {
957         struct nes_adapter *nesadapter = nesdev->nesadapter;
958         struct nes_hw_cqp_qp_context *cqp_qp_context;
959         struct nes_hw_cqp_wqe *cqp_wqe;
960         struct nes_hw_ceq *ceq;
961         struct nes_hw_ceq *nic_ceq;
962         struct nes_hw_aeq *aeq;
963         void *vmem;
964         dma_addr_t pmem;
965         u32 count=0;
966         u32 cqp_head;
967         u64 u64temp;
968         u32 u32temp;
969
970         /* allocate CQP memory */
971         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
972         /* SQ is 512 byte aligned, others are 256 byte aligned */
973         nesdev->cqp_mem_size = 512 +
974                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
975                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
976                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
977                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
978                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
979                         sizeof(struct nes_hw_cqp_qp_context);
980
981         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
982                         &nesdev->cqp_pbase);
983         if (!nesdev->cqp_vbase) {
984                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
985                 return -ENOMEM;
986         }
987         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
988
989         /* Allocate a twice the number of CQP requests as the SQ size */
990         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
991                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
992         if (nesdev->nes_cqp_requests == NULL) {
993                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
994                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
995                                 nesdev->cqp.sq_pbase);
996                 return -ENOMEM;
997         }
998
999         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1000                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1001
1002         spin_lock_init(&nesdev->cqp.lock);
1003         init_waitqueue_head(&nesdev->cqp.waitq);
1004
1005         /* Setup Various Structures */
1006         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1007                         ~(unsigned long)(512 - 1));
1008         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1009                         ~(unsigned long long)(512 - 1));
1010
1011         nesdev->cqp.sq_vbase = vmem;
1012         nesdev->cqp.sq_pbase = pmem;
1013         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1014         nesdev->cqp.sq_head = 0;
1015         nesdev->cqp.sq_tail = 0;
1016         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1017
1018         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1019         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1020
1021         nesdev->ccq.cq_vbase = vmem;
1022         nesdev->ccq.cq_pbase = pmem;
1023         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1024         nesdev->ccq.cq_head = 0;
1025         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1026         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1027
1028         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1029         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1030
1031         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1032         ceq = &nesadapter->ceq[nesdev->ceq_index];
1033         ceq->ceq_vbase = vmem;
1034         ceq->ceq_pbase = pmem;
1035         ceq->ceq_size = NES_CCEQ_SIZE;
1036         ceq->ceq_head = 0;
1037
1038         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1039         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1040
1041         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1042         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1043         nic_ceq->ceq_vbase = vmem;
1044         nic_ceq->ceq_pbase = pmem;
1045         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1046         nic_ceq->ceq_head = 0;
1047
1048         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1049         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1050
1051         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1052         aeq->aeq_vbase = vmem;
1053         aeq->aeq_pbase = pmem;
1054         aeq->aeq_size = nesadapter->max_qp;
1055         aeq->aeq_head = 0;
1056
1057         /* Setup QP Context */
1058         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1059         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1060
1061         cqp_qp_context = vmem;
1062         cqp_qp_context->context_words[0] =
1063                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1064         cqp_qp_context->context_words[1] = 0;
1065         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1066         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1067
1068
1069         /* Write the address to Create CQP */
1070         if ((sizeof(dma_addr_t) > 4)) {
1071                 nes_write_indexed(nesdev,
1072                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1073                                 ((u64)pmem) >> 32);
1074         } else {
1075                 nes_write_indexed(nesdev,
1076                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1077         }
1078         nes_write_indexed(nesdev,
1079                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1080                         (u32)pmem);
1081
1082         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1083         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1084
1085         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1086                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1087                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1088         }
1089
1090         /* Write Create CCQ WQE */
1091         cqp_head = nesdev->cqp.sq_head++;
1092         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1093         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1094         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1095                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1096                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1097         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1098                             (nesdev->ccq.cq_number |
1099                              ((u32)nesdev->ceq_index << 16)));
1100         u64temp = (u64)nesdev->ccq.cq_pbase;
1101         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1102         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1103         u64temp = (unsigned long)&nesdev->ccq;
1104         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1105                         cpu_to_le32((u32)(u64temp >> 1));
1106         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1107                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1108         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1109
1110         /* Write Create CEQ WQE */
1111         cqp_head = nesdev->cqp.sq_head++;
1112         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1113         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1114         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1115                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1116         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1117         u64temp = (u64)ceq->ceq_pbase;
1118         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1119
1120         /* Write Create AEQ WQE */
1121         cqp_head = nesdev->cqp.sq_head++;
1122         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1123         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1124         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1125                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1126         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1127         u64temp = (u64)aeq->aeq_pbase;
1128         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1129
1130         /* Write Create NIC CEQ WQE */
1131         cqp_head = nesdev->cqp.sq_head++;
1132         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1133         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1134         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1135                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1136         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1137         u64temp = (u64)nic_ceq->ceq_pbase;
1138         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1139
1140         /* Poll until CCQP done */
1141         count = 0;
1142         do {
1143                 if (count++ > 1000) {
1144                         printk(KERN_ERR PFX "Error creating CQP\n");
1145                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1146                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1147                         return -1;
1148                 }
1149                 udelay(10);
1150         } while (!(nes_read_indexed(nesdev,
1151                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1152
1153         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1154                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1155
1156         u32temp = 0x04800000;
1157         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1158
1159         /* wait for the CCQ, CEQ, and AEQ to get created */
1160         count = 0;
1161         do {
1162                 if (count++ > 1000) {
1163                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1164                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1165                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1166                         return -1;
1167                 }
1168                 udelay(10);
1169         } while (((nes_read_indexed(nesdev,
1170                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1171
1172         /* dump the QP status value */
1173         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1174                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1175
1176         nesdev->cqp.sq_tail++;
1177
1178         return 0;
1179 }
1180
1181
1182 /**
1183  * nes_destroy_cqp
1184  */
1185 int nes_destroy_cqp(struct nes_device *nesdev)
1186 {
1187         struct nes_hw_cqp_wqe *cqp_wqe;
1188         u32 count = 0;
1189         u32 cqp_head;
1190         unsigned long flags;
1191
1192         do {
1193                 if (count++ > 1000)
1194                         break;
1195                 udelay(10);
1196         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1197
1198         /* Reset CCQ */
1199         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1200                         nesdev->ccq.cq_number);
1201
1202         /* Disable device interrupts */
1203         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1204
1205         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1206
1207         /* Destroy the AEQ */
1208         cqp_head = nesdev->cqp.sq_head++;
1209         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1210         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1211         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1212                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1213         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1214
1215         /* Destroy the NIC CEQ */
1216         cqp_head = nesdev->cqp.sq_head++;
1217         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1218         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1219         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1220                         ((u32)nesdev->nic_ceq_index << 8));
1221
1222         /* Destroy the CEQ */
1223         cqp_head = nesdev->cqp.sq_head++;
1224         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1225         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1226         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1227                         (nesdev->ceq_index << 8));
1228
1229         /* Destroy the CCQ */
1230         cqp_head = nesdev->cqp.sq_head++;
1231         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1232         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1233         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1234         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1235                         ((u32)nesdev->ceq_index << 16));
1236
1237         /* Destroy CQP */
1238         cqp_head = nesdev->cqp.sq_head++;
1239         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1240         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1241         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1242                         NES_CQP_QP_TYPE_CQP);
1243         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1244
1245         barrier();
1246         /* Ring doorbell (5 WQEs) */
1247         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1248
1249         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1250
1251         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1252         count = 0;
1253         do {
1254                 if (count++ > 1000) {
1255                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1256                                         PCI_FUNC(nesdev->pcidev->devfn));
1257                         break;
1258                 }
1259                 udelay(10);
1260         } while (((nes_read_indexed(nesdev,
1261                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1262
1263         /* dump the QP status value */
1264         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1265                         PCI_FUNC(nesdev->pcidev->devfn),
1266                         nes_read_indexed(nesdev,
1267                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1268
1269         kfree(nesdev->nes_cqp_requests);
1270
1271         /* Free the control structures */
1272         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1273                         nesdev->cqp.sq_pbase);
1274
1275         return 0;
1276 }
1277
1278
1279 /**
1280  * nes_init_phy
1281  */
1282 int nes_init_phy(struct nes_device *nesdev)
1283 {
1284         struct nes_adapter *nesadapter = nesdev->nesadapter;
1285         u32 counter = 0;
1286         u32 sds;
1287         u32 mac_index = nesdev->mac_index;
1288         u32 tx_config = 0;
1289         u16 phy_data;
1290         u32 temp_phy_data = 0;
1291         u32 temp_phy_data2 = 0;
1292         u8  phy_type = nesadapter->phy_type[mac_index];
1293         u8  phy_index = nesadapter->phy_index[mac_index];
1294
1295         if ((nesadapter->OneG_Mode) &&
1296             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1297                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1298                 if (phy_type == NES_PHY_TYPE_1G) {
1299                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1300                         tx_config &= 0xFFFFFFE3;
1301                         tx_config |= 0x04;
1302                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1303                 }
1304
1305                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1306                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1307
1308                 /* Reset the PHY */
1309                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1310                 udelay(100);
1311                 counter = 0;
1312                 do {
1313                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1314                         if (counter++ > 100)
1315                                 break;
1316                 } while (phy_data & 0x8000);
1317
1318                 /* Setting no phy loopback */
1319                 phy_data &= 0xbfff;
1320                 phy_data |= 0x1140;
1321                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1322                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1323                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1324                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1325
1326                 /* Setting the interrupt mask */
1327                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1328                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1329                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1330
1331                 /* turning on flow control */
1332                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1333                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1334                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1335
1336                 /* Clear Half duplex */
1337                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1338                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1339                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1340
1341                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1342                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1343
1344                 return 0;
1345         }
1346
1347         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1348             (phy_type == NES_PHY_TYPE_ARGUS) ||
1349             (phy_type == NES_PHY_TYPE_SFP_D)) {
1350                 /* setup 10G MDIO operation */
1351                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1352                 tx_config &= 0xFFFFFFE3;
1353                 tx_config |= 0x15;
1354                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1355         }
1356         if ((phy_type == NES_PHY_TYPE_ARGUS) ||
1357             (phy_type == NES_PHY_TYPE_SFP_D)) {
1358                 /* Check firmware heartbeat */
1359                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1360                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1361                 udelay(1500);
1362                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1363                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1364
1365                 if (temp_phy_data != temp_phy_data2)
1366                         return 0;
1367
1368                 /* no heartbeat, configure the PHY */
1369                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1370                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1371                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1372                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1373                 if (phy_type == NES_PHY_TYPE_ARGUS) {
1374                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1375                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1376                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1377                 } else {
1378                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1379                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1380                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1381                 }
1382                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1383                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1384
1385                 /* setup LEDs */
1386                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1389
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1391
1392                 /* Bring PHY out of reset */
1393                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1394
1395                 /* Check for heartbeat */
1396                 counter = 0;
1397                 mdelay(690);
1398                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1399                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1400                 do {
1401                         if (counter++ > 150) {
1402                                 nes_debug(NES_DBG_PHY, "No PHY heartbeat\n");
1403                                 break;
1404                         }
1405                         mdelay(1);
1406                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1407                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1408                 } while ((temp_phy_data2 == temp_phy_data));
1409
1410                 /* wait for tracking */
1411                 counter = 0;
1412                 do {
1413                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1414                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1415                         if (counter++ > 300) {
1416                                 nes_debug(NES_DBG_PHY, "PHY did not track\n");
1417                                 break;
1418                         }
1419                         mdelay(10);
1420                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1421
1422                 /* setup signal integrity */
1423                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1428
1429                 /* reset serdes */
1430                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1431                                        mac_index * 0x200);
1432                 sds |= 0x1;
1433                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1434                                   mac_index * 0x200, sds);
1435                 sds &= 0xfffffffe;
1436                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1437                                   mac_index * 0x200, sds);
1438
1439                 counter = 0;
1440                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1441                                 && (counter++ < 5000))
1442                         ;
1443         }
1444         return 0;
1445 }
1446
1447
1448 /**
1449  * nes_replenish_nic_rq
1450  */
1451 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1452 {
1453         unsigned long flags;
1454         dma_addr_t bus_address;
1455         struct sk_buff *skb;
1456         struct nes_hw_nic_rq_wqe *nic_rqe;
1457         struct nes_hw_nic *nesnic;
1458         struct nes_device *nesdev;
1459         u32 rx_wqes_posted = 0;
1460
1461         nesnic = &nesvnic->nic;
1462         nesdev = nesvnic->nesdev;
1463         spin_lock_irqsave(&nesnic->rq_lock, flags);
1464         if (nesnic->replenishing_rq !=0) {
1465                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1466                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1467                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1468                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1469                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1470                         add_timer(&nesvnic->rq_wqes_timer);
1471                 } else
1472                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1473                 return;
1474         }
1475         nesnic->replenishing_rq = 1;
1476         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1477         do {
1478                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1479                 if (skb) {
1480                         skb->dev = nesvnic->netdev;
1481
1482                         bus_address = pci_map_single(nesdev->pcidev,
1483                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1484
1485                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1486                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1487                                         cpu_to_le32(nesvnic->max_frame_size);
1488                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1489                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1490                                         cpu_to_le32((u32)bus_address);
1491                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1492                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1493                         nesnic->rx_skb[nesnic->rq_head] = skb;
1494                         nesnic->rq_head++;
1495                         nesnic->rq_head &= nesnic->rq_size - 1;
1496                         atomic_dec(&nesvnic->rx_skbs_needed);
1497                         barrier();
1498                         if (++rx_wqes_posted == 255) {
1499                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1500                                 rx_wqes_posted = 0;
1501                         }
1502                 } else {
1503                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1504                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1505                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1506                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1507                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1508                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1509                                 add_timer(&nesvnic->rq_wqes_timer);
1510                         } else
1511                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1512                         break;
1513                 }
1514         } while (atomic_read(&nesvnic->rx_skbs_needed));
1515         barrier();
1516         if (rx_wqes_posted)
1517                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1518         nesnic->replenishing_rq = 0;
1519 }
1520
1521
1522 /**
1523  * nes_rq_wqes_timeout
1524  */
1525 static void nes_rq_wqes_timeout(unsigned long parm)
1526 {
1527         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1528         printk("%s: Timer fired.\n", __func__);
1529         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1530         if (atomic_read(&nesvnic->rx_skbs_needed))
1531                 nes_replenish_nic_rq(nesvnic);
1532 }
1533
1534
1535 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1536                                void **tcph, u64 *hdr_flags, void *priv)
1537 {
1538         unsigned int ip_len;
1539         struct iphdr *iph;
1540         skb_reset_network_header(skb);
1541         iph = ip_hdr(skb);
1542         if (iph->protocol != IPPROTO_TCP)
1543                 return -1;
1544         ip_len = ip_hdrlen(skb);
1545         skb_set_transport_header(skb, ip_len);
1546         *tcph = tcp_hdr(skb);
1547
1548         *hdr_flags = LRO_IPV4 | LRO_TCP;
1549         *iphdr = iph;
1550         return 0;
1551 }
1552
1553
1554 /**
1555  * nes_init_nic_qp
1556  */
1557 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1558 {
1559         struct nes_hw_cqp_wqe *cqp_wqe;
1560         struct nes_hw_nic_sq_wqe *nic_sqe;
1561         struct nes_hw_nic_qp_context *nic_context;
1562         struct sk_buff *skb;
1563         struct nes_hw_nic_rq_wqe *nic_rqe;
1564         struct nes_vnic *nesvnic = netdev_priv(netdev);
1565         unsigned long flags;
1566         void *vmem;
1567         dma_addr_t pmem;
1568         u64 u64temp;
1569         int ret;
1570         u32 cqp_head;
1571         u32 counter;
1572         u32 wqe_count;
1573         u8 jumbomode=0;
1574
1575         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1576         nesvnic->nic_mem_size = 256 +
1577                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1578                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1579                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1580                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1581                         sizeof(struct nes_hw_nic_qp_context);
1582
1583         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1584                         &nesvnic->nic_pbase);
1585         if (!nesvnic->nic_vbase) {
1586                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1587                 return -ENOMEM;
1588         }
1589         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1590         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1591                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1592
1593         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1594                         ~(unsigned long)(256 - 1));
1595         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1596                         ~(unsigned long long)(256 - 1));
1597
1598         /* Setup the first Fragment buffers */
1599         nesvnic->nic.first_frag_vbase = vmem;
1600
1601         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1602                 nesvnic->nic.frag_paddr[counter] = pmem;
1603                 pmem += sizeof(struct nes_first_frag);
1604         }
1605
1606         /* setup the SQ */
1607         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1608
1609         nesvnic->nic.sq_vbase = (void *)vmem;
1610         nesvnic->nic.sq_pbase = pmem;
1611         nesvnic->nic.sq_head = 0;
1612         nesvnic->nic.sq_tail = 0;
1613         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1614         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1615                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1616                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1617                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1618                                 NES_NIC_SQ_WQE_COMPLETION);
1619                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1620                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1621                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1622                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1623                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1624                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1625         }
1626
1627         nesvnic->get_cqp_request = nes_get_cqp_request;
1628         nesvnic->post_cqp_request = nes_post_cqp_request;
1629         nesvnic->mcrq_mcast_filter = NULL;
1630
1631         spin_lock_init(&nesvnic->nic.rq_lock);
1632
1633         /* setup the RQ */
1634         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1635         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1636
1637
1638         nesvnic->nic.rq_vbase = vmem;
1639         nesvnic->nic.rq_pbase = pmem;
1640         nesvnic->nic.rq_head = 0;
1641         nesvnic->nic.rq_tail = 0;
1642         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1643
1644         /* setup the CQ */
1645         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1646         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1647
1648         if (nesdev->nesadapter->netdev_count > 2)
1649                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1650         else
1651                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1652
1653         nesvnic->nic_cq.cq_vbase = vmem;
1654         nesvnic->nic_cq.cq_pbase = pmem;
1655         nesvnic->nic_cq.cq_head = 0;
1656         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1657
1658         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1659
1660         /* Send CreateCQ request to CQP */
1661         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1662         cqp_head = nesdev->cqp.sq_head;
1663
1664         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1665         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1666
1667         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1668                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1669                         ((u32)nesvnic->nic_cq.cq_size << 16));
1670         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1671                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1672         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1673         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1674         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1675         u64temp = (unsigned long)&nesvnic->nic_cq;
1676         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1677         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1678                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1679         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1680         if (++cqp_head >= nesdev->cqp.sq_size)
1681                 cqp_head = 0;
1682         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1683         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1684
1685         /* Send CreateQP request to CQP */
1686         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1687         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1688                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1689                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1690         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1691                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1692                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1693         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1694                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1695         }
1696
1697         u64temp = (u64)nesvnic->nic.sq_pbase;
1698         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1699         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1700         u64temp = (u64)nesvnic->nic.rq_pbase;
1701         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1702         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1703
1704         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1705                         NES_CQP_QP_TYPE_NIC);
1706         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1707         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1708                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1709         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1710
1711         if (++cqp_head >= nesdev->cqp.sq_size)
1712                 cqp_head = 0;
1713         nesdev->cqp.sq_head = cqp_head;
1714
1715         barrier();
1716
1717         /* Ring doorbell (2 WQEs) */
1718         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1719
1720         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1721         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1722                         nesvnic->nic.qp_id);
1723
1724         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1725                         NES_EVENT_TIMEOUT);
1726         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1727                         nesvnic->nic.qp_id, ret);
1728         if (!ret) {
1729                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1730                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1731                                 nesvnic->nic_pbase);
1732                 return -EIO;
1733         }
1734
1735         /* Populate the RQ */
1736         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1737                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1738                 if (!skb) {
1739                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1740
1741                         nes_destroy_nic_qp(nesvnic);
1742                         return -ENOMEM;
1743                 }
1744
1745                 skb->dev = netdev;
1746
1747                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1748                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1749
1750                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1751                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1752                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1753                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1754                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1755                 nesvnic->nic.rx_skb[counter] = skb;
1756         }
1757
1758         wqe_count = NES_NIC_WQ_SIZE - 1;
1759         nesvnic->nic.rq_head = wqe_count;
1760         barrier();
1761         do {
1762                 counter = min(wqe_count, ((u32)255));
1763                 wqe_count -= counter;
1764                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1765         } while (wqe_count);
1766         init_timer(&nesvnic->rq_wqes_timer);
1767         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1768         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1769         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1770         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1771         {
1772                 nes_nic_init_timer(nesdev);
1773                 if (netdev->mtu > 1500)
1774                         jumbomode = 1;
1775                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1776         }
1777         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1778         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1779         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1780         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1781         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1782         nesvnic->lro_mgr.dev            = netdev;
1783         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1784         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1785         return 0;
1786 }
1787
1788
1789 /**
1790  * nes_destroy_nic_qp
1791  */
1792 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1793 {
1794         u64 u64temp;
1795         dma_addr_t bus_address;
1796         struct nes_device *nesdev = nesvnic->nesdev;
1797         struct nes_hw_cqp_wqe *cqp_wqe;
1798         struct nes_hw_nic_sq_wqe *nic_sqe;
1799         struct nes_hw_nic_rq_wqe *nic_rqe;
1800         __le16 *wqe_fragment_length;
1801         u16  wqe_fragment_index;
1802         u64 wqe_frag;
1803         u32 cqp_head;
1804         unsigned long flags;
1805         int ret;
1806
1807         /* Free remaining NIC receive buffers */
1808         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1809                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1810                 wqe_frag  = (u64)le32_to_cpu(
1811                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1812                 wqe_frag |= ((u64)le32_to_cpu(
1813                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1814                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1815                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1816                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1817                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1818         }
1819
1820         /* Free remaining NIC transmit buffers */
1821         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1822                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1823                 wqe_fragment_index = 1;
1824                 wqe_fragment_length = (__le16 *)
1825                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1826                 /* bump past the vlan tag */
1827                 wqe_fragment_length++;
1828                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1829                         u64temp = (u64)le32_to_cpu(
1830                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1831                                 wqe_fragment_index*2]);
1832                         u64temp += ((u64)le32_to_cpu(
1833                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1834                                 + wqe_fragment_index*2]))<<32;
1835                         bus_address = (dma_addr_t)u64temp;
1836                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1837                                         nesvnic->nic.first_frag_overflow)) {
1838                                 pci_unmap_single(nesdev->pcidev,
1839                                                 bus_address,
1840                                                 le16_to_cpu(wqe_fragment_length[
1841                                                         wqe_fragment_index++]),
1842                                                 PCI_DMA_TODEVICE);
1843                         }
1844                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1845                                 if (wqe_fragment_length[wqe_fragment_index]) {
1846                                         u64temp = le32_to_cpu(
1847                                                 nic_sqe->wqe_words[
1848                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1849                                                 wqe_fragment_index*2]);
1850                                         u64temp += ((u64)le32_to_cpu(
1851                                                 nic_sqe->wqe_words[
1852                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1853                                                 wqe_fragment_index*2]))<<32;
1854                                         bus_address = (dma_addr_t)u64temp;
1855                                         pci_unmap_page(nesdev->pcidev,
1856                                                         bus_address,
1857                                                         le16_to_cpu(
1858                                                         wqe_fragment_length[
1859                                                         wqe_fragment_index]),
1860                                                         PCI_DMA_TODEVICE);
1861                                 } else
1862                                         break;
1863                         }
1864                 }
1865                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1866                         dev_kfree_skb(
1867                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1868
1869                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1870                                         & (nesvnic->nic.sq_size - 1);
1871         }
1872
1873         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1874
1875         /* Destroy NIC QP */
1876         cqp_head = nesdev->cqp.sq_head;
1877         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1878         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1879
1880         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1881                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1882         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1883                 nesvnic->nic.qp_id);
1884
1885         if (++cqp_head >= nesdev->cqp.sq_size)
1886                 cqp_head = 0;
1887
1888         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1889
1890         /* Destroy NIC CQ */
1891         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1892         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1893                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1894         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1895                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1896
1897         if (++cqp_head >= nesdev->cqp.sq_size)
1898                 cqp_head = 0;
1899
1900         nesdev->cqp.sq_head = cqp_head;
1901         barrier();
1902
1903         /* Ring doorbell (2 WQEs) */
1904         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1905
1906         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1907         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1908                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1909                         cqp_head, nesdev->cqp.sq_head,
1910                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1911
1912         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1913                         NES_EVENT_TIMEOUT);
1914
1915         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1916                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1917                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1918         if (!ret) {
1919                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1920                                 nesvnic->nic.qp_id);
1921         }
1922
1923         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1924                         nesvnic->nic_pbase);
1925 }
1926
1927 /**
1928  * nes_napi_isr
1929  */
1930 int nes_napi_isr(struct nes_device *nesdev)
1931 {
1932         struct nes_adapter *nesadapter = nesdev->nesadapter;
1933         u32 int_stat;
1934
1935         if (nesdev->napi_isr_ran) {
1936                 /* interrupt status has already been read in ISR */
1937                 int_stat = nesdev->int_stat;
1938         } else {
1939                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1940                 nesdev->int_stat = int_stat;
1941                 nesdev->napi_isr_ran = 1;
1942         }
1943
1944         int_stat &= nesdev->int_req;
1945         /* iff NIC, process here, else wait for DPC */
1946         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1947                 nesdev->napi_isr_ran = 0;
1948                 nes_write32(nesdev->regs + NES_INT_STAT,
1949                         (int_stat &
1950                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1951
1952                 /* Process the CEQs */
1953                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1954
1955                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1956                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1957                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1958                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1959                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1960                                 /* Enable Periodic timer interrupts */
1961                                 nesdev->int_req |= NES_INT_TIMER;
1962                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1963                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1964                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1965                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1966                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1967                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1968                         }
1969
1970                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1971                         {
1972                                 nes_nic_init_timer(nesdev);
1973                         }
1974                         /* Enable interrupts, except CEQs */
1975                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1976                 } else {
1977                         /* Enable interrupts, make sure timer is off */
1978                         nesdev->int_req &= ~NES_INT_TIMER;
1979                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1980                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1981                 }
1982                 nesdev->deepcq_count = 0;
1983                 return 1;
1984         } else {
1985                 return 0;
1986         }
1987 }
1988
1989 static void process_critical_error(struct nes_device *nesdev)
1990 {
1991         u32 debug_error;
1992         u32 nes_idx_debug_error_masks0 = 0;
1993         u16 error_module = 0;
1994
1995         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1996         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1997                         (u16)debug_error);
1998         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1999                         0x01010000 | (debug_error & 0x0000ffff));
2000         if (crit_err_count++ > 10)
2001                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2002         error_module = (u16) (debug_error & 0x1F00) >> 8;
2003         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2004                         nes_max_critical_error_count) {
2005                 printk(KERN_ERR PFX "Masking off critical error for module "
2006                         "0x%02X\n", (u16)error_module);
2007                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2008                         NES_IDX_DEBUG_ERROR_MASKS0);
2009                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2010                         nes_idx_debug_error_masks0 | (1 << error_module));
2011         }
2012 }
2013 /**
2014  * nes_dpc
2015  */
2016 void nes_dpc(unsigned long param)
2017 {
2018         struct nes_device *nesdev = (struct nes_device *)param;
2019         struct nes_adapter *nesadapter = nesdev->nesadapter;
2020         u32 counter;
2021         u32 loop_counter = 0;
2022         u32 int_status_bit;
2023         u32 int_stat;
2024         u32 timer_stat;
2025         u32 temp_int_stat;
2026         u32 intf_int_stat;
2027         u32 processed_intf_int = 0;
2028         u16 processed_timer_int = 0;
2029         u16 completion_ints = 0;
2030         u16 timer_ints = 0;
2031
2032         /* nes_debug(NES_DBG_ISR, "\n"); */
2033
2034         do {
2035                 timer_stat = 0;
2036                 if (nesdev->napi_isr_ran) {
2037                         nesdev->napi_isr_ran = 0;
2038                         int_stat = nesdev->int_stat;
2039                 } else
2040                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2041                 if (processed_intf_int != 0)
2042                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2043                 else
2044                         int_stat &= nesdev->int_req;
2045                 if (processed_timer_int == 0) {
2046                         processed_timer_int = 1;
2047                         if (int_stat & NES_INT_TIMER) {
2048                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2049                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2050                                         int_stat &= ~NES_INT_TIMER;
2051                                 }
2052                         }
2053                 } else {
2054                         int_stat &= ~NES_INT_TIMER;
2055                 }
2056
2057                 if (int_stat) {
2058                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2059                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2060                                 /* Ack the interrupts */
2061                                 nes_write32(nesdev->regs+NES_INT_STAT,
2062                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2063                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2064                         }
2065
2066                         temp_int_stat = int_stat;
2067                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2068                                 if (int_stat & int_status_bit) {
2069                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2070                                         temp_int_stat &= ~int_status_bit;
2071                                         completion_ints = 1;
2072                                 }
2073                                 if (!(temp_int_stat & 0x0000ffff))
2074                                         break;
2075                                 int_status_bit <<= 1;
2076                         }
2077
2078                         /* Process the AEQ for this pci function */
2079                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2080                         if (int_stat & int_status_bit) {
2081                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2082                         }
2083
2084                         /* Process the MAC interrupt for this pci function */
2085                         int_status_bit = 1 << (24 + nesdev->mac_index);
2086                         if (int_stat & int_status_bit) {
2087                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2088                         }
2089
2090                         if (int_stat & NES_INT_TIMER) {
2091                                 if (timer_stat & nesdev->timer_int_req) {
2092                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2093                                                         (timer_stat & nesdev->timer_int_req) |
2094                                                         ~(nesdev->nesadapter->timer_int_req));
2095                                         timer_ints = 1;
2096                                 }
2097                         }
2098
2099                         if (int_stat & NES_INT_INTF) {
2100                                 processed_intf_int = 1;
2101                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2102                                 intf_int_stat &= nesdev->intf_int_req;
2103                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2104                                         process_critical_error(nesdev);
2105                                 }
2106                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2107                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2108                                         BUG();
2109                                 }
2110                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2111                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2112                                         BUG();
2113                                 }
2114                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2115                         }
2116
2117                         if (int_stat & NES_INT_TSW) {
2118                         }
2119                 }
2120                 /* Don't use the interface interrupt bit stay in loop */
2121                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2122                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2123         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2124
2125         if (timer_ints == 1) {
2126                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2127                         if (completion_ints == 0) {
2128                                 nesdev->timer_only_int_count++;
2129                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2130                                         nesdev->timer_only_int_count = 0;
2131                                         nesdev->int_req &= ~NES_INT_TIMER;
2132                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2133                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2134                                 } else {
2135                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2136                                 }
2137                         } else {
2138                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2139                                 {
2140                                         nes_nic_init_timer(nesdev);
2141                                 }
2142                                 nesdev->timer_only_int_count = 0;
2143                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2144                         }
2145                 } else {
2146                         nesdev->timer_only_int_count = 0;
2147                         nesdev->int_req &= ~NES_INT_TIMER;
2148                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2149                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2150                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2151                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2152                 }
2153         } else {
2154                 if ( (completion_ints == 1) &&
2155                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2156                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2157                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2158                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2159                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2160                         nesdev->timer_only_int_count = 0;
2161                         nesdev->int_req |= NES_INT_TIMER;
2162                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2163                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2164                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2165                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2166                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2167                 } else {
2168                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2169                 }
2170         }
2171         nesdev->deepcq_count = 0;
2172 }
2173
2174
2175 /**
2176  * nes_process_ceq
2177  */
2178 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2179 {
2180         u64 u64temp;
2181         struct nes_hw_cq *cq;
2182         u32 head;
2183         u32 ceq_size;
2184
2185         /* nes_debug(NES_DBG_CQ, "\n"); */
2186         head = ceq->ceq_head;
2187         ceq_size = ceq->ceq_size;
2188
2189         do {
2190                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2191                                 NES_CEQE_VALID) {
2192                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2193                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2194                         u64temp <<= 1;
2195                         cq = *((struct nes_hw_cq **)&u64temp);
2196                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2197                         barrier();
2198                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2199
2200                         /* call the event handler */
2201                         cq->ce_handler(nesdev, cq);
2202
2203                         if (++head >= ceq_size)
2204                                 head = 0;
2205                 } else {
2206                         break;
2207                 }
2208
2209         } while (1);
2210
2211         ceq->ceq_head = head;
2212 }
2213
2214
2215 /**
2216  * nes_process_aeq
2217  */
2218 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2219 {
2220         /* u64 u64temp; */
2221         u32 head;
2222         u32 aeq_size;
2223         u32 aeqe_misc;
2224         u32 aeqe_cq_id;
2225         struct nes_hw_aeqe volatile *aeqe;
2226
2227         head = aeq->aeq_head;
2228         aeq_size = aeq->aeq_size;
2229
2230         do {
2231                 aeqe = &aeq->aeq_vbase[head];
2232                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2233                         break;
2234                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2235                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2236                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2237                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2238                                 /* dealing with an accelerated QP related AE */
2239                                 /*
2240                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2241                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2242                                  */
2243                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2244                         } else {
2245                                 /* TODO: dealing with a CQP related AE */
2246                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2247                                                 (u16)(aeqe_misc >> 16));
2248                         }
2249                 }
2250
2251                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2252
2253                 if (++head >= aeq_size)
2254                         head = 0;
2255
2256                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2257         }
2258         while (1);
2259         aeq->aeq_head = head;
2260 }
2261
2262 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2263 {
2264         struct nes_adapter *nesadapter = nesdev->nesadapter;
2265         u32 reset_value;
2266         u32 i=0;
2267         u32 u32temp;
2268
2269         if (nesadapter->hw_rev == NE020_REV) {
2270                 return;
2271         }
2272         mh_detected++;
2273
2274         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2275
2276         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2277                 reset_value |= 0x0000001d;
2278         else
2279                 reset_value |= 0x0000002d;
2280
2281         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2282                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2283                         nesadapter->link_interrupt_count[0] = 0;
2284                         nesadapter->link_interrupt_count[1] = 0;
2285                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2286                         if (0x00000040 & u32temp)
2287                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2288                         else
2289                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2290
2291                         reset_value |= 0x0000003d;
2292                 }
2293                 nesadapter->link_interrupt_count[mac_index] = 0;
2294         }
2295
2296         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2297
2298         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2299                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2300
2301         if (0x0000003d == (reset_value & 0x0000003d)) {
2302                 u32 pcs_control_status0, pcs_control_status1;
2303
2304                 for (i = 0; i < 10; i++) {
2305                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2306                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2307                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2308                              && (pcs_control_status0 & 0x00100000))
2309                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2310                                 && (pcs_control_status1 & 0x00100000)))
2311                                 continue;
2312                         else
2313                                 break;
2314                 }
2315                 if (10 == i) {
2316                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2317                         if (0x00000040 & u32temp)
2318                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2319                         else
2320                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2321
2322                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2323
2324                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2325                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2326                 }
2327         }
2328 }
2329
2330 /**
2331  * nes_process_mac_intr
2332  */
2333 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2334 {
2335         unsigned long flags;
2336         u32 pcs_control_status;
2337         struct nes_adapter *nesadapter = nesdev->nesadapter;
2338         struct nes_vnic *nesvnic;
2339         u32 mac_status;
2340         u32 mac_index = nesdev->mac_index;
2341         u32 u32temp;
2342         u16 phy_data;
2343         u16 temp_phy_data;
2344         u32 pcs_val  = 0x0f0f0000;
2345         u32 pcs_mask = 0x0f1f0000;
2346         u32 cdr_ctrl;
2347
2348         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2349         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2350                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2351                 return;
2352         }
2353         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2354         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2355
2356         /* ack the MAC interrupt */
2357         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2358         /* Clear the interrupt */
2359         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2360
2361         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2362
2363         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2364                 nesdev->link_status_interrupts++;
2365                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2366                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2367                         nes_reset_link(nesdev, mac_index);
2368                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2369                 }
2370                 /* read the PHY interrupt status register */
2371                 if ((nesadapter->OneG_Mode) &&
2372                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2373                         do {
2374                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2375                                                 nesadapter->phy_index[mac_index], &phy_data);
2376                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2377                                                 nesadapter->phy_index[mac_index], phy_data);
2378                         } while (phy_data&0x8000);
2379
2380                         temp_phy_data = 0;
2381                         do {
2382                                 nes_read_1G_phy_reg(nesdev, 0x11,
2383                                                 nesadapter->phy_index[mac_index], &phy_data);
2384                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2385                                                 nesadapter->phy_index[mac_index], phy_data);
2386                                 if (temp_phy_data == phy_data)
2387                                         break;
2388                                 temp_phy_data = phy_data;
2389                         } while (1);
2390
2391                         nes_read_1G_phy_reg(nesdev, 0x1e,
2392                                         nesadapter->phy_index[mac_index], &phy_data);
2393                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2394                                         nesadapter->phy_index[mac_index], phy_data);
2395
2396                         nes_read_1G_phy_reg(nesdev, 1,
2397                                         nesadapter->phy_index[mac_index], &phy_data);
2398                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2399                                         nesadapter->phy_index[mac_index], phy_data);
2400
2401                         if (temp_phy_data & 0x1000) {
2402                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2403                                 phy_data = 4;
2404                         } else {
2405                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2406                         }
2407                 }
2408                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2409                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2410                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2411
2412                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2413                         switch (mac_index) {
2414                         case 1:
2415                         case 3:
2416                                 pcs_control_status = nes_read_indexed(nesdev,
2417                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2418                                 break;
2419                         default:
2420                                 pcs_control_status = nes_read_indexed(nesdev,
2421                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2422                                 break;
2423                         }
2424                 } else {
2425                         pcs_control_status = nes_read_indexed(nesdev,
2426                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2427                         pcs_control_status = nes_read_indexed(nesdev,
2428                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2429                 }
2430
2431                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2432                                 mac_index, pcs_control_status);
2433                 if ((nesadapter->OneG_Mode) &&
2434                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2435                         u32temp = 0x01010000;
2436                         if (nesadapter->port_count > 2) {
2437                                 u32temp |= 0x02020000;
2438                         }
2439                         if ((pcs_control_status & u32temp)!= u32temp) {
2440                                 phy_data = 0;
2441                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2442                         }
2443                 } else {
2444                         switch (nesadapter->phy_type[mac_index]) {
2445                         case NES_PHY_TYPE_IRIS:
2446                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2447                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2448                                 u32temp = 20;
2449                                 do {
2450                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2451                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2452                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2453                                                 break;
2454                                         temp_phy_data = phy_data;
2455                                 } while (1);
2456                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2457                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2458                                 break;
2459
2460                         case NES_PHY_TYPE_ARGUS:
2461                         case NES_PHY_TYPE_SFP_D:
2462                                 /* clear the alarms */
2463                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2464                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2465                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2466                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2467                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2468                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2469                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2470                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2471                                 /* check link status */
2472                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2473                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2474
2475                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2476                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2477                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2478                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2479
2480                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2481
2482                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2483                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2484                                 break;
2485
2486                         case NES_PHY_TYPE_PUMA_1G:
2487                                 if (mac_index < 2)
2488                                         pcs_val = pcs_mask = 0x01010000;
2489                                 else
2490                                         pcs_val = pcs_mask = 0x02020000;
2491                                 /* fall through */
2492                         default:
2493                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2494                                 break;
2495                         }
2496                 }
2497
2498                 if (phy_data & 0x0004) {
2499                         if (wide_ppm_offset &&
2500                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2501                             (nesadapter->hw_rev != NE020_REV)) {
2502                                 cdr_ctrl = nes_read_indexed(nesdev,
2503                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2504                                                             mac_index * 0x200);
2505                                 nes_write_indexed(nesdev,
2506                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2507                                                   mac_index * 0x200,
2508                                                   cdr_ctrl | 0x000F0000);
2509                         }
2510                         nesadapter->mac_link_down[mac_index] = 0;
2511                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2512                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2513                                                 nesvnic->linkup);
2514                                 if (nesvnic->linkup == 0) {
2515                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2516                                                         nesvnic->netdev->name, nesvnic->netdev);
2517                                         if (netif_queue_stopped(nesvnic->netdev))
2518                                                 netif_start_queue(nesvnic->netdev);
2519                                         nesvnic->linkup = 1;
2520                                         netif_carrier_on(nesvnic->netdev);
2521                                 }
2522                         }
2523                 } else {
2524                         if (wide_ppm_offset &&
2525                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2526                             (nesadapter->hw_rev != NE020_REV)) {
2527                                 cdr_ctrl = nes_read_indexed(nesdev,
2528                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2529                                                             mac_index * 0x200);
2530                                 nes_write_indexed(nesdev,
2531                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2532                                                   mac_index * 0x200,
2533                                                   cdr_ctrl & 0xFFF0FFFF);
2534                         }
2535                         nesadapter->mac_link_down[mac_index] = 1;
2536                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2537                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2538                                                 nesvnic->linkup);
2539                                 if (nesvnic->linkup == 1) {
2540                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2541                                                         nesvnic->netdev->name, nesvnic->netdev);
2542                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2543                                                 netif_stop_queue(nesvnic->netdev);
2544                                         nesvnic->linkup = 0;
2545                                         netif_carrier_off(nesvnic->netdev);
2546                                 }
2547                         }
2548                 }
2549         }
2550
2551         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2552 }
2553
2554
2555
2556 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2557 {
2558         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2559
2560         napi_schedule(&nesvnic->napi);
2561 }
2562
2563
2564 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2565 * getting out of nic_ce_handler
2566 */
2567 #define MAX_RQES_TO_PROCESS     384
2568
2569 /**
2570  * nes_nic_ce_handler
2571  */
2572 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2573 {
2574         u64 u64temp;
2575         dma_addr_t bus_address;
2576         struct nes_hw_nic *nesnic;
2577         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2578         struct nes_adapter *nesadapter = nesdev->nesadapter;
2579         struct nes_hw_nic_rq_wqe *nic_rqe;
2580         struct nes_hw_nic_sq_wqe *nic_sqe;
2581         struct sk_buff *skb;
2582         struct sk_buff *rx_skb;
2583         __le16 *wqe_fragment_length;
2584         u32 head;
2585         u32 cq_size;
2586         u32 rx_pkt_size;
2587         u32 cqe_count=0;
2588         u32 cqe_errv;
2589         u32 cqe_misc;
2590         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2591         u16 vlan_tag;
2592         u16 pkt_type;
2593         u16 rqes_processed = 0;
2594         u8 sq_cqes = 0;
2595         u8 nes_use_lro = 0;
2596
2597         head = cq->cq_head;
2598         cq_size = cq->cq_size;
2599         cq->cqes_pending = 1;
2600         if (nesvnic->netdev->features & NETIF_F_LRO)
2601                 nes_use_lro = 1;
2602         do {
2603                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2604                                 NES_NIC_CQE_VALID) {
2605                         nesnic = &nesvnic->nic;
2606                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2607                         if (cqe_misc & NES_NIC_CQE_SQ) {
2608                                 sq_cqes++;
2609                                 wqe_fragment_index = 1;
2610                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2611                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2612                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2613                                 /* bump past the vlan tag */
2614                                 wqe_fragment_length++;
2615                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2616                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2617                                                         wqe_fragment_index * 2]);
2618                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2619                                                         wqe_fragment_index * 2])) << 32;
2620                                         bus_address = (dma_addr_t)u64temp;
2621                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2622                                                 pci_unmap_single(nesdev->pcidev,
2623                                                                 bus_address,
2624                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2625                                                                 PCI_DMA_TODEVICE);
2626                                         }
2627                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2628                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2629                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2630                                                                                 wqe_fragment_index * 2]);
2631                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2632                                                                                 + wqe_fragment_index * 2])) <<32;
2633                                                         bus_address = (dma_addr_t)u64temp;
2634                                                         pci_unmap_page(nesdev->pcidev,
2635                                                                         bus_address,
2636                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2637                                                                         PCI_DMA_TODEVICE);
2638                                                 } else
2639                                                         break;
2640                                         }
2641                                 }
2642                                 if (skb)
2643                                         dev_kfree_skb_any(skb);
2644                                 nesnic->sq_tail++;
2645                                 nesnic->sq_tail &= nesnic->sq_size-1;
2646                                 if (sq_cqes > 128) {
2647                                         barrier();
2648                                 /* restart the queue if it had been stopped */
2649                                 if (netif_queue_stopped(nesvnic->netdev))
2650                                         netif_wake_queue(nesvnic->netdev);
2651                                         sq_cqes = 0;
2652                                 }
2653                         } else {
2654                                 rqes_processed ++;
2655
2656                                 cq->rx_cqes_completed++;
2657                                 cq->rx_pkts_indicated++;
2658                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2659                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2660                                 /* Get the skb */
2661                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2662                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2663                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2664                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2665                                 pci_unmap_single(nesdev->pcidev, bus_address,
2666                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2667                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2668                                 /* rx_skb->len = rx_pkt_size; */
2669                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2670                                 skb_put(rx_skb, rx_pkt_size);
2671                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2672                                 nesnic->rq_tail++;
2673                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2674
2675                                 atomic_inc(&nesvnic->rx_skbs_needed);
2676                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2677                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2678                                                         cq->cq_number | (cqe_count << 16));
2679                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2680                                         nesdev->currcq_count += cqe_count;
2681                                         cqe_count = 0;
2682                                         nes_replenish_nic_rq(nesvnic);
2683                                 }
2684                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2685                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2686                                 rx_skb->ip_summed = CHECKSUM_NONE;
2687
2688                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2689                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2690                                         if ((cqe_errv &
2691                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2692                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2693                                                 if (nesvnic->rx_checksum_disabled == 0) {
2694                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2695                                                 }
2696                                         } else
2697                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2698                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2699                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2700
2701                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2702                                         if ((cqe_errv &
2703                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2704                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2705                                                 if (nesvnic->rx_checksum_disabled == 0) {
2706                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2707                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2708                                                                   nesvnic->netdev->name); */
2709                                                 }
2710                                         } else
2711                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2712                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2713                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2714                                         }
2715                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2716                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2717
2718                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2719                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2720                                                 rx_skb = NULL;
2721                                 }
2722                                 if (rx_skb == NULL)
2723                                         goto skip_rx_indicate0;
2724
2725
2726                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2727                                     (nesvnic->vlan_grp != NULL)) {
2728                                         vlan_tag = (u16)(le32_to_cpu(
2729                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2730                                                         >> 16);
2731                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2732                                                         nesvnic->netdev->name, vlan_tag);
2733                                         if (nes_use_lro)
2734                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2735                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2736                                         else
2737                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2738                                 } else {
2739                                         if (nes_use_lro)
2740                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2741                                         else
2742                                                 nes_netif_rx(rx_skb);
2743                                 }
2744
2745 skip_rx_indicate0:
2746                                 nesvnic->netdev->last_rx = jiffies;
2747                                 /* nesvnic->netstats.rx_packets++; */
2748                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2749                         }
2750
2751                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2752                         /* Accounting... */
2753                         cqe_count++;
2754                         if (++head >= cq_size)
2755                                 head = 0;
2756                         if (cqe_count == 255) {
2757                                 /* Replenish Nic CQ */
2758                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2759                                                 cq->cq_number | (cqe_count << 16));
2760                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2761                                 nesdev->currcq_count += cqe_count;
2762                                 cqe_count = 0;
2763                         }
2764
2765                         if (cq->rx_cqes_completed >= nesvnic->budget)
2766                                 break;
2767                 } else {
2768                         cq->cqes_pending = 0;
2769                         break;
2770                 }
2771
2772         } while (1);
2773
2774         if (nes_use_lro)
2775                 lro_flush_all(&nesvnic->lro_mgr);
2776         if (sq_cqes) {
2777                 barrier();
2778                 /* restart the queue if it had been stopped */
2779                 if (netif_queue_stopped(nesvnic->netdev))
2780                         netif_wake_queue(nesvnic->netdev);
2781         }
2782         cq->cq_head = head;
2783         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2784                         cq->cq_number, cqe_count, cq->cq_head); */
2785         cq->cqe_allocs_pending = cqe_count;
2786         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2787         {
2788                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2789                 nesdev->currcq_count += cqe_count;
2790                 nes_nic_tune_timer(nesdev);
2791         }
2792         if (atomic_read(&nesvnic->rx_skbs_needed))
2793                 nes_replenish_nic_rq(nesvnic);
2794 }
2795
2796
2797 /**
2798  * nes_cqp_ce_handler
2799  */
2800 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2801 {
2802         u64 u64temp;
2803         unsigned long flags;
2804         struct nes_hw_cqp *cqp = NULL;
2805         struct nes_cqp_request *cqp_request;
2806         struct nes_hw_cqp_wqe *cqp_wqe;
2807         u32 head;
2808         u32 cq_size;
2809         u32 cqe_count=0;
2810         u32 error_code;
2811         /* u32 counter; */
2812
2813         head = cq->cq_head;
2814         cq_size = cq->cq_size;
2815
2816         do {
2817                 /* process the CQE */
2818                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2819                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2820
2821                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2822                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2823                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2824                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2825                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2826                         cqp = *((struct nes_hw_cqp **)&u64temp);
2827
2828                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2829                         if (error_code) {
2830                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2831                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2832                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2833                                                 (u16)(error_code >> 16),
2834                                                 (u16)error_code);
2835                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2836                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2837                         }
2838
2839                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2840                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2841