]> git.openfabrics.org - ~shefty/rdma-dev.git/blob - drivers/scsi/be2iscsi/be_main.c
fc7c97d7e34d753b64a615708c22d3c26749c538
[~shefty/rdma-dev.git] / drivers / scsi / be2iscsi / be_main.c
1 /**
2  * Copyright (C) 2005 - 2011 Emulex
3  * All rights reserved.
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License version 2
7  * as published by the Free Software Foundation.  The full GNU General
8  * Public License is included in this distribution in the file called COPYING.
9  *
10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
11  *
12  * Contact Information:
13  * linux-drivers@emulex.com
14  *
15  * Emulex
16  * 3333 Susan Street
17  * Costa Mesa, CA 92626
18  */
19
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/slab.h>
23 #include <linux/interrupt.h>
24 #include <linux/blkdev.h>
25 #include <linux/pci.h>
26 #include <linux/string.h>
27 #include <linux/kernel.h>
28 #include <linux/semaphore.h>
29 #include <linux/iscsi_boot_sysfs.h>
30 #include <linux/module.h>
31
32 #include <scsi/libiscsi.h>
33 #include <scsi/scsi_transport_iscsi.h>
34 #include <scsi/scsi_transport.h>
35 #include <scsi/scsi_cmnd.h>
36 #include <scsi/scsi_device.h>
37 #include <scsi/scsi_host.h>
38 #include <scsi/scsi.h>
39 #include "be_main.h"
40 #include "be_iscsi.h"
41 #include "be_mgmt.h"
42
43 static unsigned int be_iopoll_budget = 10;
44 static unsigned int be_max_phys_size = 64;
45 static unsigned int enable_msix = 1;
46 static unsigned int gcrashmode = 0;
47 static unsigned int num_hba = 0;
48
49 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
50 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
51 MODULE_AUTHOR("ServerEngines Corporation");
52 MODULE_LICENSE("GPL");
53 module_param(be_iopoll_budget, int, 0);
54 module_param(enable_msix, int, 0);
55 module_param(be_max_phys_size, uint, S_IRUGO);
56 MODULE_PARM_DESC(be_max_phys_size, "Maximum Size (In Kilobytes) of physically"
57                                    "contiguous memory that can be allocated."
58                                    "Range is 16 - 128");
59
60 static int beiscsi_slave_configure(struct scsi_device *sdev)
61 {
62         blk_queue_max_segment_size(sdev->request_queue, 65536);
63         return 0;
64 }
65
66 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
67 {
68         struct iscsi_cls_session *cls_session;
69         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
70         struct beiscsi_io_task *aborted_io_task;
71         struct iscsi_conn *conn;
72         struct beiscsi_conn *beiscsi_conn;
73         struct beiscsi_hba *phba;
74         struct iscsi_session *session;
75         struct invalidate_command_table *inv_tbl;
76         struct be_dma_mem nonemb_cmd;
77         unsigned int cid, tag, num_invalidate;
78
79         cls_session = starget_to_session(scsi_target(sc->device));
80         session = cls_session->dd_data;
81
82         spin_lock_bh(&session->lock);
83         if (!aborted_task || !aborted_task->sc) {
84                 /* we raced */
85                 spin_unlock_bh(&session->lock);
86                 return SUCCESS;
87         }
88
89         aborted_io_task = aborted_task->dd_data;
90         if (!aborted_io_task->scsi_cmnd) {
91                 /* raced or invalid command */
92                 spin_unlock_bh(&session->lock);
93                 return SUCCESS;
94         }
95         spin_unlock_bh(&session->lock);
96         conn = aborted_task->conn;
97         beiscsi_conn = conn->dd_data;
98         phba = beiscsi_conn->phba;
99
100         /* invalidate iocb */
101         cid = beiscsi_conn->beiscsi_conn_cid;
102         inv_tbl = phba->inv_tbl;
103         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
104         inv_tbl->cid = cid;
105         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
106         num_invalidate = 1;
107         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
108                                 sizeof(struct invalidate_commands_params_in),
109                                 &nonemb_cmd.dma);
110         if (nonemb_cmd.va == NULL) {
111                 SE_DEBUG(DBG_LVL_1,
112                          "Failed to allocate memory for"
113                          "mgmt_invalidate_icds\n");
114                 return FAILED;
115         }
116         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
117
118         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
119                                    cid, &nonemb_cmd);
120         if (!tag) {
121                 shost_printk(KERN_WARNING, phba->shost,
122                              "mgmt_invalidate_icds could not be"
123                              " submitted\n");
124                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
125                                     nonemb_cmd.va, nonemb_cmd.dma);
126
127                 return FAILED;
128         } else {
129                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
130                                          phba->ctrl.mcc_numtag[tag]);
131                 free_mcc_tag(&phba->ctrl, tag);
132         }
133         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
134                             nonemb_cmd.va, nonemb_cmd.dma);
135         return iscsi_eh_abort(sc);
136 }
137
138 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
139 {
140         struct iscsi_task *abrt_task;
141         struct beiscsi_io_task *abrt_io_task;
142         struct iscsi_conn *conn;
143         struct beiscsi_conn *beiscsi_conn;
144         struct beiscsi_hba *phba;
145         struct iscsi_session *session;
146         struct iscsi_cls_session *cls_session;
147         struct invalidate_command_table *inv_tbl;
148         struct be_dma_mem nonemb_cmd;
149         unsigned int cid, tag, i, num_invalidate;
150         int rc = FAILED;
151
152         /* invalidate iocbs */
153         cls_session = starget_to_session(scsi_target(sc->device));
154         session = cls_session->dd_data;
155         spin_lock_bh(&session->lock);
156         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
157                 goto unlock;
158
159         conn = session->leadconn;
160         beiscsi_conn = conn->dd_data;
161         phba = beiscsi_conn->phba;
162         cid = beiscsi_conn->beiscsi_conn_cid;
163         inv_tbl = phba->inv_tbl;
164         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
165         num_invalidate = 0;
166         for (i = 0; i < conn->session->cmds_max; i++) {
167                 abrt_task = conn->session->cmds[i];
168                 abrt_io_task = abrt_task->dd_data;
169                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
170                         continue;
171
172                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
173                         continue;
174
175                 inv_tbl->cid = cid;
176                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
177                 num_invalidate++;
178                 inv_tbl++;
179         }
180         spin_unlock_bh(&session->lock);
181         inv_tbl = phba->inv_tbl;
182
183         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
184                                 sizeof(struct invalidate_commands_params_in),
185                                 &nonemb_cmd.dma);
186         if (nonemb_cmd.va == NULL) {
187                 SE_DEBUG(DBG_LVL_1,
188                          "Failed to allocate memory for"
189                          "mgmt_invalidate_icds\n");
190                 return FAILED;
191         }
192         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
193         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
194         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
195                                    cid, &nonemb_cmd);
196         if (!tag) {
197                 shost_printk(KERN_WARNING, phba->shost,
198                              "mgmt_invalidate_icds could not be"
199                              " submitted\n");
200                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
201                                     nonemb_cmd.va, nonemb_cmd.dma);
202                 return FAILED;
203         } else {
204                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
205                                          phba->ctrl.mcc_numtag[tag]);
206                 free_mcc_tag(&phba->ctrl, tag);
207         }
208         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
209                             nonemb_cmd.va, nonemb_cmd.dma);
210         return iscsi_eh_device_reset(sc);
211 unlock:
212         spin_unlock_bh(&session->lock);
213         return rc;
214 }
215
216 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
217 {
218         struct beiscsi_hba *phba = data;
219         struct mgmt_session_info *boot_sess = &phba->boot_sess;
220         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
221         char *str = buf;
222         int rc;
223
224         switch (type) {
225         case ISCSI_BOOT_TGT_NAME:
226                 rc = sprintf(buf, "%.*s\n",
227                             (int)strlen(boot_sess->target_name),
228                             (char *)&boot_sess->target_name);
229                 break;
230         case ISCSI_BOOT_TGT_IP_ADDR:
231                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
232                         rc = sprintf(buf, "%pI4\n",
233                                 (char *)&boot_conn->dest_ipaddr.ip_address);
234                 else
235                         rc = sprintf(str, "%pI6\n",
236                                 (char *)&boot_conn->dest_ipaddr.ip_address);
237                 break;
238         case ISCSI_BOOT_TGT_PORT:
239                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
240                 break;
241
242         case ISCSI_BOOT_TGT_CHAP_NAME:
243                 rc = sprintf(str,  "%.*s\n",
244                              boot_conn->negotiated_login_options.auth_data.chap.
245                              target_chap_name_length,
246                              (char *)&boot_conn->negotiated_login_options.
247                              auth_data.chap.target_chap_name);
248                 break;
249         case ISCSI_BOOT_TGT_CHAP_SECRET:
250                 rc = sprintf(str,  "%.*s\n",
251                              boot_conn->negotiated_login_options.auth_data.chap.
252                              target_secret_length,
253                              (char *)&boot_conn->negotiated_login_options.
254                              auth_data.chap.target_secret);
255                 break;
256         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
257                 rc = sprintf(str,  "%.*s\n",
258                              boot_conn->negotiated_login_options.auth_data.chap.
259                              intr_chap_name_length,
260                              (char *)&boot_conn->negotiated_login_options.
261                              auth_data.chap.intr_chap_name);
262                 break;
263         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
264                 rc = sprintf(str,  "%.*s\n",
265                              boot_conn->negotiated_login_options.auth_data.chap.
266                              intr_secret_length,
267                              (char *)&boot_conn->negotiated_login_options.
268                              auth_data.chap.intr_secret);
269                 break;
270         case ISCSI_BOOT_TGT_FLAGS:
271                 rc = sprintf(str, "2\n");
272                 break;
273         case ISCSI_BOOT_TGT_NIC_ASSOC:
274                 rc = sprintf(str, "0\n");
275                 break;
276         default:
277                 rc = -ENOSYS;
278                 break;
279         }
280         return rc;
281 }
282
283 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
284 {
285         struct beiscsi_hba *phba = data;
286         char *str = buf;
287         int rc;
288
289         switch (type) {
290         case ISCSI_BOOT_INI_INITIATOR_NAME:
291                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
292                 break;
293         default:
294                 rc = -ENOSYS;
295                 break;
296         }
297         return rc;
298 }
299
300 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
301 {
302         struct beiscsi_hba *phba = data;
303         char *str = buf;
304         int rc;
305
306         switch (type) {
307         case ISCSI_BOOT_ETH_FLAGS:
308                 rc = sprintf(str, "2\n");
309                 break;
310         case ISCSI_BOOT_ETH_INDEX:
311                 rc = sprintf(str, "0\n");
312                 break;
313         case ISCSI_BOOT_ETH_MAC:
314                 rc  = beiscsi_get_macaddr(buf, phba);
315                 if (rc < 0) {
316                         SE_DEBUG(DBG_LVL_1, "beiscsi_get_macaddr Failed\n");
317                         return rc;
318                 }
319         break;
320         default:
321                 rc = -ENOSYS;
322                 break;
323         }
324         return rc;
325 }
326
327
328 static mode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
329 {
330         int rc;
331
332         switch (type) {
333         case ISCSI_BOOT_TGT_NAME:
334         case ISCSI_BOOT_TGT_IP_ADDR:
335         case ISCSI_BOOT_TGT_PORT:
336         case ISCSI_BOOT_TGT_CHAP_NAME:
337         case ISCSI_BOOT_TGT_CHAP_SECRET:
338         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
339         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
340         case ISCSI_BOOT_TGT_NIC_ASSOC:
341         case ISCSI_BOOT_TGT_FLAGS:
342                 rc = S_IRUGO;
343                 break;
344         default:
345                 rc = 0;
346                 break;
347         }
348         return rc;
349 }
350
351 static mode_t beiscsi_ini_get_attr_visibility(void *data, int type)
352 {
353         int rc;
354
355         switch (type) {
356         case ISCSI_BOOT_INI_INITIATOR_NAME:
357                 rc = S_IRUGO;
358                 break;
359         default:
360                 rc = 0;
361                 break;
362         }
363         return rc;
364 }
365
366
367 static mode_t beiscsi_eth_get_attr_visibility(void *data, int type)
368 {
369         int rc;
370
371         switch (type) {
372         case ISCSI_BOOT_ETH_FLAGS:
373         case ISCSI_BOOT_ETH_MAC:
374         case ISCSI_BOOT_ETH_INDEX:
375                 rc = S_IRUGO;
376                 break;
377         default:
378                 rc = 0;
379                 break;
380         }
381         return rc;
382 }
383
384 /*------------------- PCI Driver operations and data ----------------- */
385 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
386         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
387         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
388         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
389         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
390         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
391         { 0 }
392 };
393 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
394
395 static struct scsi_host_template beiscsi_sht = {
396         .module = THIS_MODULE,
397         .name = "ServerEngines 10Gbe open-iscsi Initiator Driver",
398         .proc_name = DRV_NAME,
399         .queuecommand = iscsi_queuecommand,
400         .change_queue_depth = iscsi_change_queue_depth,
401         .slave_configure = beiscsi_slave_configure,
402         .target_alloc = iscsi_target_alloc,
403         .eh_abort_handler = beiscsi_eh_abort,
404         .eh_device_reset_handler = beiscsi_eh_device_reset,
405         .eh_target_reset_handler = iscsi_eh_session_reset,
406         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
407         .can_queue = BE2_IO_DEPTH,
408         .this_id = -1,
409         .max_sectors = BEISCSI_MAX_SECTORS,
410         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
411         .use_clustering = ENABLE_CLUSTERING,
412 };
413
414 static struct scsi_transport_template *beiscsi_scsi_transport;
415
416 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
417 {
418         struct beiscsi_hba *phba;
419         struct Scsi_Host *shost;
420
421         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
422         if (!shost) {
423                 dev_err(&pcidev->dev, "beiscsi_hba_alloc -"
424                         "iscsi_host_alloc failed\n");
425                 return NULL;
426         }
427         shost->dma_boundary = pcidev->dma_mask;
428         shost->max_id = BE2_MAX_SESSIONS;
429         shost->max_channel = 0;
430         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
431         shost->max_lun = BEISCSI_NUM_MAX_LUN;
432         shost->transportt = beiscsi_scsi_transport;
433         phba = iscsi_host_priv(shost);
434         memset(phba, 0, sizeof(*phba));
435         phba->shost = shost;
436         phba->pcidev = pci_dev_get(pcidev);
437         pci_set_drvdata(pcidev, phba);
438
439         if (iscsi_host_add(shost, &phba->pcidev->dev))
440                 goto free_devices;
441
442         return phba;
443
444 free_devices:
445         pci_dev_put(phba->pcidev);
446         iscsi_host_free(phba->shost);
447         return NULL;
448 }
449
450 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
451 {
452         if (phba->csr_va) {
453                 iounmap(phba->csr_va);
454                 phba->csr_va = NULL;
455         }
456         if (phba->db_va) {
457                 iounmap(phba->db_va);
458                 phba->db_va = NULL;
459         }
460         if (phba->pci_va) {
461                 iounmap(phba->pci_va);
462                 phba->pci_va = NULL;
463         }
464 }
465
466 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
467                                 struct pci_dev *pcidev)
468 {
469         u8 __iomem *addr;
470         int pcicfg_reg;
471
472         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
473                                pci_resource_len(pcidev, 2));
474         if (addr == NULL)
475                 return -ENOMEM;
476         phba->ctrl.csr = addr;
477         phba->csr_va = addr;
478         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
479
480         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
481         if (addr == NULL)
482                 goto pci_map_err;
483         phba->ctrl.db = addr;
484         phba->db_va = addr;
485         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
486
487         if (phba->generation == BE_GEN2)
488                 pcicfg_reg = 1;
489         else
490                 pcicfg_reg = 0;
491
492         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
493                                pci_resource_len(pcidev, pcicfg_reg));
494
495         if (addr == NULL)
496                 goto pci_map_err;
497         phba->ctrl.pcicfg = addr;
498         phba->pci_va = addr;
499         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
500         return 0;
501
502 pci_map_err:
503         beiscsi_unmap_pci_function(phba);
504         return -ENOMEM;
505 }
506
507 static int beiscsi_enable_pci(struct pci_dev *pcidev)
508 {
509         int ret;
510
511         ret = pci_enable_device(pcidev);
512         if (ret) {
513                 dev_err(&pcidev->dev, "beiscsi_enable_pci - enable device "
514                         "failed. Returning -ENODEV\n");
515                 return ret;
516         }
517
518         pci_set_master(pcidev);
519         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
520                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
521                 if (ret) {
522                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
523                         pci_disable_device(pcidev);
524                         return ret;
525                 }
526         }
527         return 0;
528 }
529
530 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
531 {
532         struct be_ctrl_info *ctrl = &phba->ctrl;
533         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
534         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
535         int status = 0;
536
537         ctrl->pdev = pdev;
538         status = beiscsi_map_pci_bars(phba, pdev);
539         if (status)
540                 return status;
541         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
542         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
543                                                   mbox_mem_alloc->size,
544                                                   &mbox_mem_alloc->dma);
545         if (!mbox_mem_alloc->va) {
546                 beiscsi_unmap_pci_function(phba);
547                 status = -ENOMEM;
548                 return status;
549         }
550
551         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
552         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
553         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
554         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
555         spin_lock_init(&ctrl->mbox_lock);
556         spin_lock_init(&phba->ctrl.mcc_lock);
557         spin_lock_init(&phba->ctrl.mcc_cq_lock);
558
559         return status;
560 }
561
562 static void beiscsi_get_params(struct beiscsi_hba *phba)
563 {
564         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
565                                     - (phba->fw_config.iscsi_cid_count
566                                     + BE2_TMFS
567                                     + BE2_NOPOUT_REQ));
568         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
569         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
570         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
571         phba->params.num_sge_per_io = BE2_SGE;
572         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
573         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
574         phba->params.eq_timer = 64;
575         phba->params.num_eq_entries =
576             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
577                                     + BE2_TMFS) / 512) + 1) * 512;
578         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
579                                 ? 1024 : phba->params.num_eq_entries;
580         SE_DEBUG(DBG_LVL_8, "phba->params.num_eq_entries=%d\n",
581                              phba->params.num_eq_entries);
582         phba->params.num_cq_entries =
583             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
584                                     + BE2_TMFS) / 512) + 1) * 512;
585         phba->params.wrbs_per_cxn = 256;
586 }
587
588 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
589                            unsigned int id, unsigned int clr_interrupt,
590                            unsigned int num_processed,
591                            unsigned char rearm, unsigned char event)
592 {
593         u32 val = 0;
594         val |= id & DB_EQ_RING_ID_MASK;
595         if (rearm)
596                 val |= 1 << DB_EQ_REARM_SHIFT;
597         if (clr_interrupt)
598                 val |= 1 << DB_EQ_CLR_SHIFT;
599         if (event)
600                 val |= 1 << DB_EQ_EVNT_SHIFT;
601         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
602         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
603 }
604
605 /**
606  * be_isr_mcc - The isr routine of the driver.
607  * @irq: Not used
608  * @dev_id: Pointer to host adapter structure
609  */
610 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
611 {
612         struct beiscsi_hba *phba;
613         struct be_eq_entry *eqe = NULL;
614         struct be_queue_info *eq;
615         struct be_queue_info *mcc;
616         unsigned int num_eq_processed;
617         struct be_eq_obj *pbe_eq;
618         unsigned long flags;
619
620         pbe_eq = dev_id;
621         eq = &pbe_eq->q;
622         phba =  pbe_eq->phba;
623         mcc = &phba->ctrl.mcc_obj.cq;
624         eqe = queue_tail_node(eq);
625         if (!eqe)
626                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
627
628         num_eq_processed = 0;
629
630         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
631                                 & EQE_VALID_MASK) {
632                 if (((eqe->dw[offsetof(struct amap_eq_entry,
633                      resource_id) / 32] &
634                      EQE_RESID_MASK) >> 16) == mcc->id) {
635                         spin_lock_irqsave(&phba->isr_lock, flags);
636                         phba->todo_mcc_cq = 1;
637                         spin_unlock_irqrestore(&phba->isr_lock, flags);
638                 }
639                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
640                 queue_tail_inc(eq);
641                 eqe = queue_tail_node(eq);
642                 num_eq_processed++;
643         }
644         if (phba->todo_mcc_cq)
645                 queue_work(phba->wq, &phba->work_cqs);
646         if (num_eq_processed)
647                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
648
649         return IRQ_HANDLED;
650 }
651
652 /**
653  * be_isr_msix - The isr routine of the driver.
654  * @irq: Not used
655  * @dev_id: Pointer to host adapter structure
656  */
657 static irqreturn_t be_isr_msix(int irq, void *dev_id)
658 {
659         struct beiscsi_hba *phba;
660         struct be_eq_entry *eqe = NULL;
661         struct be_queue_info *eq;
662         struct be_queue_info *cq;
663         unsigned int num_eq_processed;
664         struct be_eq_obj *pbe_eq;
665         unsigned long flags;
666
667         pbe_eq = dev_id;
668         eq = &pbe_eq->q;
669         cq = pbe_eq->cq;
670         eqe = queue_tail_node(eq);
671         if (!eqe)
672                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
673
674         phba = pbe_eq->phba;
675         num_eq_processed = 0;
676         if (blk_iopoll_enabled) {
677                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
678                                         & EQE_VALID_MASK) {
679                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
680                                 blk_iopoll_sched(&pbe_eq->iopoll);
681
682                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
683                         queue_tail_inc(eq);
684                         eqe = queue_tail_node(eq);
685                         num_eq_processed++;
686                 }
687                 if (num_eq_processed)
688                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
689
690                 return IRQ_HANDLED;
691         } else {
692                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
693                                                 & EQE_VALID_MASK) {
694                         spin_lock_irqsave(&phba->isr_lock, flags);
695                         phba->todo_cq = 1;
696                         spin_unlock_irqrestore(&phba->isr_lock, flags);
697                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
698                         queue_tail_inc(eq);
699                         eqe = queue_tail_node(eq);
700                         num_eq_processed++;
701                 }
702                 if (phba->todo_cq)
703                         queue_work(phba->wq, &phba->work_cqs);
704
705                 if (num_eq_processed)
706                         hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
707
708                 return IRQ_HANDLED;
709         }
710 }
711
712 /**
713  * be_isr - The isr routine of the driver.
714  * @irq: Not used
715  * @dev_id: Pointer to host adapter structure
716  */
717 static irqreturn_t be_isr(int irq, void *dev_id)
718 {
719         struct beiscsi_hba *phba;
720         struct hwi_controller *phwi_ctrlr;
721         struct hwi_context_memory *phwi_context;
722         struct be_eq_entry *eqe = NULL;
723         struct be_queue_info *eq;
724         struct be_queue_info *cq;
725         struct be_queue_info *mcc;
726         unsigned long flags, index;
727         unsigned int num_mcceq_processed, num_ioeq_processed;
728         struct be_ctrl_info *ctrl;
729         struct be_eq_obj *pbe_eq;
730         int isr;
731
732         phba = dev_id;
733         ctrl = &phba->ctrl;
734         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
735                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
736         if (!isr)
737                 return IRQ_NONE;
738
739         phwi_ctrlr = phba->phwi_ctrlr;
740         phwi_context = phwi_ctrlr->phwi_ctxt;
741         pbe_eq = &phwi_context->be_eq[0];
742
743         eq = &phwi_context->be_eq[0].q;
744         mcc = &phba->ctrl.mcc_obj.cq;
745         index = 0;
746         eqe = queue_tail_node(eq);
747         if (!eqe)
748                 SE_DEBUG(DBG_LVL_1, "eqe is NULL\n");
749
750         num_ioeq_processed = 0;
751         num_mcceq_processed = 0;
752         if (blk_iopoll_enabled) {
753                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
754                                         & EQE_VALID_MASK) {
755                         if (((eqe->dw[offsetof(struct amap_eq_entry,
756                              resource_id) / 32] &
757                              EQE_RESID_MASK) >> 16) == mcc->id) {
758                                 spin_lock_irqsave(&phba->isr_lock, flags);
759                                 phba->todo_mcc_cq = 1;
760                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
761                                 num_mcceq_processed++;
762                         } else {
763                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
764                                         blk_iopoll_sched(&pbe_eq->iopoll);
765                                 num_ioeq_processed++;
766                         }
767                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
768                         queue_tail_inc(eq);
769                         eqe = queue_tail_node(eq);
770                 }
771                 if (num_ioeq_processed || num_mcceq_processed) {
772                         if (phba->todo_mcc_cq)
773                                 queue_work(phba->wq, &phba->work_cqs);
774
775                         if ((num_mcceq_processed) && (!num_ioeq_processed))
776                                 hwi_ring_eq_db(phba, eq->id, 0,
777                                               (num_ioeq_processed +
778                                                num_mcceq_processed) , 1, 1);
779                         else
780                                 hwi_ring_eq_db(phba, eq->id, 0,
781                                                (num_ioeq_processed +
782                                                 num_mcceq_processed), 0, 1);
783
784                         return IRQ_HANDLED;
785                 } else
786                         return IRQ_NONE;
787         } else {
788                 cq = &phwi_context->be_cq[0];
789                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
790                                                 & EQE_VALID_MASK) {
791
792                         if (((eqe->dw[offsetof(struct amap_eq_entry,
793                              resource_id) / 32] &
794                              EQE_RESID_MASK) >> 16) != cq->id) {
795                                 spin_lock_irqsave(&phba->isr_lock, flags);
796                                 phba->todo_mcc_cq = 1;
797                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
798                         } else {
799                                 spin_lock_irqsave(&phba->isr_lock, flags);
800                                 phba->todo_cq = 1;
801                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
802                         }
803                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
804                         queue_tail_inc(eq);
805                         eqe = queue_tail_node(eq);
806                         num_ioeq_processed++;
807                 }
808                 if (phba->todo_cq || phba->todo_mcc_cq)
809                         queue_work(phba->wq, &phba->work_cqs);
810
811                 if (num_ioeq_processed) {
812                         hwi_ring_eq_db(phba, eq->id, 0,
813                                        num_ioeq_processed, 1, 1);
814                         return IRQ_HANDLED;
815                 } else
816                         return IRQ_NONE;
817         }
818 }
819
820 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
821 {
822         struct pci_dev *pcidev = phba->pcidev;
823         struct hwi_controller *phwi_ctrlr;
824         struct hwi_context_memory *phwi_context;
825         int ret, msix_vec, i, j;
826
827         phwi_ctrlr = phba->phwi_ctrlr;
828         phwi_context = phwi_ctrlr->phwi_ctxt;
829
830         if (phba->msix_enabled) {
831                 for (i = 0; i < phba->num_cpus; i++) {
832                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
833                                                     GFP_KERNEL);
834                         if (!phba->msi_name[i]) {
835                                 ret = -ENOMEM;
836                                 goto free_msix_irqs;
837                         }
838
839                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
840                                 phba->shost->host_no, i);
841                         msix_vec = phba->msix_entries[i].vector;
842                         ret = request_irq(msix_vec, be_isr_msix, 0,
843                                           phba->msi_name[i],
844                                           &phwi_context->be_eq[i]);
845                         if (ret) {
846                                 shost_printk(KERN_ERR, phba->shost,
847                                              "beiscsi_init_irqs-Failed to"
848                                              "register msix for i = %d\n", i);
849                                 kfree(phba->msi_name[i]);
850                                 goto free_msix_irqs;
851                         }
852                 }
853                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
854                 if (!phba->msi_name[i]) {
855                         ret = -ENOMEM;
856                         goto free_msix_irqs;
857                 }
858                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
859                         phba->shost->host_no);
860                 msix_vec = phba->msix_entries[i].vector;
861                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
862                                   &phwi_context->be_eq[i]);
863                 if (ret) {
864                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
865                                      "Failed to register beiscsi_msix_mcc\n");
866                         kfree(phba->msi_name[i]);
867                         goto free_msix_irqs;
868                 }
869
870         } else {
871                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
872                                   "beiscsi", phba);
873                 if (ret) {
874                         shost_printk(KERN_ERR, phba->shost, "beiscsi_init_irqs-"
875                                      "Failed to register irq\\n");
876                         return ret;
877                 }
878         }
879         return 0;
880 free_msix_irqs:
881         for (j = i - 1; j >= 0; j--) {
882                 kfree(phba->msi_name[j]);
883                 msix_vec = phba->msix_entries[j].vector;
884                 free_irq(msix_vec, &phwi_context->be_eq[j]);
885         }
886         return ret;
887 }
888
889 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
890                            unsigned int id, unsigned int num_processed,
891                            unsigned char rearm, unsigned char event)
892 {
893         u32 val = 0;
894         val |= id & DB_CQ_RING_ID_MASK;
895         if (rearm)
896                 val |= 1 << DB_CQ_REARM_SHIFT;
897         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
898         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
899 }
900
901 static unsigned int
902 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
903                           struct beiscsi_hba *phba,
904                           unsigned short cid,
905                           struct pdu_base *ppdu,
906                           unsigned long pdu_len,
907                           void *pbuffer, unsigned long buf_len)
908 {
909         struct iscsi_conn *conn = beiscsi_conn->conn;
910         struct iscsi_session *session = conn->session;
911         struct iscsi_task *task;
912         struct beiscsi_io_task *io_task;
913         struct iscsi_hdr *login_hdr;
914
915         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
916                                                 PDUBASE_OPCODE_MASK) {
917         case ISCSI_OP_NOOP_IN:
918                 pbuffer = NULL;
919                 buf_len = 0;
920                 break;
921         case ISCSI_OP_ASYNC_EVENT:
922                 break;
923         case ISCSI_OP_REJECT:
924                 WARN_ON(!pbuffer);
925                 WARN_ON(!(buf_len == 48));
926                 SE_DEBUG(DBG_LVL_1, "In ISCSI_OP_REJECT\n");
927                 break;
928         case ISCSI_OP_LOGIN_RSP:
929         case ISCSI_OP_TEXT_RSP:
930                 task = conn->login_task;
931                 io_task = task->dd_data;
932                 login_hdr = (struct iscsi_hdr *)ppdu;
933                 login_hdr->itt = io_task->libiscsi_itt;
934                 break;
935         default:
936                 shost_printk(KERN_WARNING, phba->shost,
937                              "Unrecognized opcode 0x%x in async msg\n",
938                              (ppdu->
939                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
940                                                 & PDUBASE_OPCODE_MASK));
941                 return 1;
942         }
943
944         spin_lock_bh(&session->lock);
945         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
946         spin_unlock_bh(&session->lock);
947         return 0;
948 }
949
950 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
951 {
952         struct sgl_handle *psgl_handle;
953
954         if (phba->io_sgl_hndl_avbl) {
955                 SE_DEBUG(DBG_LVL_8,
956                          "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
957                          phba->io_sgl_alloc_index);
958                 psgl_handle = phba->io_sgl_hndl_base[phba->
959                                                 io_sgl_alloc_index];
960                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
961                 phba->io_sgl_hndl_avbl--;
962                 if (phba->io_sgl_alloc_index == (phba->params.
963                                                  ios_per_ctrl - 1))
964                         phba->io_sgl_alloc_index = 0;
965                 else
966                         phba->io_sgl_alloc_index++;
967         } else
968                 psgl_handle = NULL;
969         return psgl_handle;
970 }
971
972 static void
973 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
974 {
975         SE_DEBUG(DBG_LVL_8, "In free_,io_sgl_free_index=%d\n",
976                  phba->io_sgl_free_index);
977         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
978                 /*
979                  * this can happen if clean_task is called on a task that
980                  * failed in xmit_task or alloc_pdu.
981                  */
982                  SE_DEBUG(DBG_LVL_8,
983                          "Double Free in IO SGL io_sgl_free_index=%d,"
984                          "value there=%p\n", phba->io_sgl_free_index,
985                          phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
986                 return;
987         }
988         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
989         phba->io_sgl_hndl_avbl++;
990         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
991                 phba->io_sgl_free_index = 0;
992         else
993                 phba->io_sgl_free_index++;
994 }
995
996 /**
997  * alloc_wrb_handle - To allocate a wrb handle
998  * @phba: The hba pointer
999  * @cid: The cid to use for allocation
1000  *
1001  * This happens under session_lock until submission to chip
1002  */
1003 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1004 {
1005         struct hwi_wrb_context *pwrb_context;
1006         struct hwi_controller *phwi_ctrlr;
1007         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1008
1009         phwi_ctrlr = phba->phwi_ctrlr;
1010         pwrb_context = &phwi_ctrlr->wrb_context[cid];
1011         if (pwrb_context->wrb_handles_available >= 2) {
1012                 pwrb_handle = pwrb_context->pwrb_handle_base[
1013                                             pwrb_context->alloc_index];
1014                 pwrb_context->wrb_handles_available--;
1015                 if (pwrb_context->alloc_index ==
1016                                                 (phba->params.wrbs_per_cxn - 1))
1017                         pwrb_context->alloc_index = 0;
1018                 else
1019                         pwrb_context->alloc_index++;
1020                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1021                                                 pwrb_context->alloc_index];
1022                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1023         } else
1024                 pwrb_handle = NULL;
1025         return pwrb_handle;
1026 }
1027
1028 /**
1029  * free_wrb_handle - To free the wrb handle back to pool
1030  * @phba: The hba pointer
1031  * @pwrb_context: The context to free from
1032  * @pwrb_handle: The wrb_handle to free
1033  *
1034  * This happens under session_lock until submission to chip
1035  */
1036 static void
1037 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1038                 struct wrb_handle *pwrb_handle)
1039 {
1040         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1041         pwrb_context->wrb_handles_available++;
1042         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1043                 pwrb_context->free_index = 0;
1044         else
1045                 pwrb_context->free_index++;
1046
1047         SE_DEBUG(DBG_LVL_8,
1048                  "FREE WRB: pwrb_handle=%p free_index=0x%x"
1049                  "wrb_handles_available=%d\n",
1050                  pwrb_handle, pwrb_context->free_index,
1051                  pwrb_context->wrb_handles_available);
1052 }
1053
1054 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1055 {
1056         struct sgl_handle *psgl_handle;
1057
1058         if (phba->eh_sgl_hndl_avbl) {
1059                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1060                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1061                 SE_DEBUG(DBG_LVL_8, "mgmt_sgl_alloc_index=%d=0x%x\n",
1062                          phba->eh_sgl_alloc_index, phba->eh_sgl_alloc_index);
1063                 phba->eh_sgl_hndl_avbl--;
1064                 if (phba->eh_sgl_alloc_index ==
1065                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1066                      1))
1067                         phba->eh_sgl_alloc_index = 0;
1068                 else
1069                         phba->eh_sgl_alloc_index++;
1070         } else
1071                 psgl_handle = NULL;
1072         return psgl_handle;
1073 }
1074
1075 void
1076 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1077 {
1078
1079         SE_DEBUG(DBG_LVL_8, "In  free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
1080                              phba->eh_sgl_free_index);
1081         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1082                 /*
1083                  * this can happen if clean_task is called on a task that
1084                  * failed in xmit_task or alloc_pdu.
1085                  */
1086                 SE_DEBUG(DBG_LVL_8,
1087                          "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
1088                          phba->eh_sgl_free_index);
1089                 return;
1090         }
1091         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1092         phba->eh_sgl_hndl_avbl++;
1093         if (phba->eh_sgl_free_index ==
1094             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1095                 phba->eh_sgl_free_index = 0;
1096         else
1097                 phba->eh_sgl_free_index++;
1098 }
1099
1100 static void
1101 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1102                struct iscsi_task *task, struct sol_cqe *psol)
1103 {
1104         struct beiscsi_io_task *io_task = task->dd_data;
1105         struct be_status_bhs *sts_bhs =
1106                                 (struct be_status_bhs *)io_task->cmd_bhs;
1107         struct iscsi_conn *conn = beiscsi_conn->conn;
1108         unsigned char *sense;
1109         u32 resid = 0, exp_cmdsn, max_cmdsn;
1110         u8 rsp, status, flags;
1111
1112         exp_cmdsn = (psol->
1113                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1114                         & SOL_EXP_CMD_SN_MASK);
1115         max_cmdsn = ((psol->
1116                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1117                         & SOL_EXP_CMD_SN_MASK) +
1118                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1119                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1120         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1121                                                 & SOL_RESP_MASK) >> 16);
1122         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1123                                                 & SOL_STS_MASK) >> 8);
1124         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1125                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1126         if (!task->sc) {
1127                 if (io_task->scsi_cmnd)
1128                         scsi_dma_unmap(io_task->scsi_cmnd);
1129
1130                 return;
1131         }
1132         task->sc->result = (DID_OK << 16) | status;
1133         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1134                 task->sc->result = DID_ERROR << 16;
1135                 goto unmap;
1136         }
1137
1138         /* bidi not initially supported */
1139         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1140                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1141                                 32] & SOL_RES_CNT_MASK);
1142
1143                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1144                         task->sc->result = DID_ERROR << 16;
1145
1146                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1147                         scsi_set_resid(task->sc, resid);
1148                         if (!status && (scsi_bufflen(task->sc) - resid <
1149                             task->sc->underflow))
1150                                 task->sc->result = DID_ERROR << 16;
1151                 }
1152         }
1153
1154         if (status == SAM_STAT_CHECK_CONDITION) {
1155                 u16 sense_len;
1156                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1157
1158                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1159                 sense_len = be16_to_cpu(*slen);
1160                 memcpy(task->sc->sense_buffer, sense,
1161                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1162         }
1163
1164         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1165                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1166                                                         & SOL_RES_CNT_MASK)
1167                          conn->rxdata_octets += (psol->
1168                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1169                              & SOL_RES_CNT_MASK);
1170         }
1171 unmap:
1172         scsi_dma_unmap(io_task->scsi_cmnd);
1173         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1174 }
1175
1176 static void
1177 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1178                    struct iscsi_task *task, struct sol_cqe *psol)
1179 {
1180         struct iscsi_logout_rsp *hdr;
1181         struct beiscsi_io_task *io_task = task->dd_data;
1182         struct iscsi_conn *conn = beiscsi_conn->conn;
1183
1184         hdr = (struct iscsi_logout_rsp *)task->hdr;
1185         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1186         hdr->t2wait = 5;
1187         hdr->t2retain = 0;
1188         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1189                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1190         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1191                                         32] & SOL_RESP_MASK);
1192         hdr->exp_cmdsn = cpu_to_be32(psol->
1193                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1194                                         & SOL_EXP_CMD_SN_MASK);
1195         hdr->max_cmdsn = be32_to_cpu((psol->
1196                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1197                                         & SOL_EXP_CMD_SN_MASK) +
1198                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1199                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1200         hdr->dlength[0] = 0;
1201         hdr->dlength[1] = 0;
1202         hdr->dlength[2] = 0;
1203         hdr->hlength = 0;
1204         hdr->itt = io_task->libiscsi_itt;
1205         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1206 }
1207
1208 static void
1209 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1210                 struct iscsi_task *task, struct sol_cqe *psol)
1211 {
1212         struct iscsi_tm_rsp *hdr;
1213         struct iscsi_conn *conn = beiscsi_conn->conn;
1214         struct beiscsi_io_task *io_task = task->dd_data;
1215
1216         hdr = (struct iscsi_tm_rsp *)task->hdr;
1217         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1218         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1219                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1220         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1221                                         32] & SOL_RESP_MASK);
1222         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1223                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1224         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1225                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1226                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1227                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1228         hdr->itt = io_task->libiscsi_itt;
1229         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1230 }
1231
1232 static void
1233 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1234                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1235 {
1236         struct hwi_wrb_context *pwrb_context;
1237         struct wrb_handle *pwrb_handle = NULL;
1238         struct hwi_controller *phwi_ctrlr;
1239         struct iscsi_task *task;
1240         struct beiscsi_io_task *io_task;
1241         struct iscsi_conn *conn = beiscsi_conn->conn;
1242         struct iscsi_session *session = conn->session;
1243
1244         phwi_ctrlr = phba->phwi_ctrlr;
1245         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
1246                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1247                                 SOL_CID_MASK) >> 6) -
1248                                 phba->fw_config.iscsi_cid_start];
1249         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1250                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1251                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1252         task = pwrb_handle->pio_handle;
1253
1254         io_task = task->dd_data;
1255         spin_lock(&phba->mgmt_sgl_lock);
1256         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1257         spin_unlock(&phba->mgmt_sgl_lock);
1258         spin_lock_bh(&session->lock);
1259         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1260         spin_unlock_bh(&session->lock);
1261 }
1262
1263 static void
1264 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1265                        struct iscsi_task *task, struct sol_cqe *psol)
1266 {
1267         struct iscsi_nopin *hdr;
1268         struct iscsi_conn *conn = beiscsi_conn->conn;
1269         struct beiscsi_io_task *io_task = task->dd_data;
1270
1271         hdr = (struct iscsi_nopin *)task->hdr;
1272         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1273                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1274         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1275                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1276         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1277                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1278                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1279                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1280         hdr->opcode = ISCSI_OP_NOOP_IN;
1281         hdr->itt = io_task->libiscsi_itt;
1282         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1283 }
1284
1285 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1286                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1287 {
1288         struct hwi_wrb_context *pwrb_context;
1289         struct wrb_handle *pwrb_handle;
1290         struct iscsi_wrb *pwrb = NULL;
1291         struct hwi_controller *phwi_ctrlr;
1292         struct iscsi_task *task;
1293         unsigned int type;
1294         struct iscsi_conn *conn = beiscsi_conn->conn;
1295         struct iscsi_session *session = conn->session;
1296
1297         phwi_ctrlr = phba->phwi_ctrlr;
1298         pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1299                                 (struct amap_sol_cqe, cid) / 32]
1300                                 & SOL_CID_MASK) >> 6) -
1301                                 phba->fw_config.iscsi_cid_start];
1302         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1303                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1304                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1305         task = pwrb_handle->pio_handle;
1306         pwrb = pwrb_handle->pwrb;
1307         type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1308                                  WRB_TYPE_MASK) >> 28;
1309
1310         spin_lock_bh(&session->lock);
1311         switch (type) {
1312         case HWH_TYPE_IO:
1313         case HWH_TYPE_IO_RD:
1314                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1315                      ISCSI_OP_NOOP_OUT)
1316                         be_complete_nopin_resp(beiscsi_conn, task, psol);
1317                 else
1318                         be_complete_io(beiscsi_conn, task, psol);
1319                 break;
1320
1321         case HWH_TYPE_LOGOUT:
1322                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1323                         be_complete_logout(beiscsi_conn, task, psol);
1324                 else
1325                         be_complete_tmf(beiscsi_conn, task, psol);
1326
1327                 break;
1328
1329         case HWH_TYPE_LOGIN:
1330                 SE_DEBUG(DBG_LVL_1,
1331                          "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1332                          "- Solicited path\n");
1333                 break;
1334
1335         case HWH_TYPE_NOP:
1336                 be_complete_nopin_resp(beiscsi_conn, task, psol);
1337                 break;
1338
1339         default:
1340                 shost_printk(KERN_WARNING, phba->shost,
1341                                 "In hwi_complete_cmd, unknown type = %d"
1342                                 "wrb_index 0x%x CID 0x%x\n", type,
1343                                 ((psol->dw[offsetof(struct amap_iscsi_wrb,
1344                                 type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1345                                 ((psol->dw[offsetof(struct amap_sol_cqe,
1346                                 cid) / 32] & SOL_CID_MASK) >> 6));
1347                 break;
1348         }
1349
1350         spin_unlock_bh(&session->lock);
1351 }
1352
1353 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1354                                           *pasync_ctx, unsigned int is_header,
1355                                           unsigned int host_write_ptr)
1356 {
1357         if (is_header)
1358                 return &pasync_ctx->async_entry[host_write_ptr].
1359                     header_busy_list;
1360         else
1361                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1362 }
1363
1364 static struct async_pdu_handle *
1365 hwi_get_async_handle(struct beiscsi_hba *phba,
1366                      struct beiscsi_conn *beiscsi_conn,
1367                      struct hwi_async_pdu_context *pasync_ctx,
1368                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1369 {
1370         struct be_bus_address phys_addr;
1371         struct list_head *pbusy_list;
1372         struct async_pdu_handle *pasync_handle = NULL;
1373         int buffer_len = 0;
1374         unsigned char buffer_index = -1;
1375         unsigned char is_header = 0;
1376
1377         phys_addr.u.a32.address_lo =
1378             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1379             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1380                                                 & PDUCQE_DPL_MASK) >> 16);
1381         phys_addr.u.a32.address_hi =
1382             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1383
1384         phys_addr.u.a64.address =
1385                         *((unsigned long long *)(&phys_addr.u.a64.address));
1386
1387         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1388                         & PDUCQE_CODE_MASK) {
1389         case UNSOL_HDR_NOTIFY:
1390                 is_header = 1;
1391
1392                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1393                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1394                         index) / 32] & PDUCQE_INDEX_MASK));
1395
1396                 buffer_len = (unsigned int)(phys_addr.u.a64.address -
1397                                 pasync_ctx->async_header.pa_base.u.a64.address);
1398
1399                 buffer_index = buffer_len /
1400                                 pasync_ctx->async_header.buffer_size;
1401
1402                 break;
1403         case UNSOL_DATA_NOTIFY:
1404                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1405                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1406                                         index) / 32] & PDUCQE_INDEX_MASK));
1407                 buffer_len = (unsigned long)(phys_addr.u.a64.address -
1408                                         pasync_ctx->async_data.pa_base.u.
1409                                         a64.address);
1410                 buffer_index = buffer_len / pasync_ctx->async_data.buffer_size;
1411                 break;
1412         default:
1413                 pbusy_list = NULL;
1414                 shost_printk(KERN_WARNING, phba->shost,
1415                         "Unexpected code=%d\n",
1416                          pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1417                                         code) / 32] & PDUCQE_CODE_MASK);
1418                 return NULL;
1419         }
1420
1421         WARN_ON(!(buffer_index <= pasync_ctx->async_data.num_entries));
1422         WARN_ON(list_empty(pbusy_list));
1423         list_for_each_entry(pasync_handle, pbusy_list, link) {
1424                 WARN_ON(pasync_handle->consumed);
1425                 if (pasync_handle->index == buffer_index)
1426                         break;
1427         }
1428
1429         WARN_ON(!pasync_handle);
1430
1431         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1432                                              phba->fw_config.iscsi_cid_start;
1433         pasync_handle->is_header = is_header;
1434         pasync_handle->buffer_len = ((pdpdu_cqe->
1435                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1436                         & PDUCQE_DPL_MASK) >> 16);
1437
1438         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1439                         index) / 32] & PDUCQE_INDEX_MASK);
1440         return pasync_handle;
1441 }
1442
1443 static unsigned int
1444 hwi_update_async_writables(struct hwi_async_pdu_context *pasync_ctx,
1445                            unsigned int is_header, unsigned int cq_index)
1446 {
1447         struct list_head *pbusy_list;
1448         struct async_pdu_handle *pasync_handle;
1449         unsigned int num_entries, writables = 0;
1450         unsigned int *pep_read_ptr, *pwritables;
1451
1452
1453         if (is_header) {
1454                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1455                 pwritables = &pasync_ctx->async_header.writables;
1456                 num_entries = pasync_ctx->async_header.num_entries;
1457         } else {
1458                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1459                 pwritables = &pasync_ctx->async_data.writables;
1460                 num_entries = pasync_ctx->async_data.num_entries;
1461         }
1462
1463         while ((*pep_read_ptr) != cq_index) {
1464                 (*pep_read_ptr)++;
1465                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1466
1467                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1468                                                      *pep_read_ptr);
1469                 if (writables == 0)
1470                         WARN_ON(list_empty(pbusy_list));
1471
1472                 if (!list_empty(pbusy_list)) {
1473                         pasync_handle = list_entry(pbusy_list->next,
1474                                                    struct async_pdu_handle,
1475                                                    link);
1476                         WARN_ON(!pasync_handle);
1477                         pasync_handle->consumed = 1;
1478                 }
1479
1480                 writables++;
1481         }
1482
1483         if (!writables) {
1484                 SE_DEBUG(DBG_LVL_1,
1485                          "Duplicate notification received - index 0x%x!!\n",
1486                          cq_index);
1487                 WARN_ON(1);
1488         }
1489
1490         *pwritables = *pwritables + writables;
1491         return 0;
1492 }
1493
1494 static unsigned int hwi_free_async_msg(struct beiscsi_hba *phba,
1495                                        unsigned int cri)
1496 {
1497         struct hwi_controller *phwi_ctrlr;
1498         struct hwi_async_pdu_context *pasync_ctx;
1499         struct async_pdu_handle *pasync_handle, *tmp_handle;
1500         struct list_head *plist;
1501         unsigned int i = 0;
1502
1503         phwi_ctrlr = phba->phwi_ctrlr;
1504         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1505
1506         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1507
1508         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1509                 list_del(&pasync_handle->link);
1510
1511                 if (i == 0) {
1512                         list_add_tail(&pasync_handle->link,
1513                                       &pasync_ctx->async_header.free_list);
1514                         pasync_ctx->async_header.free_entries++;
1515                         i++;
1516                 } else {
1517                         list_add_tail(&pasync_handle->link,
1518                                       &pasync_ctx->async_data.free_list);
1519                         pasync_ctx->async_data.free_entries++;
1520                         i++;
1521                 }
1522         }
1523
1524         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1525         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1526         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1527         return 0;
1528 }
1529
1530 static struct phys_addr *
1531 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1532                      unsigned int is_header, unsigned int host_write_ptr)
1533 {
1534         struct phys_addr *pasync_sge = NULL;
1535
1536         if (is_header)
1537                 pasync_sge = pasync_ctx->async_header.ring_base;
1538         else
1539                 pasync_sge = pasync_ctx->async_data.ring_base;
1540
1541         return pasync_sge + host_write_ptr;
1542 }
1543
1544 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1545                                    unsigned int is_header)
1546 {
1547         struct hwi_controller *phwi_ctrlr;
1548         struct hwi_async_pdu_context *pasync_ctx;
1549         struct async_pdu_handle *pasync_handle;
1550         struct list_head *pfree_link, *pbusy_list;
1551         struct phys_addr *pasync_sge;
1552         unsigned int ring_id, num_entries;
1553         unsigned int host_write_num;
1554         unsigned int writables;
1555         unsigned int i = 0;
1556         u32 doorbell = 0;
1557
1558         phwi_ctrlr = phba->phwi_ctrlr;
1559         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1560
1561         if (is_header) {
1562                 num_entries = pasync_ctx->async_header.num_entries;
1563                 writables = min(pasync_ctx->async_header.writables,
1564                                 pasync_ctx->async_header.free_entries);
1565                 pfree_link = pasync_ctx->async_header.free_list.next;
1566                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1567                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1568         } else {
1569                 num_entries = pasync_ctx->async_data.num_entries;
1570                 writables = min(pasync_ctx->async_data.writables,
1571                                 pasync_ctx->async_data.free_entries);
1572                 pfree_link = pasync_ctx->async_data.free_list.next;
1573                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1574                 ring_id = phwi_ctrlr->default_pdu_data.id;
1575         }
1576
1577         writables = (writables / 8) * 8;
1578         if (writables) {
1579                 for (i = 0; i < writables; i++) {
1580                         pbusy_list =
1581                             hwi_get_async_busy_list(pasync_ctx, is_header,
1582                                                     host_write_num);
1583                         pasync_handle =
1584                             list_entry(pfree_link, struct async_pdu_handle,
1585                                                                 link);
1586                         WARN_ON(!pasync_handle);
1587                         pasync_handle->consumed = 0;
1588
1589                         pfree_link = pfree_link->next;
1590
1591                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1592                                                 is_header, host_write_num);
1593
1594                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1595                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1596
1597                         list_move(&pasync_handle->link, pbusy_list);
1598
1599                         host_write_num++;
1600                         host_write_num = host_write_num % num_entries;
1601                 }
1602
1603                 if (is_header) {
1604                         pasync_ctx->async_header.host_write_ptr =
1605                                                         host_write_num;
1606                         pasync_ctx->async_header.free_entries -= writables;
1607                         pasync_ctx->async_header.writables -= writables;
1608                         pasync_ctx->async_header.busy_entries += writables;
1609                 } else {
1610                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1611                         pasync_ctx->async_data.free_entries -= writables;
1612                         pasync_ctx->async_data.writables -= writables;
1613                         pasync_ctx->async_data.busy_entries += writables;
1614                 }
1615
1616                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1617                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1618                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1619                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1620                                         << DB_DEF_PDU_CQPROC_SHIFT;
1621
1622                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1623         }
1624 }
1625
1626 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1627                                          struct beiscsi_conn *beiscsi_conn,
1628                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1629 {
1630         struct hwi_controller *phwi_ctrlr;
1631         struct hwi_async_pdu_context *pasync_ctx;
1632         struct async_pdu_handle *pasync_handle = NULL;
1633         unsigned int cq_index = -1;
1634
1635         phwi_ctrlr = phba->phwi_ctrlr;
1636         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1637
1638         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1639                                              pdpdu_cqe, &cq_index);
1640         BUG_ON(pasync_handle->is_header != 0);
1641         if (pasync_handle->consumed == 0)
1642                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1643                                            cq_index);
1644
1645         hwi_free_async_msg(phba, pasync_handle->cri);
1646         hwi_post_async_buffers(phba, pasync_handle->is_header);
1647 }
1648
1649 static unsigned int
1650 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1651                   struct beiscsi_hba *phba,
1652                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1653 {
1654         struct list_head *plist;
1655         struct async_pdu_handle *pasync_handle;
1656         void *phdr = NULL;
1657         unsigned int hdr_len = 0, buf_len = 0;
1658         unsigned int status, index = 0, offset = 0;
1659         void *pfirst_buffer = NULL;
1660         unsigned int num_buf = 0;
1661
1662         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1663
1664         list_for_each_entry(pasync_handle, plist, link) {
1665                 if (index == 0) {
1666                         phdr = pasync_handle->pbuffer;
1667                         hdr_len = pasync_handle->buffer_len;
1668                 } else {
1669                         buf_len = pasync_handle->buffer_len;
1670                         if (!num_buf) {
1671                                 pfirst_buffer = pasync_handle->pbuffer;
1672                                 num_buf++;
1673                         }
1674                         memcpy(pfirst_buffer + offset,
1675                                pasync_handle->pbuffer, buf_len);
1676                         offset = buf_len;
1677                 }
1678                 index++;
1679         }
1680
1681         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1682                                            (beiscsi_conn->beiscsi_conn_cid -
1683                                             phba->fw_config.iscsi_cid_start),
1684                                             phdr, hdr_len, pfirst_buffer,
1685                                             buf_len);
1686
1687         if (status == 0)
1688                 hwi_free_async_msg(phba, cri);
1689         return 0;
1690 }
1691
1692 static unsigned int
1693 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1694                      struct beiscsi_hba *phba,
1695                      struct async_pdu_handle *pasync_handle)
1696 {
1697         struct hwi_async_pdu_context *pasync_ctx;
1698         struct hwi_controller *phwi_ctrlr;
1699         unsigned int bytes_needed = 0, status = 0;
1700         unsigned short cri = pasync_handle->cri;
1701         struct pdu_base *ppdu;
1702
1703         phwi_ctrlr = phba->phwi_ctrlr;
1704         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1705
1706         list_del(&pasync_handle->link);
1707         if (pasync_handle->is_header) {
1708                 pasync_ctx->async_header.busy_entries--;
1709                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1710                         hwi_free_async_msg(phba, cri);
1711                         BUG();
1712                 }
1713
1714                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1715                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1716                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1717                                 (unsigned short)pasync_handle->buffer_len;
1718                 list_add_tail(&pasync_handle->link,
1719                               &pasync_ctx->async_entry[cri].wait_queue.list);
1720
1721                 ppdu = pasync_handle->pbuffer;
1722                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1723                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1724                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1725                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1726                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1727
1728                 if (status == 0) {
1729                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1730                             bytes_needed;
1731
1732                         if (bytes_needed == 0)
1733                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1734                                                            pasync_ctx, cri);
1735                 }
1736         } else {
1737                 pasync_ctx->async_data.busy_entries--;
1738                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1739                         list_add_tail(&pasync_handle->link,
1740                                       &pasync_ctx->async_entry[cri].wait_queue.
1741                                       list);
1742                         pasync_ctx->async_entry[cri].wait_queue.
1743                                 bytes_received +=
1744                                 (unsigned short)pasync_handle->buffer_len;
1745
1746                         if (pasync_ctx->async_entry[cri].wait_queue.
1747                             bytes_received >=
1748                             pasync_ctx->async_entry[cri].wait_queue.
1749                             bytes_needed)
1750                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1751                                                            pasync_ctx, cri);
1752                 }
1753         }
1754         return status;
1755 }
1756
1757 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1758                                          struct beiscsi_hba *phba,
1759                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1760 {
1761         struct hwi_controller *phwi_ctrlr;
1762         struct hwi_async_pdu_context *pasync_ctx;
1763         struct async_pdu_handle *pasync_handle = NULL;
1764         unsigned int cq_index = -1;
1765
1766         phwi_ctrlr = phba->phwi_ctrlr;
1767         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1768         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1769                                              pdpdu_cqe, &cq_index);
1770
1771         if (pasync_handle->consumed == 0)
1772                 hwi_update_async_writables(pasync_ctx, pasync_handle->is_header,
1773                                            cq_index);
1774         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1775         hwi_post_async_buffers(phba, pasync_handle->is_header);
1776 }
1777
1778 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1779 {
1780         struct be_queue_info *mcc_cq;
1781         struct  be_mcc_compl *mcc_compl;
1782         unsigned int num_processed = 0;
1783
1784         mcc_cq = &phba->ctrl.mcc_obj.cq;
1785         mcc_compl = queue_tail_node(mcc_cq);
1786         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1787         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1788
1789                 if (num_processed >= 32) {
1790                         hwi_ring_cq_db(phba, mcc_cq->id,
1791                                         num_processed, 0, 0);
1792                         num_processed = 0;
1793                 }
1794                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1795                         /* Interpret flags as an async trailer */
1796                         if (is_link_state_evt(mcc_compl->flags))
1797                                 /* Interpret compl as a async link evt */
1798                                 beiscsi_async_link_state_process(phba,
1799                                 (struct be_async_event_link_state *) mcc_compl);
1800                         else
1801                                 SE_DEBUG(DBG_LVL_1,
1802                                         " Unsupported Async Event, flags"
1803                                         " = 0x%08x\n", mcc_compl->flags);
1804                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1805                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1806                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1807                 }
1808
1809                 mcc_compl->flags = 0;
1810                 queue_tail_inc(mcc_cq);
1811                 mcc_compl = queue_tail_node(mcc_cq);
1812                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1813                 num_processed++;
1814         }
1815
1816         if (num_processed > 0)
1817                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1818
1819 }
1820
1821 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1822 {
1823         struct be_queue_info *cq;
1824         struct sol_cqe *sol;
1825         struct dmsg_cqe *dmsg;
1826         unsigned int num_processed = 0;
1827         unsigned int tot_nump = 0;
1828         struct beiscsi_conn *beiscsi_conn;
1829         struct beiscsi_endpoint *beiscsi_ep;
1830         struct iscsi_endpoint *ep;
1831         struct beiscsi_hba *phba;
1832
1833         cq = pbe_eq->cq;
1834         sol = queue_tail_node(cq);
1835         phba = pbe_eq->phba;
1836
1837         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1838                CQE_VALID_MASK) {
1839                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1840
1841                 ep = phba->ep_array[(u32) ((sol->
1842                                    dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1843                                    SOL_CID_MASK) >> 6) -
1844                                    phba->fw_config.iscsi_cid_start];
1845
1846                 beiscsi_ep = ep->dd_data;
1847                 beiscsi_conn = beiscsi_ep->conn;
1848
1849                 if (num_processed >= 32) {
1850                         hwi_ring_cq_db(phba, cq->id,
1851                                         num_processed, 0, 0);
1852                         tot_nump += num_processed;
1853                         num_processed = 0;
1854                 }
1855
1856                 switch ((u32) sol->dw[offsetof(struct amap_sol_cqe, code) /
1857                         32] & CQE_CODE_MASK) {
1858                 case SOL_CMD_COMPLETE:
1859                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1860                         break;
1861                 case DRIVERMSG_NOTIFY:
1862                         SE_DEBUG(DBG_LVL_8, "Received DRIVERMSG_NOTIFY\n");
1863                         dmsg = (struct dmsg_cqe *)sol;
1864                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1865                         break;
1866                 case UNSOL_HDR_NOTIFY:
1867                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_HDR_ NOTIFY\n");
1868                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1869                                              (struct i_t_dpdu_cqe *)sol);
1870                         break;
1871                 case UNSOL_DATA_NOTIFY:
1872                         SE_DEBUG(DBG_LVL_8, "Received UNSOL_DATA_NOTIFY\n");
1873                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
1874                                              (struct i_t_dpdu_cqe *)sol);
1875                         break;
1876                 case CXN_INVALIDATE_INDEX_NOTIFY:
1877                 case CMD_INVALIDATED_NOTIFY:
1878                 case CXN_INVALIDATE_NOTIFY:
1879                         SE_DEBUG(DBG_LVL_1,
1880                                  "Ignoring CQ Error notification for cmd/cxn"
1881                                  "invalidate\n");
1882                         break;
1883                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1884                 case CMD_KILLED_INVALID_STATSN_RCVD:
1885                 case CMD_KILLED_INVALID_R2T_RCVD:
1886                 case CMD_CXN_KILLED_LUN_INVALID:
1887                 case CMD_CXN_KILLED_ICD_INVALID:
1888                 case CMD_CXN_KILLED_ITT_INVALID:
1889                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1890                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1891                         SE_DEBUG(DBG_LVL_1,
1892                                  "CQ Error notification for cmd.. "
1893                                  "code %d cid 0x%x\n",
1894                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1895                                  32] & CQE_CODE_MASK,
1896                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1897                                  32] & SOL_CID_MASK));
1898                         break;
1899                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1900                         SE_DEBUG(DBG_LVL_1,
1901                                  "Digest error on def pdu ring, dropping..\n");
1902                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
1903                                              (struct i_t_dpdu_cqe *) sol);
1904                         break;
1905                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1906                 case CXN_KILLED_BURST_LEN_MISMATCH:
1907                 case CXN_KILLED_AHS_RCVD:
1908                 case CXN_KILLED_HDR_DIGEST_ERR:
1909                 case CXN_KILLED_UNKNOWN_HDR:
1910                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1911                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
1912                 case CXN_KILLED_TIMED_OUT:
1913                 case CXN_KILLED_FIN_RCVD:
1914                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
1915                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
1916                 case CXN_KILLED_OVER_RUN_RESIDUAL:
1917                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
1918                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
1919                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset CID "
1920                                  "0x%x...\n",
1921                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1922                                  32] & CQE_CODE_MASK,
1923                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1924                                  32] & CQE_CID_MASK));
1925                         iscsi_conn_failure(beiscsi_conn->conn,
1926                                            ISCSI_ERR_CONN_FAILED);
1927                         break;
1928                 case CXN_KILLED_RST_SENT:
1929                 case CXN_KILLED_RST_RCVD:
1930                         SE_DEBUG(DBG_LVL_1, "CQ Error %d, reset"
1931                                 "received/sent on CID 0x%x...\n",
1932                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1933                                  32] & CQE_CODE_MASK,
1934                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1935                                  32] & CQE_CID_MASK));
1936                         iscsi_conn_failure(beiscsi_conn->conn,
1937                                            ISCSI_ERR_CONN_FAILED);
1938                         break;
1939                 default:
1940                         SE_DEBUG(DBG_LVL_1, "CQ Error Invalid code= %d "
1941                                  "received on CID 0x%x...\n",
1942                                  sol->dw[offsetof(struct amap_sol_cqe, code) /
1943                                  32] & CQE_CODE_MASK,
1944                                  (sol->dw[offsetof(struct amap_sol_cqe, cid) /
1945                                  32] & CQE_CID_MASK));
1946                         break;
1947                 }
1948
1949                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
1950                 queue_tail_inc(cq);
1951                 sol = queue_tail_node(cq);
1952                 num_processed++;
1953         }
1954
1955         if (num_processed > 0) {
1956                 tot_nump += num_processed;
1957                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
1958         }
1959         return tot_nump;
1960 }
1961
1962 void beiscsi_process_all_cqs(struct work_struct *work)
1963 {
1964         unsigned long flags;
1965         struct hwi_controller *phwi_ctrlr;
1966         struct hwi_context_memory *phwi_context;
1967         struct be_eq_obj *pbe_eq;
1968         struct beiscsi_hba *phba =
1969             container_of(work, struct beiscsi_hba, work_cqs);
1970
1971         phwi_ctrlr = phba->phwi_ctrlr;
1972         phwi_context = phwi_ctrlr->phwi_ctxt;
1973         if (phba->msix_enabled)
1974                 pbe_eq = &phwi_context->be_eq[phba->num_cpus];
1975         else
1976                 pbe_eq = &phwi_context->be_eq[0];
1977
1978         if (phba->todo_mcc_cq) {
1979                 spin_lock_irqsave(&phba->isr_lock, flags);
1980                 phba->todo_mcc_cq = 0;
1981                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1982                 beiscsi_process_mcc_isr(phba);
1983         }
1984
1985         if (phba->todo_cq) {
1986                 spin_lock_irqsave(&phba->isr_lock, flags);
1987                 phba->todo_cq = 0;
1988                 spin_unlock_irqrestore(&phba->isr_lock, flags);
1989                 beiscsi_process_cq(pbe_eq);
1990         }
1991 }
1992
1993 static int be_iopoll(struct blk_iopoll *iop, int budget)
1994 {
1995         static unsigned int ret;
1996         struct beiscsi_hba *phba;
1997         struct be_eq_obj *pbe_eq;
1998
1999         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2000         ret = beiscsi_process_cq(pbe_eq);
2001         if (ret < budget) {
2002                 phba = pbe_eq->phba;
2003                 blk_iopoll_complete(iop);
2004                 SE_DEBUG(DBG_LVL_8, "rearm pbe_eq->q.id =%d\n", pbe_eq->q.id);
2005                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2006         }
2007         return ret;
2008 }
2009
2010 static void
2011 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2012               unsigned int num_sg, struct beiscsi_io_task *io_task)
2013 {
2014         struct iscsi_sge *psgl;
2015         unsigned int sg_len, index;
2016         unsigned int sge_len = 0;
2017         unsigned long long addr;
2018         struct scatterlist *l_sg;
2019         unsigned int offset;
2020
2021         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2022                                       io_task->bhs_pa.u.a32.address_lo);
2023         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2024                                       io_task->bhs_pa.u.a32.address_hi);
2025
2026         l_sg = sg;
2027         for (index = 0; (index < num_sg) && (index < 2); index++,
2028                                                          sg = sg_next(sg)) {
2029                 if (index == 0) {
2030                         sg_len = sg_dma_len(sg);
2031                         addr = (u64) sg_dma_address(sg);
2032                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2033                                                 ((u32)(addr & 0xFFFFFFFF)));
2034                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2035                                                         ((u32)(addr >> 32)));
2036                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2037                                                         sg_len);
2038                         sge_len = sg_len;
2039                 } else {
2040                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2041                                                         pwrb, sge_len);
2042                         sg_len = sg_dma_len(sg);
2043                         addr = (u64) sg_dma_address(sg);
2044                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2045                                                 ((u32)(addr & 0xFFFFFFFF)));
2046                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2047                                                         ((u32)(addr >> 32)));
2048                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2049                                                         sg_len);
2050                 }
2051         }
2052         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2053         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2054
2055         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2056
2057         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2058                         io_task->bhs_pa.u.a32.address_hi);
2059         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2060                         io_task->bhs_pa.u.a32.address_lo);
2061
2062         if (num_sg == 1) {
2063                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2064                                                                 1);
2065                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2066                                                                 0);
2067         } else if (num_sg == 2) {
2068                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2069                                                                 0);
2070                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2071                                                                 1);
2072         } else {
2073                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2074                                                                 0);
2075                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2076                                                                 0);
2077         }
2078         sg = l_sg;
2079         psgl++;
2080         psgl++;
2081         offset = 0;
2082         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2083                 sg_len = sg_dma_len(sg);
2084                 addr = (u64) sg_dma_address(sg);
2085                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2086                                                 (addr & 0xFFFFFFFF));
2087                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2088                                                 (addr >> 32));
2089                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2090                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2091                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2092                 offset += sg_len;
2093         }
2094         psgl--;
2095         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2096 }
2097
2098 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2099 {
2100         struct iscsi_sge *psgl;
2101         unsigned long long addr;
2102         struct beiscsi_io_task *io_task = task->dd_data;
2103         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2104         struct beiscsi_hba *phba = beiscsi_conn->phba;
2105
2106         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2107         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2108                                 io_task->bhs_pa.u.a32.address_lo);
2109         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2110                                 io_task->bhs_pa.u.a32.address_hi);
2111
2112         if (task->data) {
2113                 if (task->data_count) {
2114                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2115                         addr = (u64) pci_map_single(phba->pcidev,
2116                                                     task->data,
2117                                                     task->data_count, 1);
2118                 } else {
2119                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2120                         addr = 0;
2121                 }
2122                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2123                                                 ((u32)(addr & 0xFFFFFFFF)));
2124                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2125                                                 ((u32)(addr >> 32)));
2126                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2127                                                 task->data_count);
2128
2129                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2130         } else {
2131                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2132                 addr = 0;
2133         }
2134
2135         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2136
2137         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2138
2139         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2140                       io_task->bhs_pa.u.a32.address_hi);
2141         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2142                       io_task->bhs_pa.u.a32.address_lo);
2143         if (task->data) {
2144                 psgl++;
2145                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2146                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2147                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2148                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2149                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2150                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2151
2152                 psgl++;
2153                 if (task->data) {
2154                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2155                                                 ((u32)(addr & 0xFFFFFFFF)));
2156                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2157                                                 ((u32)(addr >> 32)));
2158                 }
2159                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2160         }
2161         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2162 }
2163
2164 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2165 {
2166         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2167         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2168         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2169
2170         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2171                                       sizeof(struct sol_cqe));
2172         num_async_pdu_buf_pages =
2173                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2174                                        phba->params.defpdu_hdr_sz);
2175         num_async_pdu_buf_sgl_pages =
2176                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2177                                        sizeof(struct phys_addr));
2178         num_async_pdu_data_pages =
2179                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2180                                        phba->params.defpdu_data_sz);
2181         num_async_pdu_data_sgl_pages =
2182                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2183                                        sizeof(struct phys_addr));
2184
2185         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2186
2187         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2188                                                  BE_ISCSI_PDU_HEADER_SIZE;
2189         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2190                                             sizeof(struct hwi_context_memory);
2191
2192
2193         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2194             * (phba->params.wrbs_per_cxn)
2195             * phba->params.cxns_per_ctrl;
2196         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2197                                  (phba->params.wrbs_per_cxn);
2198         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2199                                 phba->params.cxns_per_ctrl);
2200
2201         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2202                 phba->params.icds_per_ctrl;
2203         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2204                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2205
2206         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2207                 num_async_pdu_buf_pages * PAGE_SIZE;
2208         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2209                 num_async_pdu_data_pages * PAGE_SIZE;
2210         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2211                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2212         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2213                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2214         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2215                 phba->params.asyncpdus_per_ctrl *
2216                 sizeof(struct async_pdu_handle);
2217         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2218                 phba->params.asyncpdus_per_ctrl *
2219                 sizeof(struct async_pdu_handle);
2220         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2221                 sizeof(struct hwi_async_pdu_context) +
2222                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2223 }
2224
2225 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2226 {
2227         struct be_mem_descriptor *mem_descr;
2228         dma_addr_t bus_add;
2229         struct mem_array *mem_arr, *mem_arr_orig;
2230         unsigned int i, j, alloc_size, curr_alloc_size;
2231
2232         phba->phwi_ctrlr = kmalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2233         if (!phba->phwi_ctrlr)
2234                 return -ENOMEM;
2235
2236         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2237                                  GFP_KERNEL);
2238         if (!phba->init_mem) {
2239                 kfree(phba->phwi_ctrlr);
2240                 return -ENOMEM;
2241         }
2242
2243         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2244                                GFP_KERNEL);
2245         if (!mem_arr_orig) {
2246                 kfree(phba->init_mem);
2247                 kfree(phba->phwi_ctrlr);
2248                 return -ENOMEM;
2249         }
2250
2251         mem_descr = phba->init_mem;
2252         for (i = 0; i < SE_MEM_MAX; i++) {
2253                 j = 0;
2254                 mem_arr = mem_arr_orig;
2255                 alloc_size = phba->mem_req[i];
2256                 memset(mem_arr, 0, sizeof(struct mem_array) *
2257                        BEISCSI_MAX_FRAGS_INIT);
2258                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2259                 do {
2260                         mem_arr->virtual_address = pci_alloc_consistent(
2261                                                         phba->pcidev,
2262                                                         curr_alloc_size,
2263                                                         &bus_add);
2264                         if (!mem_arr->virtual_address) {
2265                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2266                                         goto free_mem;
2267                                 if (curr_alloc_size -
2268                                         rounddown_pow_of_two(curr_alloc_size))
2269                                         curr_alloc_size = rounddown_pow_of_two
2270                                                              (curr_alloc_size);
2271                                 else
2272                                         curr_alloc_size = curr_alloc_size / 2;
2273                         } else {
2274                                 mem_arr->bus_address.u.
2275                                     a64.address = (__u64) bus_add;
2276                                 mem_arr->size = curr_alloc_size;
2277                                 alloc_size -= curr_alloc_size;
2278                                 curr_alloc_size = min(be_max_phys_size *
2279                                                       1024, alloc_size);
2280                                 j++;
2281                                 mem_arr++;
2282                         }
2283                 } while (alloc_size);
2284                 mem_descr->num_elements = j;
2285                 mem_descr->size_in_bytes = phba->mem_req[i];
2286                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2287                                                GFP_KERNEL);
2288                 if (!mem_descr->mem_array)
2289                         goto free_mem;
2290
2291                 memcpy(mem_descr->mem_array, mem_arr_orig,
2292                        sizeof(struct mem_array) * j);
2293                 mem_descr++;
2294         }
2295         kfree(mem_arr_orig);
2296         return 0;
2297 free_mem:
2298         mem_descr->num_elements = j;
2299         while ((i) || (j)) {
2300                 for (j = mem_descr->num_elements; j > 0; j--) {
2301                         pci_free_consistent(phba->pcidev,
2302                                             mem_descr->mem_array[j - 1].size,
2303                                             mem_descr->mem_array[j - 1].
2304                                             virtual_address,
2305                                             (unsigned long)mem_descr->
2306                                             mem_array[j - 1].
2307                                             bus_address.u.a64.address);
2308                 }
2309                 if (i) {
2310                         i--;
2311                         kfree(mem_descr->mem_array);
2312                         mem_descr--;
2313                 }
2314         }
2315         kfree(mem_arr_orig);
2316         kfree(phba->init_mem);
2317         kfree(phba->phwi_ctrlr);
2318         return -ENOMEM;
2319 }
2320
2321 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2322 {
2323         beiscsi_find_mem_req(phba);
2324         return beiscsi_alloc_mem(phba);
2325 }
2326
2327 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2328 {
2329         struct pdu_data_out *pdata_out;
2330         struct pdu_nop_out *pnop_out;
2331         struct be_mem_descriptor *mem_descr;
2332
2333         mem_descr = phba->init_mem;
2334         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2335         pdata_out =
2336             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2337         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2338
2339         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2340                       IIOC_SCSI_DATA);
2341
2342         pnop_out =
2343             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2344                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2345
2346         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2347         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2348         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2349         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2350 }
2351
2352 static void beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2353 {
2354         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2355         struct wrb_handle *pwrb_handle;
2356         struct hwi_controller *phwi_ctrlr;
2357         struct hwi_wrb_context *pwrb_context;
2358         struct iscsi_wrb *pwrb;
2359         unsigned int num_cxn_wrbh;
2360         unsigned int num_cxn_wrb, j, idx, index;
2361
2362         mem_descr_wrbh = phba->init_mem;
2363         mem_descr_wrbh += HWI_MEM_WRBH;
2364
2365         mem_descr_wrb = phba->init_mem;
2366         mem_descr_wrb += HWI_MEM_WRB;
2367
2368         idx = 0;
2369         pwrb_handle = mem_descr_wrbh->mem_array[idx].virtual_address;
2370         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2371                         ((sizeof(struct wrb_handle)) *
2372                          phba->params.wrbs_per_cxn));
2373         phwi_ctrlr = phba->phwi_ctrlr;
2374
2375         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2376                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2377                 pwrb_context->pwrb_handle_base =
2378                                 kzalloc(sizeof(struct wrb_handle *) *
2379                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2380                 pwrb_context->pwrb_handle_basestd =
2381                                 kzalloc(sizeof(struct wrb_handle *) *
2382                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2383                 if (num_cxn_wrbh) {
2384                         pwrb_context->alloc_index = 0;
2385                         pwrb_context->wrb_handles_available = 0;
2386                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2387                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2388                                 pwrb_context->pwrb_handle_basestd[j] =
2389                                                                 pwrb_handle;
2390                                 pwrb_context->wrb_handles_available++;
2391                                 pwrb_handle->wrb_index = j;
2392                                 pwrb_handle++;
2393                         }
2394                         pwrb_context->free_index = 0;
2395                         num_cxn_wrbh--;
2396                 } else {
2397                         idx++;
2398                         pwrb_handle =
2399                             mem_descr_wrbh->mem_array[idx].virtual_address;
2400                         num_cxn_wrbh =
2401                             ((mem_descr_wrbh->mem_array[idx].size) /
2402                              ((sizeof(struct wrb_handle)) *
2403                               phba->params.wrbs_per_cxn));
2404                         pwrb_context->alloc_index = 0;
2405                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2406                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2407                                 pwrb_context->pwrb_handle_basestd[j] =
2408                                     pwrb_handle;
2409                                 pwrb_context->wrb_handles_available++;
2410                                 pwrb_handle->wrb_index = j;
2411                                 pwrb_handle++;
2412                         }
2413                         pwrb_context->free_index = 0;
2414                         num_cxn_wrbh--;
2415                 }
2416         }
2417         idx = 0;
2418         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2419         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2420                       ((sizeof(struct iscsi_wrb) *
2421                         phba->params.wrbs_per_cxn));
2422         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2423                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2424                 if (num_cxn_wrb) {
2425                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2426                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2427                                 pwrb_handle->pwrb = pwrb;
2428                                 pwrb++;
2429                         }
2430                         num_cxn_wrb--;
2431                 } else {
2432                         idx++;
2433                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2434                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2435                                       ((sizeof(struct iscsi_wrb) *
2436                                         phba->params.wrbs_per_cxn));
2437                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2438                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2439                                 pwrb_handle->pwrb = pwrb;
2440                                 pwrb++;
2441                         }
2442                         num_cxn_wrb--;
2443                 }
2444         }
2445 }
2446
2447 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2448 {
2449         struct hwi_controller *phwi_ctrlr;
2450         struct hba_parameters *p = &phba->params;
2451         struct hwi_async_pdu_context *pasync_ctx;
2452         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2453         unsigned int index;
2454         struct be_mem_descriptor *mem_descr;
2455
2456         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2457         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2458
2459         phwi_ctrlr = phba->phwi_ctrlr;
2460         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2461                                 mem_descr->mem_array[0].virtual_address;
2462         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2463         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2464
2465         pasync_ctx->async_header.num_entries = p->asyncpdus_per_ctrl;
2466         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2467         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2468         pasync_ctx->async_data.num_entries = p->asyncpdus_per_ctrl;
2469
2470         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2471         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2472         if (mem_descr->mem_array[0].virtual_address) {
2473                 SE_DEBUG(DBG_LVL_8,
2474                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2475                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2476         } else
2477                 shost_printk(KERN_WARNING, phba->shost,
2478                              "No Virtual address\n");
2479
2480         pasync_ctx->async_header.va_base =
2481                         mem_descr->mem_array[0].virtual_address;
2482
2483         pasync_ctx->async_header.pa_base.u.a64.address =
2484                         mem_descr->mem_array[0].bus_address.u.a64.address;
2485
2486         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2487         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2488         if (mem_descr->mem_array[0].virtual_address) {
2489                 SE_DEBUG(DBG_LVL_8,
2490                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2491                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2492         } else
2493                 shost_printk(KERN_WARNING, phba->shost,
2494                             "No Virtual address\n");
2495         pasync_ctx->async_header.ring_base =
2496                         mem_descr->mem_array[0].virtual_address;
2497
2498         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2499         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2500         if (mem_descr->mem_array[0].virtual_address) {
2501                 SE_DEBUG(DBG_LVL_8,
2502                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2503                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2504         } else
2505                 shost_printk(KERN_WARNING, phba->shost,
2506                             "No Virtual address\n");
2507
2508         pasync_ctx->async_header.handle_base =
2509                         mem_descr->mem_array[0].virtual_address;
2510         pasync_ctx->async_header.writables = 0;
2511         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2512
2513         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2514         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2515         if (mem_descr->mem_array[0].virtual_address) {
2516                 SE_DEBUG(DBG_LVL_8,
2517                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2518                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2519         } else
2520                 shost_printk(KERN_WARNING, phba->shost,
2521                             "No Virtual address\n");
2522         pasync_ctx->async_data.va_base =
2523                         mem_descr->mem_array[0].virtual_address;
2524         pasync_ctx->async_data.pa_base.u.a64.address =
2525                         mem_descr->mem_array[0].bus_address.u.a64.address;
2526
2527         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2528         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2529         if (mem_descr->mem_array[0].virtual_address) {
2530                 SE_DEBUG(DBG_LVL_8,
2531                          "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2532                          "va=%p\n", mem_descr->mem_array[0].virtual_address);
2533         } else
2534                 shost_printk(KERN_WARNING, phba->shost,
2535                              "No Virtual address\n");
2536
2537         pasync_ctx->async_data.ring_base =
2538                         mem_descr->mem_array[0].virtual_address;
2539
2540         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2541         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2542         if (!mem_descr->mem_array[0].virtual_address)
2543                 shost_printk(KERN_WARNING, phba->shost,
2544                             "No Virtual address\n");
2545
2546         pasync_ctx->async_data.handle_base =
2547                         mem_descr->mem_array[0].virtual_address;
2548         pasync_ctx->async_data.writables = 0;
2549         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2550
2551         pasync_header_h =
2552                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2553         pasync_data_h =
2554                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2555
2556         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2557                 pasync_header_h->cri = -1;
2558                 pasync_header_h->index = (char)index;
2559                 INIT_LIST_HEAD(&pasync_header_h->link);
2560                 pasync_header_h->pbuffer =
2561                         (void *)((unsigned long)
2562                         (pasync_ctx->async_header.va_base) +
2563                         (p->defpdu_hdr_sz * index));
2564
2565                 pasync_header_h->pa.u.a64.address =
2566                         pasync_ctx->async_header.pa_base.u.a64.address +
2567                         (p->defpdu_hdr_sz * index);
2568
2569                 list_add_tail(&pasync_header_h->link,
2570                                 &pasync_ctx->async_header.free_list);
2571                 pasync_header_h++;
2572                 pasync_ctx->async_header.free_entries++;
2573                 pasync_ctx->async_header.writables++;
2574
2575                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2576                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2577                                header_busy_list);
2578                 pasync_data_h->cri = -1;
2579                 pasync_data_h->index = (char)index;
2580                 INIT_LIST_HEAD(&pasync_data_h->link);
2581                 pasync_data_h->pbuffer =
2582                         (void *)((unsigned long)
2583                         (pasync_ctx->async_data.va_base) +
2584                         (p->defpdu_data_sz * index));
2585
2586                 pasync_data_h->pa.u.a64.address =
2587                     pasync_ctx->async_data.pa_base.u.a64.address +
2588                     (p->defpdu_data_sz * index);
2589
2590                 list_add_tail(&pasync_data_h->link,
2591                               &pasync_ctx->async_data.free_list);
2592                 pasync_data_h++;
2593                 pasync_ctx->async_data.free_entries++;
2594                 pasync_ctx->async_data.writables++;
2595
2596                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2597         }
2598
2599         pasync_ctx->async_header.host_write_ptr = 0;
2600         pasync_ctx->async_header.ep_read_ptr = -1;
2601         pasync_ctx->async_data.host_write_ptr = 0;
2602         pasync_ctx->async_data.ep_read_ptr = -1;
2603 }
2604
2605 static int
2606 be_sgl_create_contiguous(void *virtual_address,
2607                          u64 physical_address, u32 length,
2608                          struct be_dma_mem *sgl)
2609 {
2610         WARN_ON(!virtual_address);
2611         WARN_ON(!physical_address);
2612         WARN_ON(!length > 0);
2613         WARN_ON(!sgl);
2614
2615         sgl->va = virtual_address;
2616         sgl->dma = (unsigned long)physical_address;
2617         sgl->size = length;
2618
2619         return 0;
2620 }
2621
2622 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2623 {
2624         memset(sgl, 0, sizeof(*sgl));
2625 }
2626
2627 static void
2628 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2629                      struct mem_array *pmem, struct be_dma_mem *sgl)
2630 {
2631         if (sgl->va)
2632                 be_sgl_destroy_contiguous(sgl);
2633
2634         be_sgl_create_contiguous(pmem->virtual_address,
2635                                  pmem->bus_address.u.a64.address,
2636                                  pmem->size, sgl);
2637 }
2638
2639 static void
2640 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2641                            struct mem_array *pmem, struct be_dma_mem *sgl)
2642 {
2643         if (sgl->va)
2644                 be_sgl_destroy_contiguous(sgl);
2645
2646         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2647                                  pmem->bus_address.u.a64.address,
2648                                  pmem->size, sgl);
2649 }
2650
2651 static int be_fill_queue(struct be_queue_info *q,
2652                 u16 len, u16 entry_size, void *vaddress)
2653 {
2654         struct be_dma_mem *mem = &q->dma_mem;
2655
2656         memset(q, 0, sizeof(*q));
2657         q->len = len;
2658         q->entry_size = entry_size;
2659         mem->size = len * entry_size;
2660         mem->va = vaddress;
2661         if (!mem->va)
2662                 return -ENOMEM;
2663         memset(mem->va, 0, mem->size);
2664         return 0;
2665 }
2666
2667 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2668                              struct hwi_context_memory *phwi_context)
2669 {
2670         unsigned int i, num_eq_pages;
2671         int ret, eq_for_mcc;
2672         struct be_queue_info *eq;
2673         struct be_dma_mem *mem;
2674         void *eq_vaddress;
2675         dma_addr_t paddr;
2676
2677         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2678                                       sizeof(struct be_eq_entry));
2679
2680         if (phba->msix_enabled)
2681                 eq_for_mcc = 1;
2682         else
2683                 eq_for_mcc = 0;
2684         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2685                 eq = &phwi_context->be_eq[i].q;
2686                 mem = &eq->dma_mem;
2687                 phwi_context->be_eq[i].phba = phba;
2688                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2689                                                      num_eq_pages * PAGE_SIZE,
2690                                                      &paddr);
2691                 if (!eq_vaddress)
2692                         goto create_eq_error;
2693
2694                 mem->va = eq_vaddress;
2695                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2696                                     sizeof(struct be_eq_entry), eq_vaddress);
2697                 if (ret) {
2698                         shost_printk(KERN_ERR, phba->shost,
2699                                      "be_fill_queue Failed for EQ\n");
2700                         goto create_eq_error;
2701                 }
2702
2703                 mem->dma = paddr;
2704                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2705                                             phwi_context->cur_eqd);
2706                 if (ret) {
2707                         shost_printk(KERN_ERR, phba->shost,
2708                                      "beiscsi_cmd_eq_create"
2709                                      "Failedfor EQ\n");
2710                         goto create_eq_error;
2711                 }
2712                 SE_DEBUG(DBG_LVL_8, "eqid = %d\n", phwi_context->be_eq[i].q.id);
2713         }
2714         return 0;
2715 create_eq_error:
2716         for (i = 0; i < (phba->num_cpus + 1); i++) {
2717                 eq = &phwi_context->be_eq[i].q;
2718                 mem = &eq->dma_mem;
2719                 if (mem->va)
2720                         pci_free_consistent(phba->pcidev, num_eq_pages
2721                                             * PAGE_SIZE,
2722                                             mem->va, mem->dma);
2723         }
2724         return ret;
2725 }
2726
2727 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2728                              struct hwi_context_memory *phwi_context)
2729 {
2730         unsigned int i, num_cq_pages;
2731         int ret;
2732         struct be_queue_info *cq, *eq;
2733         struct be_dma_mem *mem;
2734         struct be_eq_obj *pbe_eq;
2735         void *cq_vaddress;
2736         dma_addr_t paddr;
2737
2738         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2739                                       sizeof(struct sol_cqe));
2740
2741         for (i = 0; i < phba->num_cpus; i++) {
2742                 cq = &phwi_context->be_cq[i];
2743                 eq = &phwi_context->be_eq[i].q;
2744                 pbe_eq = &phwi_context->be_eq[i];
2745                 pbe_eq->cq = cq;
2746                 pbe_eq->phba = phba;
2747                 mem = &cq->dma_mem;
2748                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2749                                                      num_cq_pages * PAGE_SIZE,
2750                                                      &paddr);
2751                 if (!cq_vaddress)
2752                         goto create_cq_error;
2753                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2754                                     sizeof(struct sol_cqe), cq_vaddress);
2755                 if (ret) {
2756                         shost_printk(KERN_ERR, phba->shost,
2757                                      "be_fill_queue Failed for ISCSI CQ\n");
2758                         goto create_cq_error;
2759                 }
2760
2761                 mem->dma = paddr;
2762                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2763                                             false, 0);
2764                 if (ret) {
2765                         shost_printk(KERN_ERR, phba->shost,
2766                                      "beiscsi_cmd_eq_create"
2767                                      "Failed for ISCSI CQ\n");
2768                         goto create_cq_error;
2769                 }
2770                 SE_DEBUG(DBG_LVL_8, "iscsi cq_id is %d for eq_id %d\n",
2771                                                  cq->id, eq->id);
2772                 SE_DEBUG(DBG_LVL_8, "ISCSI CQ CREATED\n");
2773         }
2774         return 0;
2775
2776 create_cq_error:
2777         for (i = 0; i < phba->num_cpus; i++) {
2778                 cq = &phwi_context->be_cq[i];
2779                 mem = &cq->dma_mem;
2780                 if (mem->va)
2781                         pci_free_consistent(phba->pcidev, num_cq_pages
2782                                             * PAGE_SIZE,
2783                                             mem->va, mem->dma);
2784         }
2785         return ret;
2786
2787 }
2788
2789 static int
2790 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2791                        struct hwi_context_memory *phwi_context,
2792                        struct hwi_controller *phwi_ctrlr,
2793                        unsigned int def_pdu_ring_sz)
2794 {
2795         unsigned int idx;
2796         int ret;
2797         struct be_queue_info *dq, *cq;
2798         struct be_dma_mem *mem;
2799         struct be_mem_descriptor *mem_descr;
2800         void *dq_vaddress;
2801
2802         idx = 0;
2803         dq = &phwi_context->be_def_hdrq;
2804         cq = &phwi_context->be_cq[0];
2805         mem = &dq->dma_mem;
2806         mem_descr = phba->init_mem;
2807         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2808         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2809         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2810                             sizeof(struct phys_addr),
2811                             sizeof(struct phys_addr), dq_vaddress);
2812         if (ret) {
2813                 shost_printk(KERN_ERR, phba->shost,
2814                              "be_fill_queue Failed for DEF PDU HDR\n");
2815                 return ret;
2816         }
2817         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2818                                   bus_address.u.a64.address;
2819         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2820                                               def_pdu_ring_sz,
2821                                               phba->params.defpdu_hdr_sz);
2822         if (ret) {
2823                 shost_printk(KERN_ERR, phba->shost,
2824                              "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2825                 return ret;
2826         }
2827         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
2828         SE_DEBUG(DBG_LVL_8, "iscsi def pdu id is %d\n",
2829                  phwi_context->be_def_hdrq.id);
2830         hwi_post_async_buffers(phba, 1);
2831         return 0;
2832 }
2833
2834 static int
2835 beiscsi_create_def_data(struct beiscsi_hba *phba,
2836                         struct hwi_context_memory *phwi_context,
2837                         struct hwi_controller *phwi_ctrlr,
2838                         unsigned int def_pdu_ring_sz)
2839 {
2840         unsigned int idx;
2841         int ret;
2842         struct be_queue_info *dataq, *cq;
2843         struct be_dma_mem *mem;
2844         struct be_mem_descriptor *mem_descr;
2845         void *dq_vaddress;
2846
2847         idx = 0;
2848         dataq = &phwi_context->be_def_dataq;
2849         cq = &phwi_context->be_cq[0];
2850         mem = &dataq->dma_mem;
2851         mem_descr = phba->init_mem;
2852         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2853         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2854         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
2855                             sizeof(struct phys_addr),
2856                             sizeof(struct phys_addr), dq_vaddress);
2857         if (ret) {
2858                 shost_printk(KERN_ERR, phba->shost,
2859                              "be_fill_queue Failed for DEF PDU DATA\n");
2860                 return ret;
2861         }
2862         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2863                                   bus_address.u.a64.address;
2864         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
2865                                               def_pdu_ring_sz,
2866                                               phba->params.defpdu_data_sz);
2867         if (ret) {
2868                 shost_printk(KERN_ERR, phba->shost,
2869                              "be_cmd_create_default_pdu_queue Failed"
2870                              " for DEF PDU DATA\n");
2871                 return ret;
2872         }
2873         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
2874         SE_DEBUG(DBG_LVL_8, "iscsi def data id is %d\n",
2875                  phwi_context->be_def_dataq.id);
2876         hwi_post_async_buffers(phba, 0);
2877         SE_DEBUG(DBG_LVL_8, "DEFAULT PDU DATA RING CREATED\n");
2878         return 0;
2879 }
2880
2881 static int
2882 beiscsi_post_pages(struct beiscsi_hba *phba)
2883 {
2884         struct be_mem_descriptor *mem_descr;
2885         struct mem_array *pm_arr;
2886         unsigned int page_offset, i;
2887         struct be_dma_mem sgl;
2888         int status;
2889
2890         mem_descr = phba->init_mem;
2891         mem_descr += HWI_MEM_SGE;
2892         pm_arr = mem_descr->mem_array;
2893
2894         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
2895                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
2896         for (i = 0; i < mem_descr->num_elements; i++) {
2897                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
2898                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
2899                                                 page_offset,
2900                                                 (pm_arr->size / PAGE_SIZE));
2901                 page_offset += pm_arr->size / PAGE_SIZE;
2902                 if (status != 0) {
2903                         shost_printk(KERN_ERR, phba->shost,
2904                                      "post sgl failed.\n");
2905                         return status;
2906                 }
2907                 pm_arr++;
2908         }
2909         SE_DEBUG(DBG_LVL_8, "POSTED PAGES\n");
2910         return 0;
2911 }
2912
2913 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
2914 {
2915         struct be_dma_mem *mem = &q->dma_mem;
2916         if (mem->va)
2917                 pci_free_consistent(phba->pcidev, mem->size,
2918                         mem->va, mem->dma);
2919 }
2920
2921 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
2922                 u16 len, u16 entry_size)
2923 {
2924         struct be_dma_mem *mem = &q->dma_mem;
2925
2926         memset(q, 0, sizeof(*q));
2927         q->len = len;
2928         q->entry_size = entry_size;
2929         mem->size = len * entry_size;
2930         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
2931         if (!mem->va)
2932                 return -ENOMEM;
2933         memset(mem->va, 0, mem->size);
2934         return 0;
2935 }
2936
2937 static int
2938 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
2939                          struct hwi_context_memory *phwi_context,
2940                          struct hwi_controller *phwi_ctrlr)
2941 {
2942         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
2943         u64 pa_addr_lo;
2944         unsigned int idx, num, i;
2945         struct mem_array *pwrb_arr;
2946         void *wrb_vaddr;
2947         struct be_dma_mem sgl;
2948         struct be_mem_descriptor *mem_descr;
2949         int status;
2950
2951         idx = 0;
2952         mem_descr = phba->init_mem;
2953         mem_descr += HWI_MEM_WRB;
2954         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
2955                            GFP_KERNEL);
2956         if (!pwrb_arr) {
2957                 shost_printk(KERN_ERR, phba->shost,
2958                              "Memory alloc failed in create wrb ring.\n");
2959                 return -ENOMEM;
2960         }
2961         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2962         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
2963         num_wrb_rings = mem_descr->mem_array[idx].size /
2964                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
2965
2966         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
2967                 if (num_wrb_rings) {
2968                         pwrb_arr[num].virtual_address = wrb_vaddr;
2969                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
2970                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2971                                             sizeof(struct iscsi_wrb);
2972                         wrb_vaddr += pwrb_arr[num].size;
2973                         pa_addr_lo += pwrb_arr[num].size;
2974                         num_wrb_rings--;
2975                 } else {
2976                         idx++;
2977                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
2978                         pa_addr_lo = mem_descr->mem_array[idx].\
2979                                         bus_address.u.a64.address;
2980                         num_wrb_rings = mem_descr->mem_array[idx].size /
2981                                         (phba->params.wrbs_per_cxn *
2982                                         sizeof(struct iscsi_wrb));
2983                         pwrb_arr[num].virtual_address = wrb_vaddr;
2984                         pwrb_arr[num].bus_address.u.a64.address\
2985                                                 = pa_addr_lo;
2986                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
2987                                                  sizeof(struct iscsi_wrb);
2988                         wrb_vaddr += pwrb_arr[num].size;
2989                         pa_addr_lo   += pwrb_arr[num].size;
2990                         num_wrb_rings--;
2991                 }
2992         }
2993         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
2994                 wrb_mem_index = 0;
2995                 offset = 0;
2996                 size = 0;
2997
2998                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
2999                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3000                                             &phwi_context->be_wrbq[i]);
3001                 if (status != 0) {
3002                         shost_printk(KERN_ERR, phba->shost,
3003                                      "wrbq create failed.");
3004                         kfree(pwrb_arr);
3005                         return status;
3006                 }
3007                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3008                                                                    id;
3009         }
3010         kfree(pwrb_arr);
3011         return 0;
3012 }